Implemented some monitors
This commit is contained in:
Marc
parent
e38e70d539
commit
c54c084dff
5
TODO.md
5
TODO.md
|
|
@ -49,3 +49,8 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
|
|||
* pip install pyinotify
|
||||
|
||||
* create custom field that returns backend python objects
|
||||
|
||||
* Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers)
|
||||
* Resource metric: KB MB B?
|
||||
|
||||
* EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import datetime
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
def generate_zone_serial():
|
||||
today = datetime.date.today()
|
||||
today = timezone.now()
|
||||
return int("%.4d%.2d%.2d%.2d" % (today.year, today.month, today.day, 0))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
from django.template import Template, Context
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from orchestra.apps.orchestration import ServiceController
|
||||
from orchestra.apps.resources import ServiceMonitor
|
||||
|
||||
from . import settings
|
||||
|
||||
|
||||
class MailmanBackend(ServiceController):
|
||||
verbose_name = "Mailman"
|
||||
|
|
@ -14,28 +17,22 @@ class MailmanTraffic(ServiceMonitor):
|
|||
model = 'lists.List'
|
||||
resource = ServiceMonitor.TRAFFIC
|
||||
|
||||
def process(self, output):
|
||||
for line in output.readlines():
|
||||
listname, value = line.strip().slpit()
|
||||
def monitor(self, mail_list):
|
||||
context = self.get_context(mail_list)
|
||||
self.append(
|
||||
"SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n"
|
||||
"SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n"
|
||||
" | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n"
|
||||
" | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n"
|
||||
" | tr '\\n' '+' \\\n"
|
||||
" | xargs -i echo {}0 )\n"
|
||||
"echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context)
|
||||
|
||||
def monitor(self, mailinglist):
|
||||
self.append(
|
||||
"LISTS=$(grep -v 'post to mailman' /var/log/mailman/post"
|
||||
" | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq"
|
||||
" | while read line; do \n"
|
||||
" grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}"
|
||||
" | sed 's/size=//' | sed 's/,//'\n"
|
||||
"done)"
|
||||
)
|
||||
self.append(
|
||||
'SUBS=""\n'
|
||||
'while read LIST; do\n'
|
||||
' NAME=$(echo "$LIST" | awk {\'print $1\'})\n'
|
||||
' SIZE=$(echo "$LIST" | awk {\'print $2\'})\n'
|
||||
' if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n'
|
||||
' SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n'
|
||||
' fi\n'
|
||||
' SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n'
|
||||
' echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n'
|
||||
'done <<< "$LISTS"'
|
||||
)
|
||||
def get_context(self, mail_list):
|
||||
return {
|
||||
'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH,
|
||||
'list_name': mail_list.name,
|
||||
'object_id': mail_list.pk,
|
||||
'last_date': timezone.localtime(self.get_last_date(mail_list)).strftime("%b %d %H:%M:%S"),
|
||||
'current_date': timezone.localtime(self.get_current_date()).strftime("%b %d %H:%M:%S"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class ListCreationForm(CleanAddressMixin, forms.ModelForm):
|
|||
help_text=_("Enter the same password as above, for verification."))
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ListAdminForm, self).__init__(*args, **kwargs)
|
||||
super(ListCreationForm, self).__init__(*args, **kwargs)
|
||||
self.fields['password1'].validators.append(validate_password)
|
||||
|
||||
def clean_password2(self):
|
||||
|
|
@ -38,7 +38,7 @@ class ListCreationForm(CleanAddressMixin, forms.ModelForm):
|
|||
return password2
|
||||
|
||||
def save(self, commit=True):
|
||||
obj = super(ListAdminForm, self).save(commit=commit)
|
||||
obj = super(ListCreationForm, self).save(commit=commit)
|
||||
obj.set_password(self.cleaned_data["password1"])
|
||||
return obj
|
||||
|
||||
|
|
|
|||
|
|
@ -6,3 +6,6 @@ from django.conf import settings
|
|||
LISTS_DOMAIN_MODEL = getattr(settings, 'LISTS_DOMAIN_MODEL', 'domains.Domain')
|
||||
|
||||
LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'grups.orchestra.lan')
|
||||
|
||||
LISTS_MAILMAN_POST_LOG_PATH = getattr(settings, 'LISTS_MAILMAN_POST_LOG_PATH',
|
||||
'/var/log/mailman/post')
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from datetime import datetime
|
||||
from functools import partial
|
||||
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from orchestra.utils import plugins
|
||||
|
|
@ -89,7 +89,7 @@ class ServiceBackend(object):
|
|||
return sorted(choices, key=lambda e: e[0])
|
||||
|
||||
def get_banner(self):
|
||||
time = datetime.now().strftime("%h %d, %Y %I:%M:%S")
|
||||
time = timezone.now().strftime("%h %d, %Y %I:%M:%S")
|
||||
return "Generated by Orchestra %s" % time
|
||||
|
||||
def execute(self, server):
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ from .helpers import send_report
|
|||
|
||||
def as_task(execute):
|
||||
def wrapper(*args, **kwargs):
|
||||
with db.transaction.commit_manually():
|
||||
# with db.transaction.commit_manually():
|
||||
log = execute(*args, **kwargs)
|
||||
db.transaction.commit()
|
||||
# db.transaction.commit()
|
||||
if log.state != log.SUCCESS:
|
||||
send_report(execute, args, log)
|
||||
return log
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import datetime
|
||||
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.utils import timezone
|
||||
|
||||
from orchestra.apps.orchestration import ServiceBackend
|
||||
from orchestra.utils.functional import cached
|
||||
|
|
@ -17,14 +18,14 @@ class ServiceMonitor(ServiceBackend):
|
|||
@classmethod
|
||||
def get_backends(cls):
|
||||
""" filter monitor classes """
|
||||
return [plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__]
|
||||
for backend in cls.plugins:
|
||||
if backend != ServiceMonitor and ServiceMonitor in backend.__mro__:
|
||||
yield backend
|
||||
|
||||
@cached
|
||||
def get_last_date(self, obj):
|
||||
from .models import MonitorData
|
||||
try:
|
||||
# TODO replace
|
||||
#return MonitorData.objects.filter(content_object=obj).latest().date
|
||||
ct = ContentType.objects.get_for_model(type(obj))
|
||||
return MonitorData.objects.filter(content_type=ct, object_id=obj.pk).latest().date
|
||||
except MonitorData.DoesNotExist:
|
||||
|
|
@ -32,7 +33,7 @@ class ServiceMonitor(ServiceBackend):
|
|||
|
||||
@cached
|
||||
def get_current_date(self):
|
||||
return datetime.datetime.now()
|
||||
return timezone.now()
|
||||
|
||||
def store(self, log):
|
||||
""" object_id value """
|
||||
|
|
|
|||
|
|
@ -36,22 +36,27 @@ class Resource(models.Model):
|
|||
verbose_name = models.CharField(_("verbose name"), max_length=256, unique=True)
|
||||
content_type = models.ForeignKey(ContentType,
|
||||
help_text=_("Model where this resource will be hooked"))
|
||||
period = models.CharField(_("period"), max_length=16, choices=PERIODS, default=LAST,
|
||||
help_text=_("Operation used for aggregating this resource monitored data."))
|
||||
period = models.CharField(_("period"), max_length=16, choices=PERIODS,
|
||||
default=LAST,
|
||||
help_text=_("Operation used for aggregating this resource monitored"
|
||||
"data."))
|
||||
ondemand = models.BooleanField(_("on demand"), default=False,
|
||||
help_text=_("If enabled the resource will not be pre-allocated, "
|
||||
"but allocated under the application demand"))
|
||||
default_allocation = models.PositiveIntegerField(_("default allocation"),
|
||||
null=True, blank=True,
|
||||
help_text=_("Default allocation value used when this is not an "
|
||||
"on demand resource"),
|
||||
null=True, blank=True)
|
||||
"on demand resource"))
|
||||
is_active = models.BooleanField(_("is active"), default=True)
|
||||
disable_trigger = models.BooleanField(_("disable trigger"), default=False,
|
||||
help_text=_("Disables monitor's resource exeeded and recovery triggers"))
|
||||
help_text=_("Disables monitors exeeded and recovery triggers"))
|
||||
crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"),
|
||||
help_text=_("Crontab for periodic execution"))
|
||||
monitors = MultiSelectField(_("monitors"), max_length=256,
|
||||
choices=ServiceMonitor.get_choices())
|
||||
null=True, blank=True,
|
||||
help_text=_("Crontab for periodic execution. "
|
||||
"Leave it empty to disable periodic monitoring"))
|
||||
monitors = MultiSelectField(_("monitors"), max_length=256, blank=True,
|
||||
choices=ServiceMonitor.get_choices(),
|
||||
help_text=_("Monitor backends used for monitoring this resource."))
|
||||
|
||||
def __unicode__(self):
|
||||
return self.name
|
||||
|
|
@ -83,7 +88,8 @@ class Resource(models.Model):
|
|||
def group_by_content_type(cls):
|
||||
prev = None
|
||||
group = []
|
||||
for resource in cls.objects.filter(is_active=True).order_by('content_type'):
|
||||
resources = cls.objects.filter(is_active=True).order_by('content_type')
|
||||
for resource in resources:
|
||||
ct = resource.content_type
|
||||
if prev != ct:
|
||||
if group:
|
||||
|
|
@ -121,7 +127,7 @@ class ResourceData(models.Model):
|
|||
|
||||
def get_used(self):
|
||||
resource = self.resource
|
||||
today = datetime.date.today()
|
||||
today = timezone.now()
|
||||
result = 0
|
||||
has_result = False
|
||||
for monitor in resource.monitors:
|
||||
|
|
@ -133,7 +139,8 @@ class ResourceData(models.Model):
|
|||
except MonitorData.DoesNotExist:
|
||||
continue
|
||||
has_result = True
|
||||
epoch = datetime(year=today.year, month=today.month, day=1)
|
||||
epoch = datetime(year=today.year, month=today.month, day=1,
|
||||
tzinfo=timezone.utc)
|
||||
total = (epoch-last.date).total_seconds()
|
||||
dataset = dataset.filter(date__year=today.year,
|
||||
date__month=today.month)
|
||||
|
|
@ -154,7 +161,8 @@ class ResourceData(models.Model):
|
|||
continue
|
||||
has_result = True
|
||||
else:
|
||||
raise NotImplementedError("%s support not implemented" % self.period)
|
||||
msg = "%s support not implemented" % self.period
|
||||
raise NotImplementedError(msg)
|
||||
return result if has_result else None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ class ResourceSerializer(serializers.ModelSerializer):
|
|||
# Monkey-patching section
|
||||
|
||||
if not running_syncdb():
|
||||
# TODO why this is even loaded during syncdb?
|
||||
for resources in Resource.group_by_content_type():
|
||||
model = resources[0].content_type.model_class()
|
||||
router.insert(model, 'resources', ResourceSerializer, required=False, many=True)
|
||||
|
|
@ -54,7 +55,7 @@ if not running_syncdb():
|
|||
|
||||
old_metadata = viewset.metadata
|
||||
def metadata(self, request, resources=resources):
|
||||
""" Display resource configuration """
|
||||
""" Provides available resources description """
|
||||
ret = old_metadata(self, request)
|
||||
ret['available_resources'] = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from celery import shared_task
|
||||
|
||||
from django.utils import timezone
|
||||
from orchestra.apps.orchestration.models import BackendOperation as Operation
|
||||
|
||||
from .backends import ServiceMonitor
|
||||
|
|
@ -26,6 +26,7 @@ def monitor(resource_id):
|
|||
for obj in model.objects.all():
|
||||
data = MonitorData.get_or_create(obj, resource)
|
||||
current = data.get_used()
|
||||
if not resource.disable_trigger:
|
||||
if data.used < data.allocated and current > data.allocated:
|
||||
op = Operation.create(backend, data.content_object, Operation.EXCEED)
|
||||
operations.append(op)
|
||||
|
|
@ -33,6 +34,6 @@ def monitor(resource_id):
|
|||
op = Operation.create(backend, data.content_object, Operation.RECOVERY)
|
||||
operation.append(op)
|
||||
data.used = current
|
||||
data.las_update = datetime.now()
|
||||
data.las_update = timezone.now()
|
||||
data.save()
|
||||
Operation.execute(operations)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from orchestra.apps.orchestration import ServiceController
|
||||
|
|
@ -49,13 +50,62 @@ class SystemUserDisk(ServiceMonitor):
|
|||
|
||||
def monitor(self, user):
|
||||
context = self.get_context(user)
|
||||
self.append("du -s %(home)s | {\n"
|
||||
" read value\n"
|
||||
" echo '%(username)s' $value\n"
|
||||
"}" % context)
|
||||
self.append("du -s %(home)s | xargs echo %(object_id)s" % context)
|
||||
|
||||
def get_context(self, user):
|
||||
context = SystemUserBackend().get_context(user)
|
||||
context['object_id'] = user.pk
|
||||
return context
|
||||
|
||||
|
||||
class FTPTraffic(ServiceMonitor):
|
||||
model = 'users.User'
|
||||
resource = ServiceMonitor.TRAFFIC
|
||||
verbose_name = _('FTP traffic')
|
||||
|
||||
def monitor(self, user):
|
||||
context = self.get_context(user)
|
||||
self.append("""
|
||||
grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk '
|
||||
BEGIN {
|
||||
ini = "%(last_date)s"
|
||||
end = "%(current_date)s"
|
||||
|
||||
months["Jan"] = "01"
|
||||
months["Feb"] = "02"
|
||||
months["Mar"] = "03"
|
||||
months["Apr"] = "04"
|
||||
months["May"] = "05"
|
||||
months["Jun"] = "06"
|
||||
months["Jul"] = "07"
|
||||
months["Aug"] = "08"
|
||||
months["Sep"] = "09"
|
||||
months["Oct"] = "10"
|
||||
months["Nov"] = "11"
|
||||
months["Dec"] = "12"
|
||||
} {
|
||||
# log: Fri Jul 11 13:23:17 2014
|
||||
split($4, t, ":")
|
||||
# line_date = year month day hour minute second
|
||||
line_date = $5 months[$2] $3 t[1] t[2] t[3]
|
||||
if ( line_date > ini && line_date < end)
|
||||
split($0, l, "\\", ")
|
||||
split(l[3], b, " ")
|
||||
sum += b[1]
|
||||
} END {
|
||||
if ( sum )
|
||||
print sum
|
||||
else
|
||||
print 0
|
||||
}
|
||||
' | xargs echo %(object_id)s """ % context)
|
||||
|
||||
def get_context(self, user):
|
||||
return {
|
||||
'log_file': settings.USERS_FTP_LOG_PATH,
|
||||
'last_date': timezone.localtime(self.get_last_date(user)).strftime("%Y%m%d%H%M%S"),
|
||||
'current_date': timezone.localtime(self.get_current_date()).strftime("%Y%m%d%H%M%S"),
|
||||
'object_id': user.pk,
|
||||
'username': user.username,
|
||||
}
|
||||
|
||||
def process(self, output):
|
||||
# TODO transaction
|
||||
for line in output.readlines():
|
||||
username, value = line.strip().slpit()
|
||||
History.store(object_id=user_id, value=value)
|
||||
|
|
|
|||
|
|
@ -29,10 +29,10 @@ class MailSystemUserBackend(ServiceController):
|
|||
self.append("chown %(username)s.%(group)s %(home)s" % context)
|
||||
|
||||
def generate_filter(self, mailbox, context):
|
||||
datetime = timezone.now().strftime("%B %d, %Y, %H:%M")
|
||||
now = timezone.now().strftime("%B %d, %Y, %H:%M")
|
||||
context['filtering'] = (
|
||||
"# Sieve Filter\n"
|
||||
"# Generated by Orchestra %s\n\n" % datetime
|
||||
"# Generated by Orchestra %s\n\n" % now
|
||||
)
|
||||
if mailbox.use_custom_filtering:
|
||||
context['filtering'] += mailbox.custom_filtering
|
||||
|
|
@ -139,7 +139,21 @@ class AutoresponseBackend(ServiceController):
|
|||
model = 'mail.Autoresponse'
|
||||
|
||||
|
||||
class MailDisk(ServiceMonitor):
|
||||
class MaildirDisk(ServiceMonitor):
|
||||
model = 'email.Mailbox'
|
||||
resource = ServiceMonitor.DISK
|
||||
verbose_name = _("Mail disk")
|
||||
verbose_name = _("Maildir disk usage")
|
||||
|
||||
def monitor(self, mailbox):
|
||||
context = self.get_context(mailbox)
|
||||
self.append(
|
||||
"SIZE=$(sed -n '2p' %(maildir_path)s | cut -d' ' -f1)\n"
|
||||
"echo %(object_id)s ${SIZE:-0}" % context
|
||||
)
|
||||
|
||||
def get_context(self, mailbox):
|
||||
context = MailSystemUserBackend().get_context(site)
|
||||
context['home'] = settings.EMAILS_HOME % context
|
||||
context['maildir_path'] = os.path.join(context['home'], 'Maildir/maildirsize')
|
||||
context['object_id'] = mailbox.pk
|
||||
return context
|
||||
|
|
|
|||
|
|
@ -2,3 +2,5 @@ from django.conf import settings
|
|||
|
||||
|
||||
USERS_SYSTEMUSER_HOME = getattr(settings, 'USERES_SYSTEMUSER_HOME', '/home/%(username)s')
|
||||
|
||||
USERS_FTP_LOG_PATH = getattr(settings, 'USERS_FTP_LOG_PATH', '/var/log/vsftpd.log')
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
|
||||
from django.template import Template, Context
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
from orchestra.apps.orchestration import ServiceController
|
||||
|
|
@ -185,8 +186,8 @@ class Apache2Traffic(ServiceMonitor):
|
|||
context = self.get_context(site)
|
||||
self.append("""
|
||||
awk 'BEGIN {
|
||||
ini = "%(last_date)s";
|
||||
end = "%(current_date)s";
|
||||
ini = "%(last_date)s"
|
||||
end = "%(current_date)s"
|
||||
|
||||
months["Jan"] = "01";
|
||||
months["Feb"] = "02";
|
||||
|
|
@ -201,32 +202,31 @@ class Apache2Traffic(ServiceMonitor):
|
|||
months["Nov"] = "11";
|
||||
months["Dec"] = "12";
|
||||
} {
|
||||
# date = [11/Jul/2014:13:50:41
|
||||
date = substr($4, 2)
|
||||
year = substr(date, 8, 4)
|
||||
month = months[substr(date, 4, 3)];
|
||||
day = substr(date, 1, 2)
|
||||
hour = substr(date, 13, 2)
|
||||
minute = substr(date, 16, 2)
|
||||
second = substr(date, 19, 2);
|
||||
second = substr(date, 19, 2)
|
||||
line_date = year month day hour minute second
|
||||
if ( line_date > ini && line_date < end)
|
||||
if ( $10 == "" )
|
||||
sum += $9
|
||||
else
|
||||
sum += $10;
|
||||
sum += $10
|
||||
} END {
|
||||
print sum;
|
||||
}' %(log_file)s | {
|
||||
read value
|
||||
echo %(site_id)s $value
|
||||
}
|
||||
""" % context)
|
||||
if ( sum )
|
||||
print sum
|
||||
else
|
||||
print 0
|
||||
}' %(log_file)s | xargs echo %(object_id)s """ % context)
|
||||
|
||||
def get_context(self, site):
|
||||
# TODO log timezone!!
|
||||
return {
|
||||
'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
|
||||
'last_date': self.get_last_date(site).strftime("%Y%m%d%H%M%S"),
|
||||
'current_date': self.get_current_date().strftime("%Y%m%d%H%M%S"),
|
||||
'site_id': site.pk,
|
||||
'last_date': timezone.localtime(self.get_last_date(site)).strftime("%Y%m%d%H%M%S"),
|
||||
'current_date': timezone.localtime(self.get_current_date()).strftime("%Y%m%d%H%M%S"),
|
||||
'object_id': site.pk,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from django.utils.timesince import timesince as django_timesince
|
|||
from django.utils.timezone import is_aware, utc
|
||||
|
||||
|
||||
# TODO deprecate in favour of celery timesince
|
||||
def timesince(d, now=None, reversed=False):
|
||||
""" Hack to provide second precision under 2 minutes """
|
||||
if not now:
|
||||
|
|
|
|||
Loading…
Reference in New Issue