From c54c084dff38221d129d1ad8d85a1355b6d4acda Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 11 Jul 2014 14:48:46 +0000 Subject: [PATCH] Implemented some monitors --- TODO.md | 5 ++ orchestra/apps/domains/utils.py | 4 +- orchestra/apps/lists/backends.py | 45 +++++++------- orchestra/apps/lists/forms.py | 4 +- orchestra/apps/lists/settings.py | 3 + orchestra/apps/orchestration/backends.py | 4 +- orchestra/apps/orchestration/manager.py | 6 +- orchestra/apps/resources/backends.py | 9 +-- orchestra/apps/resources/models.py | 32 ++++++---- orchestra/apps/resources/serializers.py | 3 +- orchestra/apps/resources/tasks.py | 17 +++--- orchestra/apps/users/backends.py | 68 ++++++++++++++++++--- orchestra/apps/users/roles/mail/backends.py | 22 +++++-- orchestra/apps/users/settings.py | 2 + orchestra/apps/websites/backends/apache.py | 28 ++++----- orchestra/utils/time.py | 1 + 16 files changed, 168 insertions(+), 85 deletions(-) diff --git a/TODO.md b/TODO.md index 0f93df6a..bc5165ed 100644 --- a/TODO.md +++ b/TODO.md @@ -49,3 +49,8 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * pip install pyinotify * create custom field that returns backend python objects + +* Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers) +* Resource metric: KB MB B? + +* EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0) diff --git a/orchestra/apps/domains/utils.py b/orchestra/apps/domains/utils.py index a5fdcb49..c67efdb1 100644 --- a/orchestra/apps/domains/utils.py +++ b/orchestra/apps/domains/utils.py @@ -1,8 +1,8 @@ -import datetime +from django.utils import timezone def generate_zone_serial(): - today = datetime.date.today() + today = timezone.now() return int("%.4d%.2d%.2d%.2d" % (today.year, today.month, today.day, 0)) diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index 4ce50cb7..e3d1d233 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -1,9 +1,12 @@ from django.template import Template, Context +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController from orchestra.apps.resources import ServiceMonitor +from . import settings + class MailmanBackend(ServiceController): verbose_name = "Mailman" @@ -14,28 +17,22 @@ class MailmanTraffic(ServiceMonitor): model = 'lists.List' resource = ServiceMonitor.TRAFFIC - def process(self, output): - for line in output.readlines(): - listname, value = line.strip().slpit() + def monitor(self, mail_list): + context = self.get_context(mail_list) + self.append( + "SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n" + "SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n" + " | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n" + " | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n" + " | tr '\\n' '+' \\\n" + " | xargs -i echo {}0 )\n" + "echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context) - def monitor(self, mailinglist): - self.append( - "LISTS=$(grep -v 'post to mailman' /var/log/mailman/post" - " | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq" - " | while read line; do \n" - " grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}" - " | sed 's/size=//' | sed 's/,//'\n" - "done)" - ) - self.append( - 'SUBS=""\n' - 'while read LIST; do\n' - ' NAME=$(echo "$LIST" | awk {\'print $1\'})\n' - ' SIZE=$(echo "$LIST" | awk {\'print $2\'})\n' - ' if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n' - ' SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n' - ' fi\n' - ' SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n' - ' echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n' - 'done <<< "$LISTS"' - ) + def get_context(self, mail_list): + return { + 'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH, + 'list_name': mail_list.name, + 'object_id': mail_list.pk, + 'last_date': timezone.localtime(self.get_last_date(mail_list)).strftime("%b %d %H:%M:%S"), + 'current_date': timezone.localtime(self.get_current_date()).strftime("%b %d %H:%M:%S"), + } diff --git a/orchestra/apps/lists/forms.py b/orchestra/apps/lists/forms.py index 828b15bc..a32457a4 100644 --- a/orchestra/apps/lists/forms.py +++ b/orchestra/apps/lists/forms.py @@ -26,7 +26,7 @@ class ListCreationForm(CleanAddressMixin, forms.ModelForm): help_text=_("Enter the same password as above, for verification.")) def __init__(self, *args, **kwargs): - super(ListAdminForm, self).__init__(*args, **kwargs) + super(ListCreationForm, self).__init__(*args, **kwargs) self.fields['password1'].validators.append(validate_password) def clean_password2(self): @@ -38,7 +38,7 @@ class ListCreationForm(CleanAddressMixin, forms.ModelForm): return password2 def save(self, commit=True): - obj = super(ListAdminForm, self).save(commit=commit) + obj = super(ListCreationForm, self).save(commit=commit) obj.set_password(self.cleaned_data["password1"]) return obj diff --git a/orchestra/apps/lists/settings.py b/orchestra/apps/lists/settings.py index 46548933..a3faf55b 100644 --- a/orchestra/apps/lists/settings.py +++ b/orchestra/apps/lists/settings.py @@ -6,3 +6,6 @@ from django.conf import settings LISTS_DOMAIN_MODEL = getattr(settings, 'LISTS_DOMAIN_MODEL', 'domains.Domain') LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'grups.orchestra.lan') + +LISTS_MAILMAN_POST_LOG_PATH = getattr(settings, 'LISTS_MAILMAN_POST_LOG_PATH', + '/var/log/mailman/post') diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index d6c52cf0..7e71fbef 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -1,6 +1,6 @@ -from datetime import datetime from functools import partial +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from orchestra.utils import plugins @@ -89,7 +89,7 @@ class ServiceBackend(object): return sorted(choices, key=lambda e: e[0]) def get_banner(self): - time = datetime.now().strftime("%h %d, %Y %I:%M:%S") + time = timezone.now().strftime("%h %d, %Y %I:%M:%S") return "Generated by Orchestra %s" % time def execute(self, server): diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index fdd22e04..1b9254ff 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -10,9 +10,9 @@ from .helpers import send_report def as_task(execute): def wrapper(*args, **kwargs): - with db.transaction.commit_manually(): - log = execute(*args, **kwargs) - db.transaction.commit() +# with db.transaction.commit_manually(): + log = execute(*args, **kwargs) +# db.transaction.commit() if log.state != log.SUCCESS: send_report(execute, args, log) return log diff --git a/orchestra/apps/resources/backends.py b/orchestra/apps/resources/backends.py index da9cc8f8..1f7e7d8d 100644 --- a/orchestra/apps/resources/backends.py +++ b/orchestra/apps/resources/backends.py @@ -1,6 +1,7 @@ import datetime from django.contrib.contenttypes.models import ContentType +from django.utils import timezone from orchestra.apps.orchestration import ServiceBackend from orchestra.utils.functional import cached @@ -17,14 +18,14 @@ class ServiceMonitor(ServiceBackend): @classmethod def get_backends(cls): """ filter monitor classes """ - return [plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__] + for backend in cls.plugins: + if backend != ServiceMonitor and ServiceMonitor in backend.__mro__: + yield backend @cached def get_last_date(self, obj): from .models import MonitorData try: - # TODO replace - #return MonitorData.objects.filter(content_object=obj).latest().date ct = ContentType.objects.get_for_model(type(obj)) return MonitorData.objects.filter(content_type=ct, object_id=obj.pk).latest().date except MonitorData.DoesNotExist: @@ -32,7 +33,7 @@ class ServiceMonitor(ServiceBackend): @cached def get_current_date(self): - return datetime.datetime.now() + return timezone.now() def store(self, log): """ object_id value """ diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index fa34d83d..4d8c0b98 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -36,22 +36,27 @@ class Resource(models.Model): verbose_name = models.CharField(_("verbose name"), max_length=256, unique=True) content_type = models.ForeignKey(ContentType, help_text=_("Model where this resource will be hooked")) - period = models.CharField(_("period"), max_length=16, choices=PERIODS, default=LAST, - help_text=_("Operation used for aggregating this resource monitored data.")) + period = models.CharField(_("period"), max_length=16, choices=PERIODS, + default=LAST, + help_text=_("Operation used for aggregating this resource monitored" + "data.")) ondemand = models.BooleanField(_("on demand"), default=False, help_text=_("If enabled the resource will not be pre-allocated, " "but allocated under the application demand")) default_allocation = models.PositiveIntegerField(_("default allocation"), + null=True, blank=True, help_text=_("Default allocation value used when this is not an " - "on demand resource"), - null=True, blank=True) + "on demand resource")) is_active = models.BooleanField(_("is active"), default=True) disable_trigger = models.BooleanField(_("disable trigger"), default=False, - help_text=_("Disables monitor's resource exeeded and recovery triggers")) + help_text=_("Disables monitors exeeded and recovery triggers")) crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"), - help_text=_("Crontab for periodic execution")) - monitors = MultiSelectField(_("monitors"), max_length=256, - choices=ServiceMonitor.get_choices()) + null=True, blank=True, + help_text=_("Crontab for periodic execution. " + "Leave it empty to disable periodic monitoring")) + monitors = MultiSelectField(_("monitors"), max_length=256, blank=True, + choices=ServiceMonitor.get_choices(), + help_text=_("Monitor backends used for monitoring this resource.")) def __unicode__(self): return self.name @@ -83,7 +88,8 @@ class Resource(models.Model): def group_by_content_type(cls): prev = None group = [] - for resource in cls.objects.filter(is_active=True).order_by('content_type'): + resources = cls.objects.filter(is_active=True).order_by('content_type') + for resource in resources: ct = resource.content_type if prev != ct: if group: @@ -121,7 +127,7 @@ class ResourceData(models.Model): def get_used(self): resource = self.resource - today = datetime.date.today() + today = timezone.now() result = 0 has_result = False for monitor in resource.monitors: @@ -133,7 +139,8 @@ class ResourceData(models.Model): except MonitorData.DoesNotExist: continue has_result = True - epoch = datetime(year=today.year, month=today.month, day=1) + epoch = datetime(year=today.year, month=today.month, day=1, + tzinfo=timezone.utc) total = (epoch-last.date).total_seconds() dataset = dataset.filter(date__year=today.year, date__month=today.month) @@ -154,7 +161,8 @@ class ResourceData(models.Model): continue has_result = True else: - raise NotImplementedError("%s support not implemented" % self.period) + msg = "%s support not implemented" % self.period + raise NotImplementedError(msg) return result if has_result else None diff --git a/orchestra/apps/resources/serializers.py b/orchestra/apps/resources/serializers.py index 17a2a608..1f7808bb 100644 --- a/orchestra/apps/resources/serializers.py +++ b/orchestra/apps/resources/serializers.py @@ -24,6 +24,7 @@ class ResourceSerializer(serializers.ModelSerializer): # Monkey-patching section if not running_syncdb(): + # TODO why this is even loaded during syncdb? for resources in Resource.group_by_content_type(): model = resources[0].content_type.model_class() router.insert(model, 'resources', ResourceSerializer, required=False, many=True) @@ -54,7 +55,7 @@ if not running_syncdb(): old_metadata = viewset.metadata def metadata(self, request, resources=resources): - """ Display resource configuration """ + """ Provides available resources description """ ret = old_metadata(self, request) ret['available_resources'] = [ { diff --git a/orchestra/apps/resources/tasks.py b/orchestra/apps/resources/tasks.py index 9a870132..0f834994 100644 --- a/orchestra/apps/resources/tasks.py +++ b/orchestra/apps/resources/tasks.py @@ -1,5 +1,5 @@ from celery import shared_task - +from django.utils import timezone from orchestra.apps.orchestration.models import BackendOperation as Operation from .backends import ServiceMonitor @@ -26,13 +26,14 @@ def monitor(resource_id): for obj in model.objects.all(): data = MonitorData.get_or_create(obj, resource) current = data.get_used() - if data.used < data.allocated and current > data.allocated: - op = Operation.create(backend, data.content_object, Operation.EXCEED) - operations.append(op) - elif res.used > res.allocated and current < res.allocated: - op = Operation.create(backend, data.content_object, Operation.RECOVERY) - operation.append(op) + if not resource.disable_trigger: + if data.used < data.allocated and current > data.allocated: + op = Operation.create(backend, data.content_object, Operation.EXCEED) + operations.append(op) + elif res.used > res.allocated and current < res.allocated: + op = Operation.create(backend, data.content_object, Operation.RECOVERY) + operation.append(op) data.used = current - data.las_update = datetime.now() + data.las_update = timezone.now() data.save() Operation.execute(operations) diff --git a/orchestra/apps/users/backends.py b/orchestra/apps/users/backends.py index cb1ebebf..ee957fa1 100644 --- a/orchestra/apps/users/backends.py +++ b/orchestra/apps/users/backends.py @@ -1,3 +1,4 @@ +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController @@ -49,13 +50,62 @@ class SystemUserDisk(ServiceMonitor): def monitor(self, user): context = self.get_context(user) - self.append("du -s %(home)s | {\n" - " read value\n" - " echo '%(username)s' $value\n" - "}" % context) + self.append("du -s %(home)s | xargs echo %(object_id)s" % context) + + def get_context(self, user): + context = SystemUserBackend().get_context(user) + context['object_id'] = user.pk + return context + + +class FTPTraffic(ServiceMonitor): + model = 'users.User' + resource = ServiceMonitor.TRAFFIC + verbose_name = _('FTP traffic') + + def monitor(self, user): + context = self.get_context(user) + self.append(""" + grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk ' + BEGIN { + ini = "%(last_date)s" + end = "%(current_date)s" + + months["Jan"] = "01" + months["Feb"] = "02" + months["Mar"] = "03" + months["Apr"] = "04" + months["May"] = "05" + months["Jun"] = "06" + months["Jul"] = "07" + months["Aug"] = "08" + months["Sep"] = "09" + months["Oct"] = "10" + months["Nov"] = "11" + months["Dec"] = "12" + } { + # log: Fri Jul 11 13:23:17 2014 + split($4, t, ":") + # line_date = year month day hour minute second + line_date = $5 months[$2] $3 t[1] t[2] t[3] + if ( line_date > ini && line_date < end) + split($0, l, "\\", ") + split(l[3], b, " ") + sum += b[1] + } END { + if ( sum ) + print sum + else + print 0 + } + ' | xargs echo %(object_id)s """ % context) + + def get_context(self, user): + return { + 'log_file': settings.USERS_FTP_LOG_PATH, + 'last_date': timezone.localtime(self.get_last_date(user)).strftime("%Y%m%d%H%M%S"), + 'current_date': timezone.localtime(self.get_current_date()).strftime("%Y%m%d%H%M%S"), + 'object_id': user.pk, + 'username': user.username, + } - def process(self, output): - # TODO transaction - for line in output.readlines(): - username, value = line.strip().slpit() - History.store(object_id=user_id, value=value) diff --git a/orchestra/apps/users/roles/mail/backends.py b/orchestra/apps/users/roles/mail/backends.py index 08e32ada..c1283d22 100644 --- a/orchestra/apps/users/roles/mail/backends.py +++ b/orchestra/apps/users/roles/mail/backends.py @@ -29,10 +29,10 @@ class MailSystemUserBackend(ServiceController): self.append("chown %(username)s.%(group)s %(home)s" % context) def generate_filter(self, mailbox, context): - datetime = timezone.now().strftime("%B %d, %Y, %H:%M") + now = timezone.now().strftime("%B %d, %Y, %H:%M") context['filtering'] = ( "# Sieve Filter\n" - "# Generated by Orchestra %s\n\n" % datetime + "# Generated by Orchestra %s\n\n" % now ) if mailbox.use_custom_filtering: context['filtering'] += mailbox.custom_filtering @@ -139,7 +139,21 @@ class AutoresponseBackend(ServiceController): model = 'mail.Autoresponse' -class MailDisk(ServiceMonitor): +class MaildirDisk(ServiceMonitor): model = 'email.Mailbox' resource = ServiceMonitor.DISK - verbose_name = _("Mail disk") + verbose_name = _("Maildir disk usage") + + def monitor(self, mailbox): + context = self.get_context(mailbox) + self.append( + "SIZE=$(sed -n '2p' %(maildir_path)s | cut -d' ' -f1)\n" + "echo %(object_id)s ${SIZE:-0}" % context + ) + + def get_context(self, mailbox): + context = MailSystemUserBackend().get_context(site) + context['home'] = settings.EMAILS_HOME % context + context['maildir_path'] = os.path.join(context['home'], 'Maildir/maildirsize') + context['object_id'] = mailbox.pk + return context diff --git a/orchestra/apps/users/settings.py b/orchestra/apps/users/settings.py index 9521bf19..012099a4 100644 --- a/orchestra/apps/users/settings.py +++ b/orchestra/apps/users/settings.py @@ -2,3 +2,5 @@ from django.conf import settings USERS_SYSTEMUSER_HOME = getattr(settings, 'USERES_SYSTEMUSER_HOME', '/home/%(username)s') + +USERS_FTP_LOG_PATH = getattr(settings, 'USERS_FTP_LOG_PATH', '/var/log/vsftpd.log') diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 28107279..2006b4c4 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -1,6 +1,7 @@ import os from django.template import Template, Context +from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController @@ -185,8 +186,8 @@ class Apache2Traffic(ServiceMonitor): context = self.get_context(site) self.append(""" awk 'BEGIN { - ini = "%(last_date)s"; - end = "%(current_date)s"; + ini = "%(last_date)s" + end = "%(current_date)s" months["Jan"] = "01"; months["Feb"] = "02"; @@ -201,32 +202,31 @@ class Apache2Traffic(ServiceMonitor): months["Nov"] = "11"; months["Dec"] = "12"; } { + # date = [11/Jul/2014:13:50:41 date = substr($4, 2) year = substr(date, 8, 4) month = months[substr(date, 4, 3)]; day = substr(date, 1, 2) hour = substr(date, 13, 2) minute = substr(date, 16, 2) - second = substr(date, 19, 2); + second = substr(date, 19, 2) line_date = year month day hour minute second if ( line_date > ini && line_date < end) if ( $10 == "" ) sum += $9 else - sum += $10; + sum += $10 } END { - print sum; - }' %(log_file)s | { - read value - echo %(site_id)s $value - } - """ % context) + if ( sum ) + print sum + else + print 0 + }' %(log_file)s | xargs echo %(object_id)s """ % context) def get_context(self, site): - # TODO log timezone!! return { 'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name), - 'last_date': self.get_last_date(site).strftime("%Y%m%d%H%M%S"), - 'current_date': self.get_current_date().strftime("%Y%m%d%H%M%S"), - 'site_id': site.pk, + 'last_date': timezone.localtime(self.get_last_date(site)).strftime("%Y%m%d%H%M%S"), + 'current_date': timezone.localtime(self.get_current_date()).strftime("%Y%m%d%H%M%S"), + 'object_id': site.pk, } diff --git a/orchestra/utils/time.py b/orchestra/utils/time.py index ca1f838d..f3cf5cf7 100644 --- a/orchestra/utils/time.py +++ b/orchestra/utils/time.py @@ -6,6 +6,7 @@ from django.utils.timesince import timesince as django_timesince from django.utils.timezone import is_aware, utc +# TODO deprecate in favour of celery timesince def timesince(d, now=None, reversed=False): """ Hack to provide second precision under 2 minutes """ if not now: