From 53a135a1d97a15a8cc43ca21a16cee8bed04921b Mon Sep 17 00:00:00 2001 From: Marc Date: Thu, 10 Jul 2014 15:19:06 +0000 Subject: [PATCH] Improvements on resource monitoring --- TODO.md | 3 +- orchestra/apps/accounts/admin.py | 2 + orchestra/apps/domains/api.py | 4 +- orchestra/apps/orchestration/backends.py | 2 +- orchestra/apps/orchestration/helpers.py | 2 +- orchestra/apps/orchestration/methods.py | 4 +- orchestra/apps/orchestration/models.py | 18 ++++--- orchestra/apps/resources/admin.py | 31 ++++++++++-- orchestra/apps/resources/backends.py | 35 +++++++++++-- orchestra/apps/resources/forms.py | 1 - orchestra/apps/resources/models.py | 38 ++++++++------ orchestra/apps/resources/serializers.py | 3 ++ orchestra/apps/users/roles/mail/backends.py | 1 + orchestra/apps/websites/backends/apache.py | 55 +++++++++++---------- orchestra/models/fields.py | 6 +++ 15 files changed, 137 insertions(+), 68 deletions(-) diff --git a/TODO.md b/TODO.md index f817d34b..0f93df6a 100644 --- a/TODO.md +++ b/TODO.md @@ -48,5 +48,4 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * passlib; nano /usr/local/lib/python2.7/dist-packages/passlib/ext/django/utils.py SortedDict -> collections.OrderedDict * pip install pyinotify - -* Backend.operations dynamically generated based on defined methods +* create custom field that returns backend python objects diff --git a/orchestra/apps/accounts/admin.py b/orchestra/apps/accounts/admin.py index cce56fc6..ad315953 100644 --- a/orchestra/apps/accounts/admin.py +++ b/orchestra/apps/accounts/admin.py @@ -71,6 +71,8 @@ class AccountAdmin(ExtendedModelAdmin): def save_model(self, request, obj, form, change): """ Save user and account, they are interdependent """ + if change: + return super(AccountAdmin, self).save_model(request, obj, form, change) obj.user.save() obj.user_id = obj.user.pk obj.save() diff --git a/orchestra/apps/domains/api.py b/orchestra/apps/domains/api.py index b5089bec..e4e62045 100644 --- a/orchestra/apps/domains/api.py +++ b/orchestra/apps/domains/api.py @@ -22,7 +22,9 @@ class DomainViewSet(AccountApiMixin, viewsets.ModelViewSet): @link() def view_zone(self, request, pk=None): domain = self.get_object() - return Response({'zone': domain.render_zone()}) + return Response({ + 'zone': domain.render_zone() + }) def metadata(self, request): ret = super(DomainViewSet, self).metadata(request) diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index 70f97b88..d6c52cf0 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -69,7 +69,7 @@ class ServiceBackend(object): @classmethod def get_backend(cls, name): - for backend in ServiceMonitor.get_backends(): + for backend in ServiceBackend.get_backends(): if backend.get_name() == name: return backend raise KeyError('This backend is not registered') diff --git a/orchestra/apps/orchestration/helpers.py b/orchestra/apps/orchestration/helpers.py index 6196e4e8..c0cf370c 100644 --- a/orchestra/apps/orchestration/helpers.py +++ b/orchestra/apps/orchestration/helpers.py @@ -38,7 +38,7 @@ def message_user(request, logs): errors = total-successes if errors: msg = 'backends have' if errors > 1 else 'backend has' - msg = _("%d out of %d {0} fail to executed".format(msg)) + msg = _("%d out of %d {0} fail to execute".format(msg)) messages.warning(request, msg % (errors, total)) else: msg = 'backends have' if successes > 1 else 'backend has' diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index 8ac97a80..2ca95b7a 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -40,12 +40,12 @@ def BashSSH(backend, log, server, cmds): channel = transport.open_session() sftp = paramiko.SFTPClient.from_transport(transport) - sftp.put(path, path) + sftp.put(path, "%s.remote" % path) sftp.close() os.remove(path) context = { - 'path': path, + 'path': "%s.remote" % path, 'digest': digest } cmd = ( diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index e2468507..5f3fad8c 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -4,6 +4,7 @@ from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import ugettext_lazy as _ +from orchestra.models.fields import NullableCharField from orchestra.utils.apps import autodiscover from orchestra.utils.functional import cached @@ -14,9 +15,8 @@ from .backends import ServiceBackend class Server(models.Model): """ Machine runing daemons (services) """ name = models.CharField(_("name"), max_length=256, unique=True) - # TODO unique address with blank=True (nullablecharfield) - address = models.CharField(_("address"), max_length=256, blank=True, - help_text=_("IP address or domain name")) + address = NullableCharField(_("address"), max_length=256, blank=True, + null=True, unique=True, help_text=_("IP address or domain name")) description = models.TextField(_("description"), blank=True) os = models.CharField(_("operative system"), max_length=32, choices=settings.ORCHESTRATION_OS_CHOICES, @@ -82,8 +82,7 @@ class BackendOperation(models.Model): MONITOR = 'monitor' log = models.ForeignKey('orchestration.BackendLog', related_name='operations') - # TODO backend and backend_class() (like content_type) - backend_class = models.CharField(_("backend"), max_length=256) + backend = models.CharField(_("backend"), max_length=256) action = models.CharField(_("action"), max_length=64) content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() @@ -94,11 +93,11 @@ class BackendOperation(models.Model): verbose_name_plural = _("Operations") def __unicode__(self): - return '%s.%s(%s)' % (self.backend_class, self.action, self.instance) + return '%s.%s(%s)' % (self.backend, self.action, self.instance) def __hash__(self): """ set() """ - backend = getattr(self, 'backend', self.backend_class) + backend = getattr(self, 'backend', self.backend) return hash(backend) + hash(self.instance) + hash(self.action) def __eq__(self, operation): @@ -107,13 +106,16 @@ class BackendOperation(models.Model): @classmethod def create(cls, backend, instance, action): - op = cls(backend_class=backend.get_name(), instance=instance, action=action) + op = cls(backend=backend.get_name(), instance=instance, action=action) op.backend = backend return op @classmethod def execute(cls, operations): return manager.execute(operations) + + def backend_class(self): + return ServiceBackend.get_backend(self.backend) autodiscover('backends') diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 48ed4217..f714b062 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -1,8 +1,9 @@ -from django.contrib import admin +from django.contrib import admin, messages from django.contrib.contenttypes import generic from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ +from orchestra.admin import ExtendedModelAdmin from orchestra.admin.filters import UsedContentTypeFilter from orchestra.admin.utils import insertattr, get_modeladmin from orchestra.core import services @@ -12,14 +13,34 @@ from .forms import ResourceForm from .models import Resource, ResourceData, MonitorData -class ResourceAdmin(admin.ModelAdmin): - # TODO warning message server/celery should be restarted when creating things - +class ResourceAdmin(ExtendedModelAdmin): list_display = ( 'name', 'verbose_name', 'content_type', 'period', 'ondemand', - 'default_allocation', 'disable_trigger' + 'default_allocation', 'disable_trigger', 'crontab', ) list_filter = (UsedContentTypeFilter, 'period', 'ondemand', 'disable_trigger') + fieldsets = ( + (None, { + 'fields': ('name', 'content_type', 'period'), + }), + (_("Configuration"), { + 'fields': ('verbose_name', 'default_allocation', 'ondemand', + 'disable_trigger', 'is_active'), + }), + (_("Monitoring"), { + 'fields': ('monitors', 'crontab'), + }), + ) + change_readonly_fields = ('name', 'content_type', 'period') + + def add_view(self, request, **kwargs): + """ Warning user if the node is not fully configured """ + if request.method == 'GET': + messages.warning(request, _( + "Restarting orchestra and celery is required to fully apply changes. " + "Remember that allocated values will be applied when objects are saved" + )) + return super(ResourceAdmin, self).add_view(request, **kwargs) def save_model(self, request, obj, form, change): super(ResourceAdmin, self).save_model(request, obj, form, change) diff --git a/orchestra/apps/resources/backends.py b/orchestra/apps/resources/backends.py index 97a94582..0ba02c4f 100644 --- a/orchestra/apps/resources/backends.py +++ b/orchestra/apps/resources/backends.py @@ -1,4 +1,9 @@ +import datetime + +from django.contrib.contenttypes.models import ContentType + from orchestra.apps.orchestration import ServiceBackend +from orchestra.utils.functional import cached class ServiceMonitor(ServiceBackend): @@ -14,14 +19,34 @@ class ServiceMonitor(ServiceBackend): """ filter monitor classes """ return [plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__] - def store(self, stdout): + @cached + def get_last_date(self, obj): + from .models import MonitorData + try: + # TODO replace + #return MonitorData.objects.filter(content_object=obj).latest().date + ct = ContentType.objects.get(app_label=obj._meta.app_label, model=obj._meta.model_name) + return MonitorData.objects.filter(content_type=ct, object_id=obj.pk).latest().date + except MonitorData.DoesNotExist: + return self.get_current_date() - datetime.timedelta(days=1) + + @cached + def get_current_date(self): + return datetime.datetime.now() + + def store(self, log): """ object_id value """ - for line in stdout.readlines(): + from .models import MonitorData + name = self.get_name() + app_label, model_name = self.model.split('.') + ct = ContentType.objects.get(app_label=app_label, model=model_name.lower()) + for line in log.stdout.splitlines(): line = line.strip() object_id, value = line.split() - # TODO date - MonitorHistory.store(self.model, object_id, value, date) + MonitorData.objects.create(monitor=name, object_id=object_id, + content_type=ct, value=value, date=self.get_current_date()) def execute(self, server): - log = super(MonitorBackend, self).execute(server) + log = super(ServiceMonitor, self).execute(server) + self.store(log) return log diff --git a/orchestra/apps/resources/forms.py b/orchestra/apps/resources/forms.py index 8bcd545c..fc46c8a4 100644 --- a/orchestra/apps/resources/forms.py +++ b/orchestra/apps/resources/forms.py @@ -21,7 +21,6 @@ class ResourceForm(forms.ModelForm): super(ResourceForm, self).__init__(*args, **kwargs) if self.resource: self.fields['verbose_name'].initial = self.resource.verbose_name - self.fields['used'].initial = self.resource.get_used() # TODO if self.resource.ondemand: self.fields['allocated'].required = False self.fields['allocated'].widget = ReadOnlyWidget(None, '') diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index c2a5a136..4cc85c7a 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -1,7 +1,7 @@ import datetime from django.db import models -from django.contrib.contenttypes import generic +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType from django.core import validators from django.utils.translation import ugettext_lazy as _ @@ -34,17 +34,22 @@ class Resource(models.Model): validators=[validators.RegexValidator(r'^[a-z0-9_\-]+$', _('Enter a valid name.'), 'invalid')]) verbose_name = models.CharField(_("verbose name"), max_length=256, unique=True) - content_type = models.ForeignKey(ContentType) # TODO filter by servicE? - period = models.CharField(_("period"), max_length=16, choices=PERIODS, - default=LAST) - ondemand = models.BooleanField(_("on demand"), default=False) + content_type = models.ForeignKey(ContentType, + help_text=_("Model where this resource will be hooked")) + period = models.CharField(_("period"), max_length=16, choices=PERIODS, default=LAST, + help_text=_("Operation used for aggregating this resource monitored data.")) + ondemand = models.BooleanField(_("on demand"), default=False, + help_text=_("If enabled the resource will not be pre-allocated, " + "but allocated under the application demand")) default_allocation = models.PositiveIntegerField(_("default allocation"), + help_text=_("Default allocation value used when this is not an " + "on demand resource"), null=True, blank=True) is_active = models.BooleanField(_("is active"), default=True) - disable_trigger = models.BooleanField(_("disable trigger"), default=False) + disable_trigger = models.BooleanField(_("disable trigger"), default=False, + help_text=_("Disables monitor's resource exeeded and recovery triggers")) crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"), help_text=_("Crontab for periodic execution")) - # TODO create custom field that returns backend python objects monitors = MultiSelectField(_("monitors"), max_length=256, choices=ServiceMonitor.get_choices()) @@ -58,13 +63,16 @@ class Resource(models.Model): try: task = PeriodicTask.objects.get(name=name) except PeriodicTask.DoesNotExist: - PeriodicTask.objects.create(name=name, task='resources.Monitor', - args=[self.pk], crontab=self.crontab) - else: - if task.crontab != self.crontab: + if self.is_active: + PeriodicTask.objects.create(name=name, task='resources.Monitor', + args=[self.pk], crontab=self.crontab) + else: + if not self.is_active: + task.delete() + elif task.crontab != self.crontab: task.crontab = self.crontab task.save() - + def delete(self, *args, **kwargs): super(Resource, self).delete(*args, **kwargs) name = 'monitor.%s' % str(self) @@ -97,7 +105,7 @@ class ResourceData(models.Model): last_update = models.DateTimeField(null=True) allocated = models.PositiveIntegerField(null=True) - content_object = generic.GenericForeignKey() + content_object = GenericForeignKey() class Meta: unique_together = ('resource', 'content_type', 'object_id') @@ -159,7 +167,7 @@ class MonitorData(models.Model): date = models.DateTimeField(auto_now_add=True) value = models.PositiveIntegerField() - content_object = generic.GenericForeignKey() + content_object = GenericForeignKey() class Meta: get_latest_by = 'date' @@ -170,7 +178,7 @@ class MonitorData(models.Model): def create_resource_relation(): - relation = generic.GenericRelation('resources.ResourceData') + relation = GenericRelation('resources.ResourceData') for resources in Resource.group_by_content_type(): model = resources[0].content_type.model_class() model.add_to_class('resources', relation) diff --git a/orchestra/apps/resources/serializers.py b/orchestra/apps/resources/serializers.py index 0e391bc2..9cb36e8a 100644 --- a/orchestra/apps/resources/serializers.py +++ b/orchestra/apps/resources/serializers.py @@ -7,6 +7,9 @@ from .models import Resource, ResourceData class ResourceSerializer(serializers.ModelSerializer): + # TODO required allocation serializers (like resource form) + # TODO create missing ResourceData (like resource form) + # TODO make default allocation available on OPTIONS (like resource form) name = serializers.SerializerMethodField('get_name') class Meta: diff --git a/orchestra/apps/users/roles/mail/backends.py b/orchestra/apps/users/roles/mail/backends.py index 122f3cea..a892703c 100644 --- a/orchestra/apps/users/roles/mail/backends.py +++ b/orchestra/apps/users/roles/mail/backends.py @@ -12,6 +12,7 @@ from . import settings class MailSystemUserBackend(ServiceController): verbose_name = _("Mail system user") model = 'mail.Mailbox' + # TODO related_models = ('resources__content_type') ?? DEFAULT_GROUP = 'postfix' diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 2af75954..28107279 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -185,47 +185,48 @@ class Apache2Traffic(ServiceMonitor): context = self.get_context(site) self.append(""" awk 'BEGIN { - ini = "%(start_date)s"; - end = "%(end_date)s"; + ini = "%(last_date)s"; + end = "%(current_date)s"; - months["Jan"]="01"; - months["Feb"]="02"; - months["Mar"]="03"; - months["Apr"]="04"; - months["May"]="05"; - months["Jun"]="06"; - months["Jul"]="07"; - months["Aug"]="08"; - months["Sep"]="09"; - months["Oct"]="10"; - months["Nov"]="11"; - months["Dec"]="12"; + months["Jan"] = "01"; + months["Feb"] = "02"; + months["Mar"] = "03"; + months["Apr"] = "04"; + months["May"] = "05"; + months["Jun"] = "06"; + months["Jul"] = "07"; + months["Aug"] = "08"; + months["Sep"] = "09"; + months["Oct"] = "10"; + months["Nov"] = "11"; + months["Dec"] = "12"; } { - date = substr($4,2) - year = substr(date,8,4) - month = months[substr(date,4,3)]; - day = substr(date,1,2) - hour = substr(date,13,2) - minute = substr(date,16,2) - second = substr(date,19,2); + date = substr($4, 2) + year = substr(date, 8, 4) + month = months[substr(date, 4, 3)]; + day = substr(date, 1, 2) + hour = substr(date, 13, 2) + minute = substr(date, 16, 2) + second = substr(date, 19, 2); line_date = year month day hour minute second if ( line_date > ini && line_date < end) if ( $10 == "" ) - sum+=$9 + sum += $9 else - sum+=$10; + sum += $10; } END { print sum; }' %(log_file)s | { read value - echo %(site_name)s $value + echo %(site_id)s $value } """ % context) def get_context(self, site): + # TODO log timezone!! return { 'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name), - 'start_date': '', - 'end_date': '', - 'site_name': '', + 'last_date': self.get_last_date(site).strftime("%Y%m%d%H%M%S"), + 'current_date': self.get_current_date().strftime("%Y%m%d%H%M%S"), + 'site_id': site.pk, } diff --git a/orchestra/models/fields.py b/orchestra/models/fields.py index 88dffd23..9668ab6b 100644 --- a/orchestra/models/fields.py +++ b/orchestra/models/fields.py @@ -52,6 +52,12 @@ class MultiSelectField(models.CharField): return [ value for value,__ in arr_choices ] +class NullableCharField(models.CharField): + def get_db_prep_value(self, value, connection=None, prepared=False): + return value or None + + if isinstalled('south'): from south.modelsinspector import add_introspection_rules add_introspection_rules([], ["^orchestra\.models\.fields\.MultiSelectField"]) + add_introspection_rules([], ["^orchestra\.models\.fields\.NullableCharField"])