From bc398644b7d5cc4dee0415885603eb5489fe9ef4 Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 11 Jul 2014 22:08:16 +0000 Subject: [PATCH] Fixes on resource monitoring --- orchestra/apps/resources/helpers.py | 68 +++++++++++++++++++++++++++++ orchestra/apps/resources/models.py | 60 ++----------------------- orchestra/models/utils.py | 19 ++++---- 3 files changed, 79 insertions(+), 68 deletions(-) create mode 100644 orchestra/apps/resources/helpers.py diff --git a/orchestra/apps/resources/helpers.py b/orchestra/apps/resources/helpers.py new file mode 100644 index 00000000..11a218f2 --- /dev/null +++ b/orchestra/apps/resources/helpers.py @@ -0,0 +1,68 @@ +import datetime + +from django.contrib.contenttypes.models import ContentType +from django.db.models.loading import get_model +from django.utils import timezone + +from orchestra.models.utils import get_model_field_path + +from .backends import ServiceMonitor + + +def get_used_resource(data): + """ Computes MonitorData.used based on related monitors """ + MonitorData = type(data) + resource = data.resource + today = timezone.now() + result = 0 + has_result = False + for monitor in resource.monitors: + # Get related dataset + resource_model = data.content_type.model_class() + monitor_model = get_model(ServiceMonitor.get_backend(monitor).model) + if resource_model == monitor_model: + dataset = MonitorData.objects.filter(monitor=monitor, + content_type=data.content_type_id, object_id=data.object_id) + else: + path = get_model_field_path(monitor_model, resource_model) + fields = '__'.join(path) + objects = monitor_model.objects.filter(**{fields: data.object_id}) + pks = objects.values_list('id', flat=True) + ct = ContentType.objects.get_for_model(monitor_model) + dataset = MonitorData.objects.filter(monitor=monitor, + content_type=ct, object_id__in=pks) + + # Process dataset according to resource.period + if resource.period == resource.MONTHLY_AVG: + try: + last = dataset.latest() + except MonitorData.DoesNotExist: + continue + has_result = True + epoch = datetime(year=today.year, month=today.month, day=1, + tzinfo=timezone.utc) + total = (epoch-last.date).total_seconds() + dataset = dataset.filter(date__year=today.year, + date__month=today.month) + for data in dataset: + slot = (previous-data.date).total_seconds() + result += data.value * slot/total + elif resource.period == resource.MONTHLY_SUM: + dataset = dataset.filter(date__year=today.year, date__month=today.month) + # FIXME Aggregation of 0s returns None! django bug? + # value = dataset.aggregate(models.Sum('value'))['value__sum'] + values = dataset.values_list('value', flat=True) + if values: + has_result = True + result += sum(values) + elif resource.period == resource.LAST: + try: + result += dataset.latest().value + except MonitorData.DoesNotExist: + continue + has_result = True + else: + msg = "%s support not implemented" % data.period + raise NotImplementedError(msg) + + return result if has_result else None diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index 73609738..80d43e21 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -1,18 +1,13 @@ -import datetime - -from django.db import models -from django.db.models.loading import get_model from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.contrib.contenttypes.models import ContentType from django.core import validators -from django.utils import timezone +from django.db import models from django.utils.translation import ugettext_lazy as _ from djcelery.models import PeriodicTask, CrontabSchedule from orchestra.models.fields import MultiSelectField -from orchestra.models.utils import get_model_field_path -from orchestra.utils.apps import autodiscover +from . import helpers from .backends import ServiceMonitor @@ -131,56 +126,7 @@ class ResourceData(models.Model): allocated=resource.default_allocation) def get_used(self): - resource = self.resource - today = timezone.now() - result = 0 - has_result = False - for monitor in resource.monitors: - resource_model = self.content_type.model_class() - monitor_model = get_model(ServiceMonitor.get_backend(monitor).model) - if resource_model == monitor_model: - dataset = MonitorData.objects.filter(monitor=monitor, - content_type=self.content_type_id, object_id=self.object_id) - else: - path = get_model_field_path(monitor_model, resource_model) - fields = '__'.join(path) - objects = monitor_model.objects.filter(**{fields: self.object_id}) - pks = objects.values_list('id', flat=True) - ct = ContentType.objects.get_for_model(monitor_model) - dataset = MonitorData.objects.filter(monitor=monitor, - content_type=ct, object_id__in=pks) - if resource.period == resource.MONTHLY_AVG: - try: - last = dataset.latest() - except MonitorData.DoesNotExist: - continue - has_result = True - epoch = datetime(year=today.year, month=today.month, day=1, - tzinfo=timezone.utc) - total = (epoch-last.date).total_seconds() - dataset = dataset.filter(date__year=today.year, - date__month=today.month) - for data in dataset: - slot = (previous-data.date).total_seconds() - result += data.value * slot/total - elif resource.period == resource.MONTHLY_SUM: - data = dataset.filter(date__year=today.year, date__month=today.month) - # FIXME Aggregation of 0s returns None! django bug? - # value = data.aggregate(models.Sum('value'))['value__sum'] - values = data.values_list('value', flat=True) - if values: - has_result = True - result += sum(values) - elif resource.period == resource.LAST: - try: - result += dataset.latest().value - except MonitorData.DoesNotExist: - continue - has_result = True - else: - msg = "%s support not implemented" % self.period - raise NotImplementedError(msg) - return result if has_result else None + return helpers.get_used(self) class MonitorData(models.Model): diff --git a/orchestra/models/utils.py b/orchestra/models/utils.py index 445b9619..ff4d9874 100644 --- a/orchestra/models/utils.py +++ b/orchestra/models/utils.py @@ -4,7 +4,6 @@ from django.utils import importlib def get_model(label, import_module=True): - """ returns the modeladmin registred for model """ app_label, model_name = label.split('.') model = loading.get_model(app_label, model_name) if model is None: @@ -43,21 +42,20 @@ def get_field_value(obj, field_name): try: rel = getattr(rel, name) except AttributeError: - # maybe it is a query manager + # maybe is a query manager rel = getattr(rel.get(), name) return rel def get_model_field_path(origin, target): """ BFS search on model relaion fields """ - mqueue = [] - mqueue.append([origin]) - pqueue = [[]] - while mqueue: - model = mqueue.pop(0) - path = pqueue.pop(0) + queue = [] + queue.append(([origin], [])) + while queue: + model, path = queue.pop(0) if len(model) > 4: - raise RuntimeError('maximum recursion depth exceeded while looking for %s" % target') + msg = "maximum recursion depth exceeded while looking for %s" + raise RuntimeError(msg % target) node = model[-1] if node == target: return path @@ -65,7 +63,6 @@ def get_model_field_path(origin, target): if field.rel: new_model = list(model) new_model.append(field.rel.to) - mqueue.append(new_model) new_path = list(path) new_path.append(field.name) - pqueue.append(new_path) + queue.append((new_model, new_path))