Resource monitoring implementation

This commit is contained in:
Marc 2014-07-10 10:03:22 +00:00
parent 9b9abc3c91
commit cc445559d0
10 changed files with 156 additions and 63 deletions

View File

@ -2,6 +2,8 @@ from django.conf import settings as django_settings
from django.db import models from django.db import models
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from orchestra.core import services
from . import settings from . import settings
@ -23,3 +25,6 @@ class Account(models.Model):
def name(self): def name(self):
self._cached_name = getattr(self, '_cached_name', self.user.username) self._cached_name = getattr(self, '_cached_name', self.user.username)
return self._cached_name return self._cached_name
services.register(Account, menu=False)

View File

@ -30,7 +30,7 @@ class RouteAdmin(admin.ModelAdmin):
def display_model(self, route): def display_model(self, route):
try: try:
return route.get_backend().model return route.backend_class().model
except KeyError: except KeyError:
return "<span style='color: red;'>NOT AVAILABLE</span>" return "<span style='color: red;'>NOT AVAILABLE</span>"
display_model.short_description = _("model") display_model.short_description = _("model")
@ -38,7 +38,7 @@ class RouteAdmin(admin.ModelAdmin):
def display_actions(self, route): def display_actions(self, route):
try: try:
return '<br>'.join(route.get_backend().get_actions()) return '<br>'.join(route.backend_class().get_actions())
except KeyError: except KeyError:
return "<span style='color: red;'>NOT AVAILABLE</span>" return "<span style='color: red;'>NOT AVAILABLE</span>"
display_actions.short_description = _("actions") display_actions.short_description = _("actions")

View File

@ -67,6 +67,13 @@ class ServiceBackend(object):
def get_backends(cls): def get_backends(cls):
return cls.plugins return cls.plugins
@classmethod
def get_backend(cls, name):
for backend in ServiceMonitor.get_backends():
if backend.get_name() == name:
return backend
raise KeyError('This backend is not registered')
@classmethod @classmethod
def get_choices(cls): def get_choices(cls):
backends = cls.get_backends() backends = cls.get_backends()

View File

@ -5,6 +5,7 @@ from django.db import models
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from orchestra.utils.apps import autodiscover from orchestra.utils.apps import autodiscover
from orchestra.utils.functional import cached
from . import settings, manager from . import settings, manager
from .backends import ServiceBackend from .backends import ServiceBackend
@ -146,30 +147,39 @@ class Route(models.Model):
# raise ValidationError(msg % (self.backend, self.method) # raise ValidationError(msg % (self.backend, self.method)
@classmethod @classmethod
def get_servers(cls, operation): @cached
# TODO use cached data sctructure and refactor def get_routing_table(cls):
backend = operation.backend table = {}
servers = [] for route in cls.objects.filter(is_active=True):
for action in route.backend_class().get_actions():
key = (route.backend, action)
try: try:
routes = cls.objects.filter(is_active=True, backend=backend.get_name()) table[key].append(route)
except cls.DoesNotExist: except KeyError:
table[key] = [route]
return table
@classmethod
def get_servers(cls, operation):
table = cls.get_routing_table()
servers = []
key = (operation.backend.get_name(), operation.action)
try:
routes = table[key]
except KeyError:
return servers return servers
safe_locals = { safe_locals = {
'instance': operation.instance 'instance': operation.instance
} }
actions = backend.get_actions()
for route in routes: for route in routes:
if operation.action in actions and eval(route.match, safe_locals): if eval(route.match, safe_locals):
servers.append(route.host) servers.append(route.host)
return servers return servers
def get_backend(self): def backend_class(self):
for backend in ServiceBackend.get_backends(): return ServiceBackend.get_backend(self.backend)
if backend.get_name() == self.backend:
return backend
raise KeyError('This backend is not registered')
# def get_method_class(self): # def method_class(self):
# for method in MethodBackend.get_backends(): # for method in MethodBackend.get_backends():
# if method.get_name() == self.method: # if method.get_name() == self.method:
# return method # return method

View File

@ -5,6 +5,7 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.admin.filters import UsedContentTypeFilter from orchestra.admin.filters import UsedContentTypeFilter
from orchestra.admin.utils import insertattr, get_modeladmin from orchestra.admin.utils import insertattr, get_modeladmin
from orchestra.core import services
from orchestra.utils import running_syncdb from orchestra.utils import running_syncdb
from .forms import ResourceForm from .forms import ResourceForm
@ -12,6 +13,8 @@ from .models import Resource, ResourceData, MonitorData
class ResourceAdmin(admin.ModelAdmin): class ResourceAdmin(admin.ModelAdmin):
# TODO warning message server/celery should be restarted when creating things
list_display = ( list_display = (
'name', 'verbose_name', 'content_type', 'period', 'ondemand', 'name', 'verbose_name', 'content_type', 'period', 'ondemand',
'default_allocation', 'disable_trigger' 'default_allocation', 'disable_trigger'
@ -30,6 +33,13 @@ class ResourceAdmin(admin.ModelAdmin):
inlines.append(inline) inlines.append(inline)
modeladmin.inlines = inlines modeladmin.inlines = inlines
def formfield_for_dbfield(self, db_field, **kwargs):
""" filter service content_types """
if db_field.name == 'content_type':
models = [ model._meta.model_name for model in services.get().keys() ]
kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models)
return super(ResourceAdmin, self).formfield_for_dbfield(db_field, **kwargs)
class ResourceDataAdmin(admin.ModelAdmin): class ResourceDataAdmin(admin.ModelAdmin):
list_display = ('id', 'resource', 'used', 'allocated', 'last_update',) # TODO content_object list_display = ('id', 'resource', 'used', 'allocated', 'last_update',) # TODO content_object

View File

@ -9,10 +9,6 @@ class ResourcesConfig(AppConfig):
verbose_name = 'Resources' verbose_name = 'Resources'
def ready(self): def ready(self):
from .models import Resource
# TODO execute on Resource.save()
if not running_syncdb(): if not running_syncdb():
relation = generic.GenericRelation('resources.ResourceData') from .models import create_resource_relation
for resources in Resource.group_by_content_type(): create_resource_relation()
model = resources[0].content_type.model_class()
model.add_to_class('resources', relation)

View File

@ -9,7 +9,7 @@ class ResourceForm(forms.ModelForm):
required=False) required=False)
used = forms.IntegerField(label=_("Used"), widget=ShowTextWidget(), used = forms.IntegerField(label=_("Used"), widget=ShowTextWidget(),
required=False) required=False)
last_update = forms.CharField(label=_("Last update"), widget=ShowTextWidget(), last_update = forms.DateTimeField(label=_("Last update"), widget=ShowTextWidget(),
required=False) required=False)
allocated = forms.IntegerField(label=_("Allocated")) allocated = forms.IntegerField(label=_("Allocated"))
@ -21,7 +21,7 @@ class ResourceForm(forms.ModelForm):
super(ResourceForm, self).__init__(*args, **kwargs) super(ResourceForm, self).__init__(*args, **kwargs)
if self.resource: if self.resource:
self.fields['verbose_name'].initial = self.resource.verbose_name self.fields['verbose_name'].initial = self.resource.verbose_name
self.fields['used'].initial = self.resource.get_current() self.fields['used'].initial = self.resource.get_used() # TODO
if self.resource.ondemand: if self.resource.ondemand:
self.fields['allocated'].required = False self.fields['allocated'].required = False
self.fields['allocated'].widget = ReadOnlyWidget(None, '') self.fields['allocated'].widget = ReadOnlyWidget(None, '')

View File

@ -42,12 +42,35 @@ class Resource(models.Model):
null=True, blank=True) null=True, blank=True)
is_active = models.BooleanField(_("is active"), default=True) is_active = models.BooleanField(_("is active"), default=True)
disable_trigger = models.BooleanField(_("disable trigger"), default=False) disable_trigger = models.BooleanField(_("disable trigger"), default=False)
crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"),
help_text=_("Crontab for periodic execution"))
# TODO create custom field that returns backend python objects
monitors = MultiSelectField(_("monitors"), max_length=256, monitors = MultiSelectField(_("monitors"), max_length=256,
choices=ServiceMonitor.get_choices()) choices=ServiceMonitor.get_choices())
def __unicode__(self): def __unicode__(self):
return self.name return self.name
def save(self, *args, **kwargs):
super(Resource, self).save(*args, **kwargs)
# Create Celery periodic task
name = 'monitor.%s' % str(self)
try:
task = PeriodicTask.objects.get(name=name)
except PeriodicTask.DoesNotExist:
PeriodicTask.objects.create(name=name, task='resources.Monitor',
args=[self.pk], crontab=self.crontab)
else:
if task.crontab != self.crontab:
task.crontab = self.crontab
task.save()
def delete(self, *args, **kwargs):
super(Resource, self).delete(*args, **kwargs)
name = 'monitor.%s' % str(self)
PeriodicTask.objects.filter(name=name, task='resources.Monitor',
args=[self.pk]).delete()
@classmethod @classmethod
def group_by_content_type(cls): def group_by_content_type(cls):
prev = None prev = None
@ -64,13 +87,39 @@ class Resource(models.Model):
if group: if group:
yield group yield group
def get_current(self):
class ResourceData(models.Model):
""" Stores computed resource usage and allocation """
resource = models.ForeignKey(Resource, related_name='dataset')
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
used = models.PositiveIntegerField(null=True)
last_update = models.DateTimeField(null=True)
allocated = models.PositiveIntegerField(null=True)
content_object = generic.GenericForeignKey()
class Meta:
unique_together = ('resource', 'content_type', 'object_id')
verbose_name_plural = _("resource data")
@classmethod
def get_or_create(cls, obj, resource):
try:
return cls.objects.get(content_object=obj, resource=resource)
except cls.DoesNotExists:
return cls.objects.create(content_object=obj, resource=resource,
allocated=resource.defalt_allocation)
def get_used(self):
resource = self.resource
today = datetime.date.today() today = datetime.date.today()
result = 0 result = 0
has_result = False has_result = False
for monitor in self.monitors: for monitor in resource.monitors:
dataset = MonitorData.objects.filter(monitor=monitor) dataset = MonitorData.objects.filter(monitor=monitor,
if self.period == self.MONTHLY_AVG: content_type=self.content_type, object_id=self.object_id)
if resource.period == resource.MONTHLY_AVG:
try: try:
last = dataset.latest() last = dataset.latest()
except MonitorData.DoesNotExist: except MonitorData.DoesNotExist:
@ -83,14 +132,14 @@ class Resource(models.Model):
for data in dataset: for data in dataset:
slot = (previous-data.date).total_seconds() slot = (previous-data.date).total_seconds()
result += data.value * slot/total result += data.value * slot/total
elif self.period == self.MONTHLY_SUM: elif resource.period == resource.MONTHLY_SUM:
data = dataset.filter(date__year=today.year, data = dataset.filter(date__year=today.year,
date__month=today.month) date__month=today.month)
value = data.aggregate(models.Sum('value'))['value__sum'] value = data.aggregate(models.Sum('value'))['value__sum']
if value: if value:
has_result = True has_result = True
result += value result += value
elif self.period == self.LAST: elif resource.period == resource.LAST:
try: try:
result += dataset.latest().value result += dataset.latest().value
except MonitorData.DoesNotExist: except MonitorData.DoesNotExist:
@ -101,21 +150,6 @@ class Resource(models.Model):
return result if has_result else None return result if has_result else None
class ResourceData(models.Model):
""" Stores computed resource usage and allocation """
resource = models.ForeignKey(Resource)
content_type = models.ForeignKey(ContentType)
object_id = models.PositiveIntegerField()
used = models.PositiveIntegerField(null=True)
last_update = models.DateTimeField(null=True)
allocated = models.PositiveIntegerField(null=True)
content_object = generic.GenericForeignKey()
class Meta:
unique_together = ('resource', 'content_type', 'object_id')
verbose_name_plural = _("resource data")
class MonitorData(models.Model): class MonitorData(models.Model):
""" Stores monitored data """ """ Stores monitored data """
monitor = models.CharField(_("monitor"), max_length=256, monitor = models.CharField(_("monitor"), max_length=256,
@ -133,3 +167,10 @@ class MonitorData(models.Model):
def __unicode__(self): def __unicode__(self):
return str(self.monitor) return str(self.monitor)
def create_resource_relation():
relation = generic.GenericRelation('resources.ResourceData')
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
model.add_to_class('resources', relation)

View File

@ -1,14 +1,38 @@
from celery import shared_task from celery import shared_task
from orchestra.apps.orchestration.models import BackendOperation as Operation
from .backends import ServiceMonitor from .backends import ServiceMonitor
from .models import MonitorData
@shared_task @shared_task(name='resources.Monitor')
def monitor(backend_name): def monitor(resource_id):
routes = Route.objects.filter(is_active=True, backend=backend_name) resource = Resource.objects.get(pk=resource_id)
for route in routes:
pass # Execute monitors
for backend in ServiceMonitor.get_backends(): for monitor_name in resource.monitors:
if backend.get_name() == backend_name: backend = ServiceMonitor.get_backend(monitor_name)
# TODO execute monitor BackendOperation model = backend.model
pass operations = []
# Execute monitor
for obj in model.objects.all():
operations.append(Operation.create(backend, obj, Operation.MONITOR))
Operation.execute(operations)
# Update used resources and trigger resource exceeded and revovery
operations = []
model = resource.model
for obj in model.objects.all():
data = MonitorData.get_or_create(obj, resource)
current = data.get_used()
if data.used < data.allocated and current > data.allocated:
op = Operation.create(backend, data.content_object, Operation.EXCEED)
operations.append(op)
elif res.used > res.allocated and current < res.allocated:
op = Operation.create(backend, data.content_object, Operation.RECOVERY)
operation.append(op)
data.used = current
data.las_update = datetime.now()
data.save()
Operation.execute(operations)