Resource monitoring implementation
This commit is contained in:
parent
9b9abc3c91
commit
cc445559d0
|
@ -2,6 +2,8 @@ from django.conf import settings as django_settings
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
|
from orchestra.core import services
|
||||||
|
|
||||||
from . import settings
|
from . import settings
|
||||||
|
|
||||||
|
|
||||||
|
@ -23,3 +25,6 @@ class Account(models.Model):
|
||||||
def name(self):
|
def name(self):
|
||||||
self._cached_name = getattr(self, '_cached_name', self.user.username)
|
self._cached_name = getattr(self, '_cached_name', self.user.username)
|
||||||
return self._cached_name
|
return self._cached_name
|
||||||
|
|
||||||
|
|
||||||
|
services.register(Account, menu=False)
|
||||||
|
|
|
@ -30,7 +30,7 @@ class RouteAdmin(admin.ModelAdmin):
|
||||||
|
|
||||||
def display_model(self, route):
|
def display_model(self, route):
|
||||||
try:
|
try:
|
||||||
return route.get_backend().model
|
return route.backend_class().model
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return "<span style='color: red;'>NOT AVAILABLE</span>"
|
return "<span style='color: red;'>NOT AVAILABLE</span>"
|
||||||
display_model.short_description = _("model")
|
display_model.short_description = _("model")
|
||||||
|
@ -38,7 +38,7 @@ class RouteAdmin(admin.ModelAdmin):
|
||||||
|
|
||||||
def display_actions(self, route):
|
def display_actions(self, route):
|
||||||
try:
|
try:
|
||||||
return '<br>'.join(route.get_backend().get_actions())
|
return '<br>'.join(route.backend_class().get_actions())
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return "<span style='color: red;'>NOT AVAILABLE</span>"
|
return "<span style='color: red;'>NOT AVAILABLE</span>"
|
||||||
display_actions.short_description = _("actions")
|
display_actions.short_description = _("actions")
|
||||||
|
|
|
@ -67,6 +67,13 @@ class ServiceBackend(object):
|
||||||
def get_backends(cls):
|
def get_backends(cls):
|
||||||
return cls.plugins
|
return cls.plugins
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_backend(cls, name):
|
||||||
|
for backend in ServiceMonitor.get_backends():
|
||||||
|
if backend.get_name() == name:
|
||||||
|
return backend
|
||||||
|
raise KeyError('This backend is not registered')
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_choices(cls):
|
def get_choices(cls):
|
||||||
backends = cls.get_backends()
|
backends = cls.get_backends()
|
||||||
|
|
|
@ -5,6 +5,7 @@ from django.db import models
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from orchestra.utils.apps import autodiscover
|
from orchestra.utils.apps import autodiscover
|
||||||
|
from orchestra.utils.functional import cached
|
||||||
|
|
||||||
from . import settings, manager
|
from . import settings, manager
|
||||||
from .backends import ServiceBackend
|
from .backends import ServiceBackend
|
||||||
|
@ -146,30 +147,39 @@ class Route(models.Model):
|
||||||
# raise ValidationError(msg % (self.backend, self.method)
|
# raise ValidationError(msg % (self.backend, self.method)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_servers(cls, operation):
|
@cached
|
||||||
# TODO use cached data sctructure and refactor
|
def get_routing_table(cls):
|
||||||
backend = operation.backend
|
table = {}
|
||||||
servers = []
|
for route in cls.objects.filter(is_active=True):
|
||||||
|
for action in route.backend_class().get_actions():
|
||||||
|
key = (route.backend, action)
|
||||||
try:
|
try:
|
||||||
routes = cls.objects.filter(is_active=True, backend=backend.get_name())
|
table[key].append(route)
|
||||||
except cls.DoesNotExist:
|
except KeyError:
|
||||||
|
table[key] = [route]
|
||||||
|
return table
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_servers(cls, operation):
|
||||||
|
table = cls.get_routing_table()
|
||||||
|
servers = []
|
||||||
|
key = (operation.backend.get_name(), operation.action)
|
||||||
|
try:
|
||||||
|
routes = table[key]
|
||||||
|
except KeyError:
|
||||||
return servers
|
return servers
|
||||||
safe_locals = {
|
safe_locals = {
|
||||||
'instance': operation.instance
|
'instance': operation.instance
|
||||||
}
|
}
|
||||||
actions = backend.get_actions()
|
|
||||||
for route in routes:
|
for route in routes:
|
||||||
if operation.action in actions and eval(route.match, safe_locals):
|
if eval(route.match, safe_locals):
|
||||||
servers.append(route.host)
|
servers.append(route.host)
|
||||||
return servers
|
return servers
|
||||||
|
|
||||||
def get_backend(self):
|
def backend_class(self):
|
||||||
for backend in ServiceBackend.get_backends():
|
return ServiceBackend.get_backend(self.backend)
|
||||||
if backend.get_name() == self.backend:
|
|
||||||
return backend
|
|
||||||
raise KeyError('This backend is not registered')
|
|
||||||
|
|
||||||
# def get_method_class(self):
|
# def method_class(self):
|
||||||
# for method in MethodBackend.get_backends():
|
# for method in MethodBackend.get_backends():
|
||||||
# if method.get_name() == self.method:
|
# if method.get_name() == self.method:
|
||||||
# return method
|
# return method
|
||||||
|
|
|
@ -5,6 +5,7 @@ from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from orchestra.admin.filters import UsedContentTypeFilter
|
from orchestra.admin.filters import UsedContentTypeFilter
|
||||||
from orchestra.admin.utils import insertattr, get_modeladmin
|
from orchestra.admin.utils import insertattr, get_modeladmin
|
||||||
|
from orchestra.core import services
|
||||||
from orchestra.utils import running_syncdb
|
from orchestra.utils import running_syncdb
|
||||||
|
|
||||||
from .forms import ResourceForm
|
from .forms import ResourceForm
|
||||||
|
@ -12,6 +13,8 @@ from .models import Resource, ResourceData, MonitorData
|
||||||
|
|
||||||
|
|
||||||
class ResourceAdmin(admin.ModelAdmin):
|
class ResourceAdmin(admin.ModelAdmin):
|
||||||
|
# TODO warning message server/celery should be restarted when creating things
|
||||||
|
|
||||||
list_display = (
|
list_display = (
|
||||||
'name', 'verbose_name', 'content_type', 'period', 'ondemand',
|
'name', 'verbose_name', 'content_type', 'period', 'ondemand',
|
||||||
'default_allocation', 'disable_trigger'
|
'default_allocation', 'disable_trigger'
|
||||||
|
@ -30,6 +33,13 @@ class ResourceAdmin(admin.ModelAdmin):
|
||||||
inlines.append(inline)
|
inlines.append(inline)
|
||||||
modeladmin.inlines = inlines
|
modeladmin.inlines = inlines
|
||||||
|
|
||||||
|
def formfield_for_dbfield(self, db_field, **kwargs):
|
||||||
|
""" filter service content_types """
|
||||||
|
if db_field.name == 'content_type':
|
||||||
|
models = [ model._meta.model_name for model in services.get().keys() ]
|
||||||
|
kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models)
|
||||||
|
return super(ResourceAdmin, self).formfield_for_dbfield(db_field, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
class ResourceDataAdmin(admin.ModelAdmin):
|
class ResourceDataAdmin(admin.ModelAdmin):
|
||||||
list_display = ('id', 'resource', 'used', 'allocated', 'last_update',) # TODO content_object
|
list_display = ('id', 'resource', 'used', 'allocated', 'last_update',) # TODO content_object
|
||||||
|
|
|
@ -9,10 +9,6 @@ class ResourcesConfig(AppConfig):
|
||||||
verbose_name = 'Resources'
|
verbose_name = 'Resources'
|
||||||
|
|
||||||
def ready(self):
|
def ready(self):
|
||||||
from .models import Resource
|
|
||||||
# TODO execute on Resource.save()
|
|
||||||
if not running_syncdb():
|
if not running_syncdb():
|
||||||
relation = generic.GenericRelation('resources.ResourceData')
|
from .models import create_resource_relation
|
||||||
for resources in Resource.group_by_content_type():
|
create_resource_relation()
|
||||||
model = resources[0].content_type.model_class()
|
|
||||||
model.add_to_class('resources', relation)
|
|
||||||
|
|
|
@ -9,7 +9,7 @@ class ResourceForm(forms.ModelForm):
|
||||||
required=False)
|
required=False)
|
||||||
used = forms.IntegerField(label=_("Used"), widget=ShowTextWidget(),
|
used = forms.IntegerField(label=_("Used"), widget=ShowTextWidget(),
|
||||||
required=False)
|
required=False)
|
||||||
last_update = forms.CharField(label=_("Last update"), widget=ShowTextWidget(),
|
last_update = forms.DateTimeField(label=_("Last update"), widget=ShowTextWidget(),
|
||||||
required=False)
|
required=False)
|
||||||
allocated = forms.IntegerField(label=_("Allocated"))
|
allocated = forms.IntegerField(label=_("Allocated"))
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ class ResourceForm(forms.ModelForm):
|
||||||
super(ResourceForm, self).__init__(*args, **kwargs)
|
super(ResourceForm, self).__init__(*args, **kwargs)
|
||||||
if self.resource:
|
if self.resource:
|
||||||
self.fields['verbose_name'].initial = self.resource.verbose_name
|
self.fields['verbose_name'].initial = self.resource.verbose_name
|
||||||
self.fields['used'].initial = self.resource.get_current()
|
self.fields['used'].initial = self.resource.get_used() # TODO
|
||||||
if self.resource.ondemand:
|
if self.resource.ondemand:
|
||||||
self.fields['allocated'].required = False
|
self.fields['allocated'].required = False
|
||||||
self.fields['allocated'].widget = ReadOnlyWidget(None, '')
|
self.fields['allocated'].widget = ReadOnlyWidget(None, '')
|
||||||
|
|
|
@ -42,12 +42,35 @@ class Resource(models.Model):
|
||||||
null=True, blank=True)
|
null=True, blank=True)
|
||||||
is_active = models.BooleanField(_("is active"), default=True)
|
is_active = models.BooleanField(_("is active"), default=True)
|
||||||
disable_trigger = models.BooleanField(_("disable trigger"), default=False)
|
disable_trigger = models.BooleanField(_("disable trigger"), default=False)
|
||||||
|
crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"),
|
||||||
|
help_text=_("Crontab for periodic execution"))
|
||||||
|
# TODO create custom field that returns backend python objects
|
||||||
monitors = MultiSelectField(_("monitors"), max_length=256,
|
monitors = MultiSelectField(_("monitors"), max_length=256,
|
||||||
choices=ServiceMonitor.get_choices())
|
choices=ServiceMonitor.get_choices())
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return self.name
|
return self.name
|
||||||
|
|
||||||
|
def save(self, *args, **kwargs):
|
||||||
|
super(Resource, self).save(*args, **kwargs)
|
||||||
|
# Create Celery periodic task
|
||||||
|
name = 'monitor.%s' % str(self)
|
||||||
|
try:
|
||||||
|
task = PeriodicTask.objects.get(name=name)
|
||||||
|
except PeriodicTask.DoesNotExist:
|
||||||
|
PeriodicTask.objects.create(name=name, task='resources.Monitor',
|
||||||
|
args=[self.pk], crontab=self.crontab)
|
||||||
|
else:
|
||||||
|
if task.crontab != self.crontab:
|
||||||
|
task.crontab = self.crontab
|
||||||
|
task.save()
|
||||||
|
|
||||||
|
def delete(self, *args, **kwargs):
|
||||||
|
super(Resource, self).delete(*args, **kwargs)
|
||||||
|
name = 'monitor.%s' % str(self)
|
||||||
|
PeriodicTask.objects.filter(name=name, task='resources.Monitor',
|
||||||
|
args=[self.pk]).delete()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def group_by_content_type(cls):
|
def group_by_content_type(cls):
|
||||||
prev = None
|
prev = None
|
||||||
|
@ -64,13 +87,39 @@ class Resource(models.Model):
|
||||||
if group:
|
if group:
|
||||||
yield group
|
yield group
|
||||||
|
|
||||||
def get_current(self):
|
|
||||||
|
class ResourceData(models.Model):
|
||||||
|
""" Stores computed resource usage and allocation """
|
||||||
|
resource = models.ForeignKey(Resource, related_name='dataset')
|
||||||
|
content_type = models.ForeignKey(ContentType)
|
||||||
|
object_id = models.PositiveIntegerField()
|
||||||
|
used = models.PositiveIntegerField(null=True)
|
||||||
|
last_update = models.DateTimeField(null=True)
|
||||||
|
allocated = models.PositiveIntegerField(null=True)
|
||||||
|
|
||||||
|
content_object = generic.GenericForeignKey()
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
unique_together = ('resource', 'content_type', 'object_id')
|
||||||
|
verbose_name_plural = _("resource data")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_or_create(cls, obj, resource):
|
||||||
|
try:
|
||||||
|
return cls.objects.get(content_object=obj, resource=resource)
|
||||||
|
except cls.DoesNotExists:
|
||||||
|
return cls.objects.create(content_object=obj, resource=resource,
|
||||||
|
allocated=resource.defalt_allocation)
|
||||||
|
|
||||||
|
def get_used(self):
|
||||||
|
resource = self.resource
|
||||||
today = datetime.date.today()
|
today = datetime.date.today()
|
||||||
result = 0
|
result = 0
|
||||||
has_result = False
|
has_result = False
|
||||||
for monitor in self.monitors:
|
for monitor in resource.monitors:
|
||||||
dataset = MonitorData.objects.filter(monitor=monitor)
|
dataset = MonitorData.objects.filter(monitor=monitor,
|
||||||
if self.period == self.MONTHLY_AVG:
|
content_type=self.content_type, object_id=self.object_id)
|
||||||
|
if resource.period == resource.MONTHLY_AVG:
|
||||||
try:
|
try:
|
||||||
last = dataset.latest()
|
last = dataset.latest()
|
||||||
except MonitorData.DoesNotExist:
|
except MonitorData.DoesNotExist:
|
||||||
|
@ -83,14 +132,14 @@ class Resource(models.Model):
|
||||||
for data in dataset:
|
for data in dataset:
|
||||||
slot = (previous-data.date).total_seconds()
|
slot = (previous-data.date).total_seconds()
|
||||||
result += data.value * slot/total
|
result += data.value * slot/total
|
||||||
elif self.period == self.MONTHLY_SUM:
|
elif resource.period == resource.MONTHLY_SUM:
|
||||||
data = dataset.filter(date__year=today.year,
|
data = dataset.filter(date__year=today.year,
|
||||||
date__month=today.month)
|
date__month=today.month)
|
||||||
value = data.aggregate(models.Sum('value'))['value__sum']
|
value = data.aggregate(models.Sum('value'))['value__sum']
|
||||||
if value:
|
if value:
|
||||||
has_result = True
|
has_result = True
|
||||||
result += value
|
result += value
|
||||||
elif self.period == self.LAST:
|
elif resource.period == resource.LAST:
|
||||||
try:
|
try:
|
||||||
result += dataset.latest().value
|
result += dataset.latest().value
|
||||||
except MonitorData.DoesNotExist:
|
except MonitorData.DoesNotExist:
|
||||||
|
@ -101,21 +150,6 @@ class Resource(models.Model):
|
||||||
return result if has_result else None
|
return result if has_result else None
|
||||||
|
|
||||||
|
|
||||||
class ResourceData(models.Model):
|
|
||||||
""" Stores computed resource usage and allocation """
|
|
||||||
resource = models.ForeignKey(Resource)
|
|
||||||
content_type = models.ForeignKey(ContentType)
|
|
||||||
object_id = models.PositiveIntegerField()
|
|
||||||
used = models.PositiveIntegerField(null=True)
|
|
||||||
last_update = models.DateTimeField(null=True)
|
|
||||||
allocated = models.PositiveIntegerField(null=True)
|
|
||||||
|
|
||||||
content_object = generic.GenericForeignKey()
|
|
||||||
|
|
||||||
class Meta:
|
|
||||||
unique_together = ('resource', 'content_type', 'object_id')
|
|
||||||
verbose_name_plural = _("resource data")
|
|
||||||
|
|
||||||
class MonitorData(models.Model):
|
class MonitorData(models.Model):
|
||||||
""" Stores monitored data """
|
""" Stores monitored data """
|
||||||
monitor = models.CharField(_("monitor"), max_length=256,
|
monitor = models.CharField(_("monitor"), max_length=256,
|
||||||
|
@ -133,3 +167,10 @@ class MonitorData(models.Model):
|
||||||
|
|
||||||
def __unicode__(self):
|
def __unicode__(self):
|
||||||
return str(self.monitor)
|
return str(self.monitor)
|
||||||
|
|
||||||
|
|
||||||
|
def create_resource_relation():
|
||||||
|
relation = generic.GenericRelation('resources.ResourceData')
|
||||||
|
for resources in Resource.group_by_content_type():
|
||||||
|
model = resources[0].content_type.model_class()
|
||||||
|
model.add_to_class('resources', relation)
|
||||||
|
|
|
@ -1,14 +1,38 @@
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
|
|
||||||
|
from orchestra.apps.orchestration.models import BackendOperation as Operation
|
||||||
|
|
||||||
from .backends import ServiceMonitor
|
from .backends import ServiceMonitor
|
||||||
|
from .models import MonitorData
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task(name='resources.Monitor')
|
||||||
def monitor(backend_name):
|
def monitor(resource_id):
|
||||||
routes = Route.objects.filter(is_active=True, backend=backend_name)
|
resource = Resource.objects.get(pk=resource_id)
|
||||||
for route in routes:
|
|
||||||
pass
|
# Execute monitors
|
||||||
for backend in ServiceMonitor.get_backends():
|
for monitor_name in resource.monitors:
|
||||||
if backend.get_name() == backend_name:
|
backend = ServiceMonitor.get_backend(monitor_name)
|
||||||
# TODO execute monitor BackendOperation
|
model = backend.model
|
||||||
pass
|
operations = []
|
||||||
|
# Execute monitor
|
||||||
|
for obj in model.objects.all():
|
||||||
|
operations.append(Operation.create(backend, obj, Operation.MONITOR))
|
||||||
|
Operation.execute(operations)
|
||||||
|
|
||||||
|
# Update used resources and trigger resource exceeded and revovery
|
||||||
|
operations = []
|
||||||
|
model = resource.model
|
||||||
|
for obj in model.objects.all():
|
||||||
|
data = MonitorData.get_or_create(obj, resource)
|
||||||
|
current = data.get_used()
|
||||||
|
if data.used < data.allocated and current > data.allocated:
|
||||||
|
op = Operation.create(backend, data.content_object, Operation.EXCEED)
|
||||||
|
operations.append(op)
|
||||||
|
elif res.used > res.allocated and current < res.allocated:
|
||||||
|
op = Operation.create(backend, data.content_object, Operation.RECOVERY)
|
||||||
|
operation.append(op)
|
||||||
|
data.used = current
|
||||||
|
data.las_update = datetime.now()
|
||||||
|
data.save()
|
||||||
|
Operation.execute(operations)
|
||||||
|
|
Loading…
Reference in New Issue