From c731a73889f93fdcba8505b41c35aeddb6ffcb38 Mon Sep 17 00:00:00 2001 From: Marc Date: Mon, 21 Jul 2014 12:20:04 +0000 Subject: [PATCH] Added support for service handlers --- orchestra/apps/accounts/admin.py | 2 +- orchestra/apps/orchestration/backends.py | 24 +---- orchestra/apps/orchestration/models.py | 2 +- orchestra/apps/orders/admin.py | 15 +-- orchestra/apps/orders/handlers.py | 39 +++++++ orchestra/apps/orders/models.py | 131 ++++++++++++++--------- orchestra/apps/resources/admin.py | 2 +- orchestra/apps/resources/backends.py | 4 +- orchestra/apps/resources/models.py | 4 +- orchestra/conf/base_settings.py | 1 + orchestra/core/caches.py | 39 +++++++ orchestra/utils/functional.py | 13 ++- orchestra/utils/plugins.py | 37 +++++++ 13 files changed, 224 insertions(+), 89 deletions(-) create mode 100644 orchestra/apps/orders/handlers.py create mode 100644 orchestra/core/caches.py diff --git a/orchestra/apps/accounts/admin.py b/orchestra/apps/accounts/admin.py index ad315953..18645882 100644 --- a/orchestra/apps/accounts/admin.py +++ b/orchestra/apps/accounts/admin.py @@ -61,7 +61,7 @@ class AccountAdmin(ExtendedModelAdmin): messages.warning(request, 'This account is disabled.') context = { 'services': sorted( - [ model._meta for model in services.get().keys() ], + [ model._meta for model in services.get() ], key=lambda i: i.verbose_name_plural.lower() ) } diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index a3626b7d..bb058d78 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -8,7 +8,7 @@ from orchestra.utils import plugins from . import methods -class ServiceBackend(object): +class ServiceBackend(plugins.Plugin): """ Service management backend base class @@ -16,7 +16,6 @@ class ServiceBackend(object): be conviniently supported. Each backend generates the configuration for all the changes of all modified objects, reloading the daemon just once. """ - verbose_name = None model = None related_models = () # ((model, accessor__attribute),) script_method = methods.BashSSH @@ -65,28 +64,11 @@ class ServiceBackend(object): @classmethod def get_backends(cls): - return cls.plugins + return cls.get_plugins() @classmethod def get_backend(cls, name): - for backend in ServiceBackend.get_backends(): - if backend.get_name() == name: - return backend - raise KeyError('This backend is not registered') - - @classmethod - def get_choices(cls): - backends = cls.get_backends() - choices = [] - for b in backends: - # don't evaluate b.verbose_name ugettext_lazy - verbose = getattr(b.verbose_name, '_proxy____args', [b.verbose_name]) - if verbose[0]: - verbose = b.verbose_name - else: - verbose = b.get_name() - choices.append((b.get_name(), verbose)) - return sorted(choices, key=lambda e: e[0]) + return cls.get_plugin(name) def get_banner(self): time = timezone.now().strftime("%h %d, %Y %I:%M:%S") diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index e852c3df..bdafe42e 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -125,7 +125,7 @@ class Route(models.Model): Defines the routing that determine in which server a backend is executed """ backend = models.CharField(_("backend"), max_length=256, - choices=ServiceBackend.get_choices()) + choices=ServiceBackend.get_plugin_choices()) host = models.ForeignKey(Server, verbose_name=_("host")) match = models.CharField(_("match"), max_length=256, blank=True, default='True', help_text=_("Python expression used for selecting the targe host, " diff --git a/orchestra/apps/orders/admin.py b/orchestra/apps/orders/admin.py index 5f2249ad..b51a91c6 100644 --- a/orchestra/apps/orders/admin.py +++ b/orchestra/apps/orders/admin.py @@ -12,7 +12,7 @@ class ServiceAdmin(admin.ModelAdmin): fieldsets = ( (None, { 'classes': ('wide',), - 'fields': ('description', 'model', 'match', 'is_active') + 'fields': ('description', 'content_type', 'match', 'handler', 'is_active') }), (_("Billing options"), { 'classes': ('wide',), @@ -22,16 +22,17 @@ class ServiceAdmin(admin.ModelAdmin): (_("Pricing options"), { 'classes': ('wide',), 'fields': ('metric', 'pricing_period', 'rate_algorithm', - 'orders_effect', ('on_cancel', 'on_disable', 'on_register'), - 'payment_style', 'trial_period', 'refound_period', 'tax',) + 'orders_effect', 'on_cancel', 'payment_style', + 'trial_period', 'refound_period', 'tax') }), ) def formfield_for_dbfield(self, db_field, **kwargs): """ Improve performance of account field and filter by account """ - if db_field.name == 'model': - models = [model._meta.model_name for model in services.get().keys()] - kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models) + if db_field.name == 'content_type': + models = [model._meta.model_name for model in services.get()] + queryset = db_field.rel.to.objects + kwargs['queryset'] = queryset.filter(model__in=models) if db_field.name in ['match', 'metric']: kwargs['widget'] = forms.TextInput(attrs={'size':'160'}) return super(ServiceAdmin, self).formfield_for_dbfield(db_field, **kwargs) @@ -43,7 +44,7 @@ class OrderAdmin(AccountAdminMixin, admin.ModelAdmin): class MetricStorageAdmin(admin.ModelAdmin): - list_display = ('order', 'value', 'date') + list_display = ('order', 'value', 'created_on', 'updated_on') list_filter = ('order__service',) diff --git a/orchestra/apps/orders/handlers.py b/orchestra/apps/orders/handlers.py new file mode 100644 index 00000000..2de9cbb8 --- /dev/null +++ b/orchestra/apps/orders/handlers.py @@ -0,0 +1,39 @@ +from django.contrib.contenttypes.models import ContentType +from django.utils.translation import ugettext_lazy as _ + +from orchestra.utils import plugins + + +class ServiceHandler(plugins.Plugin): + model = None + + __metaclass__ = plugins.PluginMount + + def __init__(self, service): + self.service = service + + @classmethod + def get_plugin_choices(cls): + choices = super(ServiceHandler, cls).get_plugin_choices() + return [('', _("Default"))] + choices + + def __getattr__(self, attr): + return getattr(self.service, attr) + + def matches(self, instance): + safe_locals = { + instance._meta.model_name: instance + } + return eval(self.match, safe_locals) + + def get_metric(self, instance): + safe_locals = { + instance._meta.model_name: instance + } + return eval(self.metric, safe_locals) + + def get_content_type(self): + if not self.model: + return self.content_type + app_label, model = self.model.split('.') + return ContentType.objects.get_by_natural_key(app_label, model.lower()) diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index 421d2c35..2b8a014b 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -5,13 +5,22 @@ from django.dispatch import receiver from django.contrib.admin.models import LogEntry from django.contrib.contenttypes import generic from django.contrib.contenttypes.models import ContentType +from django.core.validators import ValidationError from django.utils import timezone +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ +from orchestra.core import caches +from orchestra.utils.apps import autodiscover + from . import settings +from .handlers import ServiceHandler from .helpers import search_for_related +autodiscover('handlers') + + class Service(models.Model): NEVER = 'NEVER' MONTHLY = 'MONTHLY' @@ -34,10 +43,12 @@ class Service(models.Model): MATCH_PRICE = 'MATCH_PRICE' description = models.CharField(_("description"), max_length=256, unique=True) - model = models.ForeignKey(ContentType, verbose_name=_("model")) - match = models.CharField(_("match"), max_length=256) + content_type = models.ForeignKey(ContentType, verbose_name=_("content type")) + match = models.CharField(_("match"), max_length=256, blank=True) + handler = models.CharField(_("handler"), max_length=256, blank=True, + help_text=_("Handler used to process this Service."), + choices=ServiceHandler.get_plugin_choices()) is_active = models.BooleanField(_("is active"), default=True) - # TODO class based Service definition (like ServiceBackend) # Billing billing_period = models.CharField(_("billing period"), max_length=16, help_text=_("Renewal period for recurring invoicing"), @@ -105,21 +116,22 @@ class Service(models.Model): (REFOUND, _("Refound")), ), default=DISCOUNT) - on_disable = models.CharField(_("on disable"), max_length=16, - help_text=_("Defines the behaviour of this service when disabled"), - choices=( - (NOTHING, _("Nothing")), - (DISCOUNT, _("Discount")), - (REFOUND, _("Refound")), - ), - default=DISCOUNT) - on_register = models.CharField(_("on register"), max_length=16, - help_text=_("Defines the behaviour of this service on registration"), - choices=( - (NOTHING, _("Nothing")), - (DISCOUNT, _("Discount (fixed BP)")), - ), - default=DISCOUNT) + # TODO remove, orders are not disabled (they are cancelled user.is_active) +# on_disable = models.CharField(_("on disable"), max_length=16, +# help_text=_("Defines the behaviour of this service when disabled"), +# choices=( +# (NOTHING, _("Nothing")), +# (DISCOUNT, _("Discount")), +# (REFOUND, _("Refound")), +# ), +# default=DISCOUNT) +# on_register = models.CharField(_("on register"), max_length=16, +# help_text=_("Defines the behaviour of this service on registration"), +# choices=( +# (NOTHING, _("Nothing")), +# (DISCOUNT, _("Discount (fixed BP)")), +# ), +# default=DISCOUNT) payment_style = models.CharField(_("payment style"), max_length=16, help_text=_("Designates whether this service should be paid after " "consumtion (postpay/on demand) or prepaid"), @@ -151,27 +163,38 @@ class Service(models.Model): return self.description @classmethod - def get_services(cls, instance, **kwargs): - # TODO get per-request cache from thread local - cache = kwargs.get('cache', {}) + def get_services(cls, instance): + cache = caches.get_request_cache() ct = ContentType.objects.get_for_model(instance) + services = cache.get(ct) + if services is None: + services = cls.objects.filter(content_type=ct, is_active=True) + cache.set(ct, services) + return services + + @cached_property + def proxy(self): + if self.handler: + return ServiceHandler.get_plugin(self.handler)(self) + return ServiceHandler(self) + + def clean(self): + content_type = self.proxy.get_content_type() + if self.content_type != content_type: + msg =_("Content type must be equal to '%s'." % str(content_type)) + raise ValidationError(msg) + if not self.match: + msg =_("Match should be provided") + raise ValidationError(msg) try: - return cache[ct] - except KeyError: - cache[ct] = cls.objects.filter(model=ct, is_active=True) - return cache[ct] - - def matches(self, instance): - safe_locals = { - instance._meta.model_name: instance - } - return eval(self.match, safe_locals) - - def get_metric(self, instance): - safe_locals = { - instance._meta.model_name: instance - } - return eval(self.metric, safe_locals) + obj = content_type.model_class().objects.all()[0] + except IndexError: + pass + else: + try: + self.proxy.matches(obj) + except Exception as e: + raise ValidationError(_(str(e))) class OrderQuerySet(models.QuerySet): @@ -242,10 +265,11 @@ class Order(models.Model): class MetricStorage(models.Model): order = models.ForeignKey(Order, verbose_name=_("order")) value = models.BigIntegerField(_("value")) - date = models.DateTimeField(_("date"), auto_now_add=True) + created_on = models.DateTimeField(_("created on"), auto_now_add=True) + updated_on = models.DateTimeField(_("updated on"), auto_now=True) class Meta: - get_latest_by = 'date' + get_latest_by = 'created_on' def __unicode__(self): return unicode(self.order) @@ -259,24 +283,29 @@ class MetricStorage(models.Model): else: if metric.value != value: cls.objects.create(order=order, value=value) + else: + metric.save() @receiver(pre_delete, dispatch_uid="orders.cancel_orders") def cancel_orders(sender, **kwargs): - if not sender in [MetricStorage, LogEntry, Order, Service]: - instance = kwargs['instance'] - for order in Order.objects.by_object(instance).active(): - order.cancel() + if (not sender in [MetricStorage, LogEntry, Order, Service] and + not Service in sender.__mro__): + instance = kwargs['instance'] + for order in Order.objects.by_object(instance).active(): + order.cancel() @receiver(post_save, dispatch_uid="orders.update_orders") @receiver(post_delete, dispatch_uid="orders.update_orders") def update_orders(sender, **kwargs): - if not sender in [MetricStorage, LogEntry, Order, Service]: - instance = kwargs['instance'] - if instance.pk: - # post_save - Order.update_orders(instance) - related = search_for_related(instance) - if related: - Order.update_orders(related) + if (not sender in [MetricStorage, LogEntry, Order, Service] and + not Service in sender.__mro__): + instance = kwargs['instance'] + print kwargs + if instance.pk: + # post_save + Order.update_orders(instance) + related = search_for_related(instance) + if related: + Order.update_orders(related) diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 1f93d2bf..3f5fba38 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -59,7 +59,7 @@ class ResourceAdmin(ExtendedModelAdmin): def formfield_for_dbfield(self, db_field, **kwargs): """ filter service content_types """ if db_field.name == 'content_type': - models = [ model._meta.model_name for model in services.get().keys() ] + models = [ model._meta.model_name for model in services.get() ] kwargs['queryset'] = db_field.rel.to.objects.filter(model__in=models) return super(ResourceAdmin, self).formfield_for_dbfield(db_field, **kwargs) diff --git a/orchestra/apps/resources/backends.py b/orchestra/apps/resources/backends.py index 9d52210e..04644c5b 100644 --- a/orchestra/apps/resources/backends.py +++ b/orchestra/apps/resources/backends.py @@ -31,7 +31,7 @@ class ServiceMonitor(ServiceBackend): def content_type(self): app_label, model = self.model.split('.') model = model.lower() - return ContentType.objects.get(app_label=app_label, model=model) + return ContentType.objects.get_by_natural_key(app_label, model) def get_last_data(self, object_id): from .models import MonitorData @@ -56,7 +56,7 @@ class ServiceMonitor(ServiceBackend): from .models import MonitorData name = self.get_name() app_label, model_name = self.model.split('.') - ct = ContentType.objects.get(app_label=app_label, model=model_name.lower()) + ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower()) for line in log.stdout.splitlines(): line = line.strip() object_id, value = self.process(line) diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index a1d80e25..1940df8d 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -58,7 +58,7 @@ class Resource(models.Model): help_text=_("Crontab for periodic execution. " "Leave it empty to disable periodic monitoring")) monitors = MultiSelectField(_("monitors"), max_length=256, blank=True, - choices=ServiceMonitor.get_choices(), + choices=ServiceMonitor.get_plugin_choices(), help_text=_("Monitor backends used for monitoring this resource.")) is_active = models.BooleanField(_("is active"), default=True) @@ -144,7 +144,7 @@ class ResourceData(models.Model): class MonitorData(models.Model): """ Stores monitored data """ monitor = models.CharField(_("monitor"), max_length=256, - choices=ServiceMonitor.get_choices()) + choices=ServiceMonitor.get_plugin_choices()) content_type = models.ForeignKey(ContentType) object_id = models.PositiveIntegerField() date = models.DateTimeField(_("date"), auto_now_add=True) diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index c5789e17..941f5680 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -43,6 +43,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.transaction.TransactionMiddleware', + 'orchestra.core.cache.RequestCacheMiddleware', 'orchestra.apps.orchestration.middlewares.OperationsMiddleware', # Uncomment the next line for simple clickjacking protection: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', diff --git a/orchestra/core/caches.py b/orchestra/core/caches.py new file mode 100644 index 00000000..478d7dd9 --- /dev/null +++ b/orchestra/core/caches.py @@ -0,0 +1,39 @@ +from threading import currentThread + +from django.core.cache.backends.locmem import LocMemCache + + +_request_cache = {} + + +class RequestCache(LocMemCache): + """ LocMemCache is a threadsafe local memory cache """ + def __init__(self): + name = 'locmemcache@%i' % hash(currentThread()) + super(RequestCache, self).__init__(name, {}) + + +def get_request_cache(): + """ + Returns per-request cache when running RequestCacheMiddleware otherwise a + new LocMemCache instance (when running periodic tasks or shell) + """ + try: + return _request_cache[currentThread()] + except KeyError: + cache = RequestCache() + _request_cache[currentThread()] = cache + return cache + + +class RequestCacheMiddleware(object): + def process_request(self, request): + cache = _request_cache.get(currentThread(), RequestCache()) + _request_cache[currentThread()] = cache + cache.clear() + + def process_response(self, request, response): + # TODO not sure if this actually saves memory, remove otherwise + if currentThread() in _request_cache: + _request_cache[currentThread()].clear() + return response diff --git a/orchestra/utils/functional.py b/orchestra/utils/functional.py index 2dcb5fee..01800b3d 100644 --- a/orchestra/utils/functional.py +++ b/orchestra/utils/functional.py @@ -2,8 +2,15 @@ def cached(func): """ caches func return value """ def cached_func(self, *args, **kwargs): attr = '_cached_' + func.__name__ - if not hasattr(self, attr): - setattr(self, attr, func(self, *args, **kwargs)) - return getattr(self, attr) + key = (args, tuple(kwargs.items())) + try: + return getattr(self, attr)[key] + except KeyError: + value = func(self, *args, **kwargs) + getattr(self, attr)[key] = value + except AttributeError: + value = func(self, *args, **kwargs) + setattr(self, attr, {key: value}) + return value return cached_func diff --git a/orchestra/utils/plugins.py b/orchestra/utils/plugins.py index 6b53a4ea..330fde57 100644 --- a/orchestra/utils/plugins.py +++ b/orchestra/utils/plugins.py @@ -1,3 +1,40 @@ +from .functional import cached + + +class Plugin(object): + verbose_name = None + + @classmethod + def get_plugin_name(cls): + return cls.__name__ + + @classmethod + def get_plugins(cls): + return cls.plugins + + @classmethod + @cached + def get_plugin(cls, name): + for plugin in cls.get_plugins(): + if plugin.get_plugin_name() == name: + return plugin + raise KeyError('This plugin is not registered') + + @classmethod + def get_plugin_choices(cls): + plugins = cls.get_plugins() + choices = [] + for p in plugins: + # don't evaluate p.verbose_name ugettext_lazy + verbose = getattr(p.verbose_name, '_proxy____args', [p.verbose_name]) + if verbose[0]: + verbose = p.verbose_name + else: + verbose = p.get_plugin_name() + choices.append((p.get_plugin_name(), verbose)) + return sorted(choices, key=lambda e: e[0]) + + class PluginMount(type): def __init__(cls, name, bases, attrs): if not hasattr(cls, 'plugins'):