From 9c5af583dc2db80b2b093d5538ec97c105c4791c Mon Sep 17 00:00:00 2001 From: Marc Date: Thu, 17 Jul 2014 16:09:24 +0000 Subject: [PATCH] Improvements on orders app --- orchestra/apps/domains/admin.py | 14 +- orchestra/apps/domains/models.py | 9 +- orchestra/apps/orchestration/manager.py | 4 +- orchestra/apps/orchestration/middlewares.py | 5 +- orchestra/apps/orchestration/models.py | 45 ++--- orchestra/apps/orders/middlewares.py | 79 ++++++++ orchestra/apps/orders/models.py | 60 +++++- orchestra/apps/prices/admin.py | 7 +- orchestra/apps/prices/models.py | 2 +- orchestra/apps/resources/admin.py | 6 +- orchestra/conf/base_settings.py | 4 +- orchestra/core/__init__.py | 3 + .../static/orchestra/icons/Dialog-accept.png | Bin 0 -> 2797 bytes .../static/orchestra/icons/Dialog-accept.svg | 183 ++++++++++++++++++ 14 files changed, 375 insertions(+), 46 deletions(-) create mode 100644 orchestra/apps/orders/middlewares.py create mode 100644 orchestra/static/orchestra/icons/Dialog-accept.png create mode 100644 orchestra/static/orchestra/icons/Dialog-accept.svg diff --git a/orchestra/apps/domains/admin.py b/orchestra/apps/domains/admin.py index 9a460eb3..e49af15b 100644 --- a/orchestra/apps/domains/admin.py +++ b/orchestra/apps/domains/admin.py @@ -50,7 +50,9 @@ class DomainInline(admin.TabularInline): class DomainAdmin(ChangeListDefaultFilter, AccountAdminMixin, ExtendedModelAdmin): fields = ('name', 'account') - list_display = ('structured_name', 'is_top', 'websites', 'account_link') + list_display = ( + 'structured_name', 'display_is_top', 'websites', 'account_link' + ) inlines = [RecordInline, DomainInline] list_filter = [TopDomainListFilter] change_readonly_fields = ('name',) @@ -59,17 +61,17 @@ class DomainAdmin(ChangeListDefaultFilter, AccountAdminMixin, ExtendedModelAdmin form = DomainAdminForm def structured_name(self, domain): - if not self.is_top(domain): + if not domain.is_top: return ' '*4 + domain.name return domain.name structured_name.short_description = _("name") structured_name.allow_tags = True structured_name.admin_order_field = 'structured_name' - def is_top(self, domain): - return not bool(domain.top) - is_top.boolean = True - is_top.admin_order_field = 'top' + def display_is_top(self, domain): + return domain.is_top + display_is_top.boolean = True + display_is_top.admin_order_field = 'top' def websites(self, domain): if apps.isinstalled('orchestra.apps.websites'): diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index 21aef911..1f7cf33a 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -1,11 +1,11 @@ from django.core.exceptions import ValidationError from django.db import models +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from orchestra.core import services from orchestra.core.validators import (validate_ipv4_address, validate_ipv6_address, validate_hostname, validate_ascii) -from orchestra.utils.functional import cached from . import settings, validators, utils @@ -22,11 +22,14 @@ class Domain(models.Model): def __unicode__(self): return self.name - @property - @cached + @cached_property def origin(self): return self.top or self + @cached_property + def is_top(self): + return not bool(self.top) + def get_records(self): """ proxy method, needed for input validation """ return self.records.all() diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index fdd22e04..ce9360f8 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -35,8 +35,10 @@ def execute(operations): router = import_class(settings.ORCHESTRATION_ROUTER) # Generate scripts per server+backend scripts = {} + cache = {} for operation in operations: - servers = router.get_servers(operation) + servers = router.get_servers(operation, cache=cache) + print cache for server in servers: key = (server, operation.backend) if key not in scripts: diff --git a/orchestra/apps/orchestration/middlewares.py b/orchestra/apps/orchestration/middlewares.py index cd86cb1a..744bb162 100644 --- a/orchestra/apps/orchestration/middlewares.py +++ b/orchestra/apps/orchestration/middlewares.py @@ -4,6 +4,7 @@ from threading import local from django.db.models.signals import pre_delete, post_save from django.dispatch import receiver from django.http.response import HttpResponseServerError + from orchestra.utils.python import OrderedSet from .backends import ServiceBackend @@ -12,12 +13,12 @@ from .models import BackendLog from .models import BackendOperation as Operation -@receiver(post_save) +@receiver(post_save, dispatch_uid='orchestration.post_save_collector') def post_save_collector(sender, *args, **kwargs): if sender != BackendLog: OperationsMiddleware.collect(Operation.SAVE, **kwargs) -@receiver(pre_delete) +@receiver(pre_delete, dispatch_uid='orchestration.pre_delete_collector') def pre_delete_collector(sender, *args, **kwargs): if sender != BackendLog: OperationsMiddleware.collect(Operation.DELETE, **kwargs) diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index 5f3fad8c..e852c3df 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -6,7 +6,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.models.fields import NullableCharField from orchestra.utils.apps import autodiscover -from orchestra.utils.functional import cached from . import settings, manager from .backends import ServiceBackend @@ -56,8 +55,8 @@ class BackendLog(models.Model): server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs') script = models.TextField(_("script")) - stdout = models.TextField() - stderr = models.TextField() + stdout = models.TextField(_("stdout")) + stderr = models.TextField(_("stdin")) traceback = models.TextField(_("traceback")) exit_code = models.IntegerField(_("exit code"), null=True) task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True, @@ -149,35 +148,31 @@ class Route(models.Model): # raise ValidationError(msg % (self.backend, self.method) @classmethod - @cached - def get_routing_table(cls): - table = {} - for route in cls.objects.filter(is_active=True): - for action in route.backend_class().get_actions(): - key = (route.backend, action) - try: - table[key].append(route) - except KeyError: - table[key] = [route] - return table - - @classmethod - def get_servers(cls, operation): - table = cls.get_routing_table() + def get_servers(cls, operation, **kwargs): + cache = kwargs.get('cache', {}) servers = [] - key = (operation.backend.get_name(), operation.action) + backend = operation.backend + key = (backend.get_name(), operation.action) try: - routes = table[key] + routes = cache[key] except KeyError: - return servers - safe_locals = { - 'instance': operation.instance - } + cache[key] = [] + for route in cls.objects.filter(is_active=True, backend=backend.get_name()): + for action in backend.get_actions(): + _key = (route.backend, action) + cache[_key] = [route] + routes = cache[key] for route in routes: - if eval(route.match, safe_locals): + if route.matches(operation.instance): servers.append(route.host) return servers + def matches(self, instance): + safe_locals = { + 'instance': instance + } + return eval(self.match, safe_locals) + def backend_class(self): return ServiceBackend.get_backend(self.backend) diff --git a/orchestra/apps/orders/middlewares.py b/orchestra/apps/orders/middlewares.py new file mode 100644 index 00000000..7807b81a --- /dev/null +++ b/orchestra/apps/orders/middlewares.py @@ -0,0 +1,79 @@ +from threading import local + +from django.db.models.signals import pre_delete, pre_save +from django.dispatch import receiver +from django.http.response import HttpResponseServerError + +from orchestra.core import services +from orchestra.utils.python import OrderedSet + +from .models import Order + + +@receiver(pre_save, dispatch_uid='orders.ppre_save_collector') +def pre_save_collector(sender, *args, **kwargs): + if sender in services: + OrderMiddleware.collect(Order.SAVE, **kwargs) + +@receiver(pre_delete, dispatch_uid='orders.pre_delete_collector') +def pre_delete_collector(sender, *args, **kwargs): + if sender in services: + OrderMiddleware.collect(Order.DELETE, **kwargs) + + +class OrderCandidate(object): + def __unicode__(self): + return "{}.{}()".format(str(self.instance), self.action) + + def __init__(self, instance, action): + self.instance = instance + self.action = action + + def __hash__(self): + """ set() """ + opts = self.instance._meta + model = opts.app_label + opts.model_name + return hash(model + str(self.instance.pk) + self.action) + + def __eq__(self, candidate): + """ set() """ + return hash(self) == hash(candidate) + + +class OrderMiddleware(object): + """ + Stores all the operations derived from save and delete signals and executes them + at the end of the request/response cycle + """ + # Thread local is used because request object is not available on model signals + thread_locals = local() + + @classmethod + def get_order_candidates(cls): + # Check if an error poped up before OrdersMiddleware.process_request() + if hasattr(cls.thread_locals, 'request'): + request = cls.thread_locals.request + if not hasattr(request, 'order_candidates'): + request.order_candidates = OrderedSet() + return request.order_candidates + return set() + + @classmethod + def collect(cls, action, **kwargs): + """ Collects all pending operations derived from model signals """ + request = getattr(cls.thread_locals, 'request', None) + if request is None: + return + order_candidates = cls.get_order_candidates() + instance = kwargs['instance'] + order_candidates.add(OrderCandidate(instance, action)) + + def process_request(self, request): + """ Store request on a thread local variable """ + type(self).thread_locals.request = request + + def process_response(self, request, response): + if not isinstance(response, HttpResponseServerError): + candidates = type(self).get_order_candidates() + Order.process_candidates(candidates) + return response diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index bdd7d9e3..5ab2e362 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -143,8 +143,27 @@ class Service(models.Model): def __unicode__(self): return self.description + @classmethod + def get_services(cls, instance, **kwargs): + cache = kwargs.get('cache', {}) + ct = ContentType.objects.get_for_model(type(instance)) + try: + return cache[ct] + except KeyError: + cache[ct] = cls.objects.filter(model=ct, is_active=True) + return cache[ct] + + def matches(self, instance): + safe_locals = { + 'instance': instance + } + return eval(self.match, safe_locals) + class Order(models.Model): + SAVE = 'SAVE' + DELETE = 'DELETE' + account = models.ForeignKey('accounts.Account', verbose_name=_("account"), related_name='orders') content_type = models.ForeignKey(ContentType) @@ -161,7 +180,46 @@ class Order(models.Model): content_object = generic.GenericForeignKey() def __unicode__(self): - return self.service + return str(self.service) + + def update(self): + instance = self.content_object + if self.service.metric: + metric = self.service.get_metric(instance) + self.store_metric(instance, metric) + description = "{}: {}".format(self.service.description, str(instance)) + if self.description != description: + self.description = description + self.save() + + @classmethod + def process_candidates(cls, candidates): + cache = {} + for candidate in candidates: + instance = candidate.instance + if candidate.action == cls.DELETE: + cls.objects.filter_for_object(instance).cancel() + else: + for service in Service.get_services(instance, cache=cache): + print cache + if not instance.pk: + if service.matches(instance): + order = cls.objects.create(content_object=instance, + account_id=instance.account_id, service=service) + order.update() + else: + ct = ContentType.objects.get_for_model(instance) + orders = cls.objects.filter(content_type=ct, service=service, + object_id=instance.pk) + if service.matches(instance): + if not orders: + order = cls.objects.create(content_object=instance, + service=service, account_id=instance.account_id) + else: + order = orders.get() + order.update() + elif orders: + orders.get().cancel() class MetricStorage(models.Model): diff --git a/orchestra/apps/prices/admin.py b/orchestra/apps/prices/admin.py index 8a6749db..7df5bbbb 100644 --- a/orchestra/apps/prices/admin.py +++ b/orchestra/apps/prices/admin.py @@ -1,13 +1,16 @@ from django.contrib import admin from orchestra.admin.utils import insertattr +from orchestra.apps.accounts.admin import AccountAdminMixin from orchestra.apps.orders.models import Service from .models import Pack, Rate -class PackAdmin(admin.ModelAdmin): - pass +class PackAdmin(AccountAdminMixin, admin.ModelAdmin): + list_display = ('name', 'account_link') + list_filter = ('name',) + admin.site.register(Pack, PackAdmin) diff --git a/orchestra/apps/prices/models.py b/orchestra/apps/prices/models.py index c43021a4..2cad63ae 100644 --- a/orchestra/apps/prices/models.py +++ b/orchestra/apps/prices/models.py @@ -15,7 +15,7 @@ class Pack(models.Model): default=settings.PRICES_DEFAULT_PACK) def __unicode__(self): - return self.pack + return self.name class Rate(models.Model): diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index bb0956c9..542e503f 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -37,10 +37,10 @@ class ResourceAdmin(ExtendedModelAdmin): def add_view(self, request, **kwargs): """ Warning user if the node is not fully configured """ - if request.method == 'GET': + if request.method == 'POST': messages.warning(request, _( - "Restarting orchestra and celery is required to fully apply changes. " - "Remember that allocated values will be applied when objects are saved" + "Restarting orchestra and celerybeat is required to fully apply changes. " + "Remember that new allocated values will be applied when objects are saved." )) return super(ResourceAdmin, self).add_view(request, **kwargs) diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index 0ca433f7..d144bd53 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -43,7 +43,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.transaction.TransactionMiddleware', -# 'orchestra.apps.contacts.middlewares.ContractMiddleware', + 'orchestra.apps.orders.middlewares.OrderMiddleware', 'orchestra.apps.orchestration.middlewares.OperationsMiddleware', # Uncomment the next line for simple clickjacking protection: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', @@ -178,7 +178,7 @@ FLUENT_DASHBOARD_APP_ICONS = { 'contacts/contact': 'contact.png', 'orders/order': 'basket.png', 'orders/service': 'price.png', - 'prices/pack': 'pack.png', + 'prices/pack': 'Dialog-accept.png', # Administration 'users/user': 'Mr-potato.png', 'djcelery/taskstate': 'taskstate.png', diff --git a/orchestra/core/__init__.py b/orchestra/core/__init__.py index b926baee..11fd8189 100644 --- a/orchestra/core/__init__.py +++ b/orchestra/core/__init__.py @@ -1,6 +1,9 @@ class Service(object): _registry = {} + def __contains__(self, key): + return key in self._registry + def register(self, model, **kwargs): if model in self._registry: raise KeyError("%s already registered" % str(model)) diff --git a/orchestra/static/orchestra/icons/Dialog-accept.png b/orchestra/static/orchestra/icons/Dialog-accept.png new file mode 100644 index 0000000000000000000000000000000000000000..bb55f01b2874364ff1eb8d5cec4e9277db786b47 GIT binary patch literal 2797 zcmVW+aF##yI$LCV^f6G zGdkMYy^r&K=bkzD&b>=Q2z;G|eboRKDqnj7@?v$gb-#9*MB@8Iv{?W(f}>oK)F1#j zo~;wZAOZbCh-03hLoe@0on8#Pw2)`uyUzq_6qP?DKtC)Emj@e{GzBUmwTfRYfKOH- z7$Cp_4%luAx|P7dXpi%G-)F{1Vn`>5zdAPi?JK*D{{Ja~AN+wLuHxIP%Kp2rT6s;d zzGAf`$r5yL6owavD=cu~Wwl`>K_(vpER290hUd8G8tCv^&$Q^aXYP|7tNGwA5&zN& zZ2Ob%N0O{O*|g$Hf78S?jsx;{2N*ZJw;oz@Fq z3IPJqwr71$6o<=pUbpV%5R(~V?m&*G=PQgP6e<<)Y#Z;jzMU8vi~i~GkFAFV<_2s| z1a3V*!GPy`d0Az{#;aFt3?%pnY;G(F_*7AuA7LuQN2fn9x&}JmHzaH8(Yu76uU%dO z&CSGGmiXSjqH*0f8tPW7W9~3K;Vc^POp!=Np;Ur1eI0h|nU7Db>#=UwyI1hp+T=~6 zEp@&pm)A6IsIP5MW6nhgEbh!5kctqIs!<@7A?ZaiV0WOtW`&y2lN(w@*OLJD%+`L^ z5!klhe|KqN#qTykGxawufvx+rhJYNpy{@8Gc7(lX;AQgRmdMYcQXuPhTbTG~ z>^CsI?0L+Az>>-uIUogZ-?~q0n6A%h2}nebU9oCyP#1B~BChwY3*V1`R0zuSJWRCc zhY)o>o3>3-Byh#5)j^5qvFXZl&^0#`Be2`bipm&Y#Ei2h@GVrSYax+39ytVfaB#tD zM~C?;h$utZq&nazD=c9Ic3X2Z<>1Zv;be`xsj{RJHc}u+S!LNqW>S`;<;qH26}$r~ z^Jn#E@)(XKo(6$U(TR&eWkIB%qq4XRCu`(Qc?mEk-C0{%9kzv)17yN=xdx{R+!%QX z8U>Jqy~H1pmBslC&A{JAS99@%LE~)JTC!-sR2$NSccKLczUV?SuxfZ+yRCB00cPa@H__> zoOT?JJ_1G(f^sQBav2ozP1D0An=)ze;6hp!MO+5b-_7@A>#Pk(7kX(3KrKfVt1q4_&3g{I$(2S$}>k9&mQ0i3x37tZhMXj zGcjk40C$5v@{>UNA(}~uW*?D8?FNKd6+Dk)VyR*a9S4VY!{DPi^$l+nJ?1Dvayd%X zdMHfGDtEX6AE_W>g1f=2;mSg}zF&`;wk0!6pAIv#kwQTf%gew>$;uo7R*bdU_rSP1 z3EmMVK1jZR^Y$mR5>L2;IA?r_sP#GcWOgE$JPw6an64$S@7JT5ZL&sS?_R;TKO1nF z0EFORBq1{vf${Pi{uJgS0vw}G zA4Z*i_@yv};7IWV6lp1JH|0)!Kgfv_0w3a@7>~)c2Ux~+YvC_Yr8Gfd8dMfQkQJj! zU4vo!97@;<6idsp!q4IcUL3j~0`q zJYzO5Ba@SwCv0^2j$^r&y(H0C z2rBurLdWJh4vqW_ScC;(m=&Ub_!PLXk4`Pi8G$3agyC@eqh!)RffNA%xWap7i zQ}ZYsk;*ZWIq_LS#~bk{5O=@uhTEhRun>~E35Po$9oZ#vwnC;&_O6RZ+dEH@N7VVv#W3G!wrz@Y9z>%Hio6*G3C;C`QM5L0e zvN3M}ac^eN+QPu$k%t$8c~T-$5%e)jM3W<*9NB5UIbEF@r(4$+_qCkrNLDG!AW`;v z!TX8*uy``3jU|$JE%qpqUNom{z9Nx?D!CRdr$0-&wzzMmdNUGuSf5B&`z1J!{o87JtA8eIF~wscH`9rg9m`6NnWRdthLQ^4oz{~Vgl=B`(jGB9Tm3n0 z?rz<$T_!X3<~6G>uP7+?N!?Z__`H_}vkL>mQneoXh~s^Dy8WWf`Au^<#hLT=`u3-( zLgYZDDH+zy`8+=%^E%-8_5B!kL?E&wM(tqe_N zd`_q##O#M~^FF0akH`dpNziGP(<)(nmQU}3LZp?a)lKUJ5j(9GfRqq(Kf_wkL3aiz zmIghc6W=L)p9$VeD+l1^^Kf3<|NpnjS1kVpmw~n)uP@Ds00000NkvXXu0mjfN@Nk| literal 0 HcmV?d00001 diff --git a/orchestra/static/orchestra/icons/Dialog-accept.svg b/orchestra/static/orchestra/icons/Dialog-accept.svg new file mode 100644 index 00000000..24262737 --- /dev/null +++ b/orchestra/static/orchestra/icons/Dialog-accept.svg @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + Rodney Dawes + + + + + Jakub Steiner, Garrett LeSage + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +