diff --git a/orchestra/apps/domains/admin.py b/orchestra/apps/domains/admin.py index 9a460eb3..e49af15b 100644 --- a/orchestra/apps/domains/admin.py +++ b/orchestra/apps/domains/admin.py @@ -50,7 +50,9 @@ class DomainInline(admin.TabularInline): class DomainAdmin(ChangeListDefaultFilter, AccountAdminMixin, ExtendedModelAdmin): fields = ('name', 'account') - list_display = ('structured_name', 'is_top', 'websites', 'account_link') + list_display = ( + 'structured_name', 'display_is_top', 'websites', 'account_link' + ) inlines = [RecordInline, DomainInline] list_filter = [TopDomainListFilter] change_readonly_fields = ('name',) @@ -59,17 +61,17 @@ class DomainAdmin(ChangeListDefaultFilter, AccountAdminMixin, ExtendedModelAdmin form = DomainAdminForm def structured_name(self, domain): - if not self.is_top(domain): + if not domain.is_top: return ' '*4 + domain.name return domain.name structured_name.short_description = _("name") structured_name.allow_tags = True structured_name.admin_order_field = 'structured_name' - def is_top(self, domain): - return not bool(domain.top) - is_top.boolean = True - is_top.admin_order_field = 'top' + def display_is_top(self, domain): + return domain.is_top + display_is_top.boolean = True + display_is_top.admin_order_field = 'top' def websites(self, domain): if apps.isinstalled('orchestra.apps.websites'): diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index 21aef911..1f7cf33a 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -1,11 +1,11 @@ from django.core.exceptions import ValidationError from django.db import models +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from orchestra.core import services from orchestra.core.validators import (validate_ipv4_address, validate_ipv6_address, validate_hostname, validate_ascii) -from orchestra.utils.functional import cached from . import settings, validators, utils @@ -22,11 +22,14 @@ class Domain(models.Model): def __unicode__(self): return self.name - @property - @cached + @cached_property def origin(self): return self.top or self + @cached_property + def is_top(self): + return not bool(self.top) + def get_records(self): """ proxy method, needed for input validation """ return self.records.all() diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index fdd22e04..ce9360f8 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -35,8 +35,10 @@ def execute(operations): router = import_class(settings.ORCHESTRATION_ROUTER) # Generate scripts per server+backend scripts = {} + cache = {} for operation in operations: - servers = router.get_servers(operation) + servers = router.get_servers(operation, cache=cache) + print cache for server in servers: key = (server, operation.backend) if key not in scripts: diff --git a/orchestra/apps/orchestration/middlewares.py b/orchestra/apps/orchestration/middlewares.py index cd86cb1a..744bb162 100644 --- a/orchestra/apps/orchestration/middlewares.py +++ b/orchestra/apps/orchestration/middlewares.py @@ -4,6 +4,7 @@ from threading import local from django.db.models.signals import pre_delete, post_save from django.dispatch import receiver from django.http.response import HttpResponseServerError + from orchestra.utils.python import OrderedSet from .backends import ServiceBackend @@ -12,12 +13,12 @@ from .models import BackendLog from .models import BackendOperation as Operation -@receiver(post_save) +@receiver(post_save, dispatch_uid='orchestration.post_save_collector') def post_save_collector(sender, *args, **kwargs): if sender != BackendLog: OperationsMiddleware.collect(Operation.SAVE, **kwargs) -@receiver(pre_delete) +@receiver(pre_delete, dispatch_uid='orchestration.pre_delete_collector') def pre_delete_collector(sender, *args, **kwargs): if sender != BackendLog: OperationsMiddleware.collect(Operation.DELETE, **kwargs) diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index 5f3fad8c..e852c3df 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -6,7 +6,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.models.fields import NullableCharField from orchestra.utils.apps import autodiscover -from orchestra.utils.functional import cached from . import settings, manager from .backends import ServiceBackend @@ -56,8 +55,8 @@ class BackendLog(models.Model): server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs') script = models.TextField(_("script")) - stdout = models.TextField() - stderr = models.TextField() + stdout = models.TextField(_("stdout")) + stderr = models.TextField(_("stdin")) traceback = models.TextField(_("traceback")) exit_code = models.IntegerField(_("exit code"), null=True) task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True, @@ -149,35 +148,31 @@ class Route(models.Model): # raise ValidationError(msg % (self.backend, self.method) @classmethod - @cached - def get_routing_table(cls): - table = {} - for route in cls.objects.filter(is_active=True): - for action in route.backend_class().get_actions(): - key = (route.backend, action) - try: - table[key].append(route) - except KeyError: - table[key] = [route] - return table - - @classmethod - def get_servers(cls, operation): - table = cls.get_routing_table() + def get_servers(cls, operation, **kwargs): + cache = kwargs.get('cache', {}) servers = [] - key = (operation.backend.get_name(), operation.action) + backend = operation.backend + key = (backend.get_name(), operation.action) try: - routes = table[key] + routes = cache[key] except KeyError: - return servers - safe_locals = { - 'instance': operation.instance - } + cache[key] = [] + for route in cls.objects.filter(is_active=True, backend=backend.get_name()): + for action in backend.get_actions(): + _key = (route.backend, action) + cache[_key] = [route] + routes = cache[key] for route in routes: - if eval(route.match, safe_locals): + if route.matches(operation.instance): servers.append(route.host) return servers + def matches(self, instance): + safe_locals = { + 'instance': instance + } + return eval(self.match, safe_locals) + def backend_class(self): return ServiceBackend.get_backend(self.backend) diff --git a/orchestra/apps/orders/middlewares.py b/orchestra/apps/orders/middlewares.py new file mode 100644 index 00000000..7807b81a --- /dev/null +++ b/orchestra/apps/orders/middlewares.py @@ -0,0 +1,79 @@ +from threading import local + +from django.db.models.signals import pre_delete, pre_save +from django.dispatch import receiver +from django.http.response import HttpResponseServerError + +from orchestra.core import services +from orchestra.utils.python import OrderedSet + +from .models import Order + + +@receiver(pre_save, dispatch_uid='orders.ppre_save_collector') +def pre_save_collector(sender, *args, **kwargs): + if sender in services: + OrderMiddleware.collect(Order.SAVE, **kwargs) + +@receiver(pre_delete, dispatch_uid='orders.pre_delete_collector') +def pre_delete_collector(sender, *args, **kwargs): + if sender in services: + OrderMiddleware.collect(Order.DELETE, **kwargs) + + +class OrderCandidate(object): + def __unicode__(self): + return "{}.{}()".format(str(self.instance), self.action) + + def __init__(self, instance, action): + self.instance = instance + self.action = action + + def __hash__(self): + """ set() """ + opts = self.instance._meta + model = opts.app_label + opts.model_name + return hash(model + str(self.instance.pk) + self.action) + + def __eq__(self, candidate): + """ set() """ + return hash(self) == hash(candidate) + + +class OrderMiddleware(object): + """ + Stores all the operations derived from save and delete signals and executes them + at the end of the request/response cycle + """ + # Thread local is used because request object is not available on model signals + thread_locals = local() + + @classmethod + def get_order_candidates(cls): + # Check if an error poped up before OrdersMiddleware.process_request() + if hasattr(cls.thread_locals, 'request'): + request = cls.thread_locals.request + if not hasattr(request, 'order_candidates'): + request.order_candidates = OrderedSet() + return request.order_candidates + return set() + + @classmethod + def collect(cls, action, **kwargs): + """ Collects all pending operations derived from model signals """ + request = getattr(cls.thread_locals, 'request', None) + if request is None: + return + order_candidates = cls.get_order_candidates() + instance = kwargs['instance'] + order_candidates.add(OrderCandidate(instance, action)) + + def process_request(self, request): + """ Store request on a thread local variable """ + type(self).thread_locals.request = request + + def process_response(self, request, response): + if not isinstance(response, HttpResponseServerError): + candidates = type(self).get_order_candidates() + Order.process_candidates(candidates) + return response diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index bdd7d9e3..5ab2e362 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -143,8 +143,27 @@ class Service(models.Model): def __unicode__(self): return self.description + @classmethod + def get_services(cls, instance, **kwargs): + cache = kwargs.get('cache', {}) + ct = ContentType.objects.get_for_model(type(instance)) + try: + return cache[ct] + except KeyError: + cache[ct] = cls.objects.filter(model=ct, is_active=True) + return cache[ct] + + def matches(self, instance): + safe_locals = { + 'instance': instance + } + return eval(self.match, safe_locals) + class Order(models.Model): + SAVE = 'SAVE' + DELETE = 'DELETE' + account = models.ForeignKey('accounts.Account', verbose_name=_("account"), related_name='orders') content_type = models.ForeignKey(ContentType) @@ -161,7 +180,46 @@ class Order(models.Model): content_object = generic.GenericForeignKey() def __unicode__(self): - return self.service + return str(self.service) + + def update(self): + instance = self.content_object + if self.service.metric: + metric = self.service.get_metric(instance) + self.store_metric(instance, metric) + description = "{}: {}".format(self.service.description, str(instance)) + if self.description != description: + self.description = description + self.save() + + @classmethod + def process_candidates(cls, candidates): + cache = {} + for candidate in candidates: + instance = candidate.instance + if candidate.action == cls.DELETE: + cls.objects.filter_for_object(instance).cancel() + else: + for service in Service.get_services(instance, cache=cache): + print cache + if not instance.pk: + if service.matches(instance): + order = cls.objects.create(content_object=instance, + account_id=instance.account_id, service=service) + order.update() + else: + ct = ContentType.objects.get_for_model(instance) + orders = cls.objects.filter(content_type=ct, service=service, + object_id=instance.pk) + if service.matches(instance): + if not orders: + order = cls.objects.create(content_object=instance, + service=service, account_id=instance.account_id) + else: + order = orders.get() + order.update() + elif orders: + orders.get().cancel() class MetricStorage(models.Model): diff --git a/orchestra/apps/prices/admin.py b/orchestra/apps/prices/admin.py index 8a6749db..7df5bbbb 100644 --- a/orchestra/apps/prices/admin.py +++ b/orchestra/apps/prices/admin.py @@ -1,13 +1,16 @@ from django.contrib import admin from orchestra.admin.utils import insertattr +from orchestra.apps.accounts.admin import AccountAdminMixin from orchestra.apps.orders.models import Service from .models import Pack, Rate -class PackAdmin(admin.ModelAdmin): - pass +class PackAdmin(AccountAdminMixin, admin.ModelAdmin): + list_display = ('name', 'account_link') + list_filter = ('name',) + admin.site.register(Pack, PackAdmin) diff --git a/orchestra/apps/prices/models.py b/orchestra/apps/prices/models.py index c43021a4..2cad63ae 100644 --- a/orchestra/apps/prices/models.py +++ b/orchestra/apps/prices/models.py @@ -15,7 +15,7 @@ class Pack(models.Model): default=settings.PRICES_DEFAULT_PACK) def __unicode__(self): - return self.pack + return self.name class Rate(models.Model): diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index bb0956c9..542e503f 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -37,10 +37,10 @@ class ResourceAdmin(ExtendedModelAdmin): def add_view(self, request, **kwargs): """ Warning user if the node is not fully configured """ - if request.method == 'GET': + if request.method == 'POST': messages.warning(request, _( - "Restarting orchestra and celery is required to fully apply changes. " - "Remember that allocated values will be applied when objects are saved" + "Restarting orchestra and celerybeat is required to fully apply changes. " + "Remember that new allocated values will be applied when objects are saved." )) return super(ResourceAdmin, self).add_view(request, **kwargs) diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index 0ca433f7..d144bd53 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -43,7 +43,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.transaction.TransactionMiddleware', -# 'orchestra.apps.contacts.middlewares.ContractMiddleware', + 'orchestra.apps.orders.middlewares.OrderMiddleware', 'orchestra.apps.orchestration.middlewares.OperationsMiddleware', # Uncomment the next line for simple clickjacking protection: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', @@ -178,7 +178,7 @@ FLUENT_DASHBOARD_APP_ICONS = { 'contacts/contact': 'contact.png', 'orders/order': 'basket.png', 'orders/service': 'price.png', - 'prices/pack': 'pack.png', + 'prices/pack': 'Dialog-accept.png', # Administration 'users/user': 'Mr-potato.png', 'djcelery/taskstate': 'taskstate.png', diff --git a/orchestra/core/__init__.py b/orchestra/core/__init__.py index b926baee..11fd8189 100644 --- a/orchestra/core/__init__.py +++ b/orchestra/core/__init__.py @@ -1,6 +1,9 @@ class Service(object): _registry = {} + def __contains__(self, key): + return key in self._registry + def register(self, model, **kwargs): if model in self._registry: raise KeyError("%s already registered" % str(model)) diff --git a/orchestra/static/orchestra/icons/Dialog-accept.png b/orchestra/static/orchestra/icons/Dialog-accept.png new file mode 100644 index 00000000..bb55f01b Binary files /dev/null and b/orchestra/static/orchestra/icons/Dialog-accept.png differ diff --git a/orchestra/static/orchestra/icons/Dialog-accept.svg b/orchestra/static/orchestra/icons/Dialog-accept.svg new file mode 100644 index 00000000..24262737 --- /dev/null +++ b/orchestra/static/orchestra/icons/Dialog-accept.svg @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + Rodney Dawes + + + + + Jakub Steiner, Garrett LeSage + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +