From df99f8d7457322b05ac7fba0a85a53d782db4eaa Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Fri, 15 Jul 2016 12:44:38 +0000 Subject: [PATCH] Added advance orchestration functionality --- orchestra/contrib/orchestration/actions.py | 71 ++++++++++- orchestra/contrib/orchestration/admin.py | 4 +- .../management/commands/orchestrate.py | 112 +++++++++++------- orchestra/contrib/orchestration/models.py | 2 +- .../admin/orchestration/orchestrate.html | 17 +++ .../admin/orchestra/generic_confirmation.html | 4 +- orchestra/templatetags/utils.py | 2 +- 7 files changed, 159 insertions(+), 53 deletions(-) create mode 100644 orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html diff --git a/orchestra/contrib/orchestration/actions.py b/orchestra/contrib/orchestration/actions.py index b9e9c9c7..9a36a1a1 100644 --- a/orchestra/contrib/orchestration/actions.py +++ b/orchestra/contrib/orchestration/actions.py @@ -1,3 +1,5 @@ +from collections import defaultdict + from django.contrib import messages from django.contrib.admin import helpers from django.shortcuts import render @@ -6,9 +8,10 @@ from django.utils.translation import ungettext, ugettext_lazy as _ from orchestra.admin.utils import get_object_from_url, change_url from orchestra.contrib.orchestration.helpers import message_user +from orchestra.utils.python import OrderedSet -from . import Operation -from .models import BackendOperation +from . import manager, Operation +from .models import BackendOperation, Route, Server def retry_backend(modeladmin, request, queryset): @@ -26,7 +29,6 @@ def retry_backend(modeladmin, request, queryset): else: logs = Operation.execute(operations) message_user(request, logs) - Operation.execute(operations) return opts = modeladmin.model._meta display_objects = [] @@ -61,3 +63,66 @@ def retry_backend(modeladmin, request, queryset): return render(request, 'admin/orchestration/backends/retry.html', context) retry_backend.short_description = _("Retry") retry_backend.url_name = 'retry' + + +def orchestrate(modeladmin, request, queryset): + operations = set() + action = Operation.SAVE + operations = OrderedSet() + if queryset.model is Route: + for route in queryset: + routes = [route] + backend = route.backend_class + if action not in backend.actions: + continue + for instance in backend.model_class().objects.all(): + if route.matches(instance): + operations.add(Operation(backend, instance, action, routes=routes)) + elif queryset.model is Server: + models = set() + for server in queryset: + routes = server.routes.all() + for route in routes.filter(is_active=True): + model = route.backend_class.model_class() + models.add(model) + querysets = [model.objects.order_by('id') for model in models] + + route_cache = {} + for model in models: + for instance in model.objects.all(): + manager.collect(instance, action, operations=operations, route_cache=route_cache) + routes = [] + result = [] + for operation in operations: + routes = [route for route in operation.routes if route.host in queryset] + operation.routes = routes + if routes: + result.append(operation) + operations = result + if not operations: + messages.warning(request, _("No operations.")) + + if request.POST.get('post') == 'generic_confirmation': + logs = Operation.execute(operations) + message_user(request, logs) + return + + opts = modeladmin.model._meta + display_objects = {} + for operation in operations: + try: + display_objects[operation.backend].append(operation) + except KeyError: + display_objects[operation.backend] = [operation] + context = { + 'title': _("Are you sure to execute the following operations?"), + 'action_name': _('Orchestrate'), + 'action_value': 'orchestrate', + 'display_objects': display_objects, + 'queryset': queryset, + 'opts': opts, + 'app_label': opts.app_label, + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'obj': get_object_from_url(modeladmin, request), + } + return render(request, 'admin/orchestration/orchestrate.html', context) diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index 1a81a6d9..34a2f247 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -8,7 +8,7 @@ from orchestra.admin.utils import admin_link, admin_date, admin_colored, display from orchestra.plugins.admin import display_plugin_field from . import settings, helpers -from .actions import retry_backend +from .actions import retry_backend, orchestrate from .backends import ServiceBackend from .forms import RouteForm from .models import Server, Route, BackendLog, BackendOperation @@ -39,6 +39,7 @@ class RouteAdmin(ExtendedModelAdmin): ordering = ('backend',) add_fields = ('backend', 'host', 'match', 'async', 'is_active') change_form = RouteForm + actions = (orchestrate,) BACKEND_HELP_TEXT = helpers.get_backends_help_text(ServiceBackend.get_backends()) DEFAULT_MATCH = { @@ -173,6 +174,7 @@ class BackendLogAdmin(ChangeViewActionsMixin, admin.ModelAdmin): class ServerAdmin(admin.ModelAdmin): list_display = ('name', 'address', 'os', 'display_ping', 'display_uptime') list_filter = ('os',) + actions = (orchestrate,) def display_ping(self, instance): return self._remote_state[instance.pk][0] diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py index 8fb7def3..36eed548 100644 --- a/orchestra/contrib/orchestration/management/commands/orchestrate.py +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -1,11 +1,11 @@ import time from django.core.management.base import BaseCommand, CommandError -from django.apps import apps +from django.db.models import Q from orchestra.contrib.orchestration import manager, Operation from orchestra.contrib.orchestration.models import Server from orchestra.contrib.orchestration.backends import ServiceBackend -from orchestra.utils.python import import_class, OrderedSet, AttrDict +from orchestra.utils.python import OrderedSet from orchestra.utils.sys import confirm @@ -13,70 +13,90 @@ class Command(BaseCommand): help = 'Runs orchestration backends.' def add_arguments(self, parser): - parser.add_argument('model', + parser.add_argument('model', nargs='?', help='Label of a model to execute the orchestration.') parser.add_argument('query', nargs='*', help='Query arguments for filter().') parser.add_argument('--noinput', action='store_false', dest='interactive', default=True, help='Tells Django to NOT prompt the user for input of any kind.') - parser.add_argument('--action', action='store', dest='action', + parser.add_argument('-a', '--action', action='store', dest='action', default='save', help='Executes action. Defaults to "save".') - parser.add_argument('--servers', action='store', dest='servers', + parser.add_argument('-s', '--servers', action='store', dest='servers', default='', help='Overrides route server resolution with the provided server.') - parser.add_argument('--backends', action='store', dest='backends', + parser.add_argument('-b', '--backends', action='store', dest='backends', default='', help='Overrides backend.') - parser.add_argument('--listbackends', action='store_true', dest='list_backends', default=False, + parser.add_argument('-l', '--listbackends', action='store_true', dest='list_backends', default=False, help='List available baclends.') parser.add_argument('--dry-run', action='store_true', dest='dry', default=False, help='Only prints scrtipt.') + + def collect_operations(self, **options): + model = options.get('model') + backends = options.get('backends') or set() + if backends: + backends = set(backends.split(',')) + servers = options.get('servers') or set() + if servers: + servers = set([Server.objects.get(Q(address=server)|Q(name=server)) for server in servers.split(',')]) + action = options.get('action') + if not model: + models = set() + if servers: + for server in servers: + if backends: + routes = server.routes.filter(backend__in=backends) + else: + routes = server.routes.all() + elif backends: + routes = Route.objects.filter(backend__in=backends) + else: + raise CommandError("Model or --servers or --backends?") + for route in routes.filter(is_active=True): + model = route.backend_class.model_class() + models.add(model) + querysets = [model.objects.order_by('id') for model in models] + else: + kwargs = {} + for comp in options.get('query', []): + comps = iter(comp.split('=')) + for arg in comps: + kwargs[arg] = next(comps).strip().rstrip(',') + model = apps.get_model(*model.split('.')) + queryset = model.objects.filter(**kwargs).order_by('id') + querysets = [queryset] + + operations = OrderedSet() + route_cache = {} + for queryset in querysets: + for instance in queryset: + manager.collect(instance, action, operations=operations, route_cache=route_cache) + if backends: + result = [] + for operation in operations: + if operation.backend in backends: + result.append(operation) + operations = result + if servers: + routes = [] + result = [] + for operation in operations: + routes = [route for route in operation.routes if route.host in servers] + operation.routes = routes + if routes: + result.append(operation) + operations = result + return operations + def handle(self, *args, **options): list_backends = options.get('list_backends') if list_backends: for backend in ServiceBackend.get_backends(): self.stdout.write(str(backend).split("'")[1]) return - model = apps.get_model(*options['model'].split('.')) - action = options.get('action') interactive = options.get('interactive') - servers = options.get('servers') - backends = options.get('backends') - if (servers and not backends) or (not servers and backends): - raise CommandError("--backends and --servers go in tandem.") dry = options.get('dry') - kwargs = {} - for comp in options.get('query', []): - comps = iter(comp.split('=')) - for arg in comps: - kwargs[arg] = next(comps).strip().rstrip(',') - operations = OrderedSet() - route_cache = {} - queryset = model.objects.filter(**kwargs).order_by('id') - if servers: - servers = servers.split(',') - backends = backends.split(',') - routes = [] - # Get and create missing Servers - for server in servers: - try: - server = Server.objects.get(address=server) - except Server.DoesNotExist: - server = Server(name=server, address=server) - server.full_clean() - server.save() - routes.append(AttrDict( - host=server, - async=False, - action_is_async=lambda self: False, - )) - # Generate operations for the given backend - for instance in queryset: - for backend in backends: - backend = import_class(backend) - operations.add(Operation(backend, instance, action, routes=routes)) - else: - for instance in queryset: - manager.collect(instance, action, operations=operations, route_cache=route_cache) + operations = self.collect_operations(**options) scripts, serialize = manager.generate(operations) servers = set() # Print scripts diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py index 887db725..2952b72f 100644 --- a/orchestra/contrib/orchestration/models.py +++ b/orchestra/contrib/orchestration/models.py @@ -199,7 +199,7 @@ class Route(models.Model): """ backend = models.CharField(_("backend"), max_length=256, choices=ServiceBackend.get_choices()) - host = models.ForeignKey(Server, verbose_name=_("host")) + host = models.ForeignKey(Server, verbose_name=_("host"), related_name='routes') match = models.CharField(_("match"), max_length=256, blank=True, default='True', help_text=_("Python expression used for selecting the targe host, " "instance referes to the current object.")) diff --git a/orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html b/orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html new file mode 100644 index 00000000..54803cba --- /dev/null +++ b/orchestra/contrib/orchestration/templates/admin/orchestration/orchestrate.html @@ -0,0 +1,17 @@ +{% extends "admin/orchestra/generic_confirmation.html" %} +{% load utils %} + +{% block display_objects %} + +{% endblock %} diff --git a/orchestra/templates/admin/orchestra/generic_confirmation.html b/orchestra/templates/admin/orchestra/generic_confirmation.html index 5aab199b..c918deaa 100644 --- a/orchestra/templates/admin/orchestra/generic_confirmation.html +++ b/orchestra/templates/admin/orchestra/generic_confirmation.html @@ -29,7 +29,9 @@

{{ content_message | safe }}

-
    {{ display_objects | unordered_list }}
+ {% block display_objects %} +
    {{ display_objects | unordered_list }}
+ {% endblock %}
{% csrf_token %} {% block form %} {% if form %} diff --git a/orchestra/templatetags/utils.py b/orchestra/templatetags/utils.py index 1bd41c6f..f57fb317 100644 --- a/orchestra/templatetags/utils.py +++ b/orchestra/templatetags/utils.py @@ -11,7 +11,7 @@ from django.template.defaultfilters import date from django.utils.safestring import mark_safe from orchestra import get_version -from orchestra.admin.utils import change_url +from orchestra.admin.utils import change_url, admin_link as utils_admin_link from orchestra.utils.apps import isinstalled