From 4b15c742ff26299501205a57d2d827a39a3952fb Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Fri, 21 Nov 2014 13:53:39 +0000 Subject: [PATCH] Refactored orchestration backends to support multiple executables --- TODO.md | 4 -- orchestra/apps/databases/backends.py | 2 +- orchestra/apps/lists/backends.py | 1 + orchestra/apps/mailboxes/backends.py | 1 + orchestra/apps/orchestration/backends.py | 72 ++++++++++++++++++---- orchestra/apps/orchestration/methods.py | 18 ++++-- orchestra/apps/resources/actions.py | 32 +++++++--- orchestra/apps/resources/models.py | 2 +- orchestra/apps/resources/tasks.py | 18 +++--- orchestra/apps/systemusers/backends.py | 3 +- orchestra/apps/webapps/models.py | 2 +- orchestra/apps/websites/backends/apache.py | 1 + orchestra/utils/tests.py | 2 +- 13 files changed, 114 insertions(+), 44 deletions(-) diff --git a/TODO.md b/TODO.md index 15b3e077..d53b4e8f 100644 --- a/TODO.md +++ b/TODO.md @@ -174,16 +174,12 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * admin systemuser home/directory, add default home and empty directory with has_shell on admin - * Resource used_list_display=True, allocated_list_displat=True, allow resources to show up on list_display - * Move plugins back from apps to orchestra main app * BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when) -* Resource.monitor(async=True) admin action - * Validate a model path exists between resource.content_type and backend.model * Periodic task for cleaning old monitoring data diff --git a/orchestra/apps/databases/backends.py b/orchestra/apps/databases/backends.py index b6316ef6..5a40d4f7 100644 --- a/orchestra/apps/databases/backends.py +++ b/orchestra/apps/databases/backends.py @@ -109,7 +109,7 @@ class MysqlDisk(ServiceMonitor): )) def prepare(self): - """ slower """ + super(MysqlDisk, self).prepare() self.append(textwrap.dedent("""\ function monitor () { { du -bs "/var/lib/mysql/$1" || echo 0; } | awk {'print $1'} diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index cdb9fb7a..3629dc9c 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -146,6 +146,7 @@ class MailmanTraffic(ServiceMonitor): verbose_name = _("Mailman traffic") def prepare(self): + super(MailmanTraffic, self).prepare() current_date = timezone.localtime(self.current_date) current_date = current_date.strftime("%b %d %H:%M:%S") self.append(textwrap.dedent("""\ diff --git a/orchestra/apps/mailboxes/backends.py b/orchestra/apps/mailboxes/backends.py index f11d6508..c01893bf 100644 --- a/orchestra/apps/mailboxes/backends.py +++ b/orchestra/apps/mailboxes/backends.py @@ -216,6 +216,7 @@ class MaildirDisk(ServiceMonitor): verbose_name = _("Maildir disk usage") def prepare(self): + super(MaildirDisk, self).prepare() current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") self.append(textwrap.dedent("""\ function monitor () { diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index 2a02fa49..8f055832 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -2,6 +2,7 @@ from functools import partial from django.db.models.loading import get_model from django.utils import timezone +from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from orchestra.apps import plugins @@ -19,14 +20,13 @@ class ServiceBackend(plugins.Plugin): """ model = None related_models = () # ((model, accessor__attribute),) - script_method = methods.BashSSH + script_method = methods.SSH + script_executable = '/bin/bash' function_method = methods.Python type = 'task' # 'sync' ignore_fields = [] actions = [] - # TODO type: 'script', execution:'task' - __metaclass__ = plugins.PluginMount def __unicode__(self): @@ -36,7 +36,28 @@ class ServiceBackend(plugins.Plugin): return unicode(self) def __init__(self): - self.cmds = [] + self.head = [] + self.content = [] + self.tail = [] + + def __getattribute__(self, attr): + """ Select head, content or tail section depending on the method name """ + IGNORE_ATTRS = ( + 'append', + 'cmd_section', + 'head', + 'tail', + 'content', + 'script_method', + 'function_method' + ) + if attr == 'prepare': + self.cmd_section = self.head + elif attr == 'commit': + self.cmd_section = self.tail + elif attr not in IGNORE_ATTRS: + self.cmd_section = self.content + return super(ServiceBackend, self).__getattribute__(attr) @classmethod def get_actions(cls): @@ -91,16 +112,34 @@ class ServiceBackend(plugins.Plugin): def model_class(cls): return get_model(cls.model) + @property + def scripts(self): + """ group commands based on their method """ + if not self.content: + return [] + scripts = {} + for method, cmd in self.content: + scripts[method] = [] + for method, commands in self.head + self.content + self.tail: + try: + scripts[method] += commands + except KeyError: + pass + return list(scripts.iteritems()) + def get_banner(self): time = timezone.now().strftime("%h %d, %Y %I:%M:%S") return "Generated by Orchestra at %s" % time def execute(self, server, async=False): from .models import BackendLog - state = BackendLog.STARTED if self.cmds else BackendLog.SUCCESS + scripts = self.scripts + state = BackendLog.STARTED + if not scripts: + state = BackendLog.SUCCESS log = BackendLog.objects.create(backend=self.get_name(), state=state, server=server) - for method, cmds in self.cmds: - method(log, server, cmds, async) + for method, commands in scripts: + method(log, server, commands, async) if log.state != BackendLog.SUCCESS: break return log @@ -113,22 +152,29 @@ class ServiceBackend(plugins.Plugin): else: method = self.function_method cmd = partial(*cmd) - if not self.cmds or self.cmds[-1][0] != method: - self.cmds.append((method, [cmd])) + if not self.cmd_section or self.cmd_section[-1][0] != method: + self.cmd_section.append((method, [cmd])) else: - self.cmds[-1][1].append(cmd) + self.cmd_section[-1][1].append(cmd) def prepare(self): - """ hook for executing something at the beging """ - pass + """ + hook for executing something at the beging + define functions or initialize state + """ + self.append( + 'set -e\n' + 'set -o pipefail' + ) def commit(self): """ + hook for executing something at the end apply the configuration, usually reloading a service reloading a service is done in a separated method in order to reload the service once in bulk operations """ - pass + self.append('exit 0') class ServiceController(ServiceBackend): diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index b9eeba69..abe69f08 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -18,8 +18,14 @@ logger = logging.getLogger(__name__) transports = {} -def BashSSH(backend, log, server, cmds, async=False): - script = '\n'.join(['set -e', 'set -o pipefail'] + cmds + ['exit 0']) +def SSH(backend, log, server, cmds, async=False): + """ + Executes cmds to remote server using SSH + + The script is first copied using SCP in order to overflood the channel with large scripts + Then the script is executed using the defined backend.script_executable + """ + script = '\n'.join(cmds) script = script.replace('\r', '') digest = hashlib.md5(script).hexdigest() path = os.path.join(settings.ORCHESTRATION_TEMP_SCRIPT_PATH, digest) @@ -41,9 +47,9 @@ def BashSSH(backend, log, server, cmds, async=False): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) addr = server.get_address() + key = settings.ORCHESTRATION_SSH_KEY_PATH try: - # TODO timeout - ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH, timeout=10) + ssh.connect(addr, username='root', key_filename=key, timeout=10) except socket.error: logger.error('%s timed out on %s' % (backend, server)) log.state = log.TIMEOUT @@ -60,12 +66,13 @@ def BashSSH(backend, log, server, cmds, async=False): # Execute it context = { + 'executable': backend.script_executable, 'remote_path': remote_path, 'digest': digest, 'remove': '' if djsettings.DEBUG else "rm -fr %(remote_path)s\n", } cmd = ( - "[[ $(md5sum %(remote_path)s|awk {'print $1'}) == %(digest)s ]] && bash %(remote_path)s\n" + "[[ $(md5sum %(remote_path)s|awk {'print $1'}) == %(digest)s ]] && %(executable)s %(remote_path)s\n" "RETURN_CODE=$?\n" "%(remove)s" "exit $RETURN_CODE" % context @@ -108,6 +115,7 @@ def BashSSH(backend, log, server, cmds, async=False): def Python(backend, log, server, cmds, async=False): + # TODO collect stdout? script = [ str(cmd.func.func_name) + str(cmd.args) for cmd in cmds ] script = json.dumps(script, indent=4).replace('"', '') log.script = '\n'.join([log.script, script]) diff --git a/orchestra/apps/resources/actions.py b/orchestra/apps/resources/actions.py index 1e7532f1..5a1f5865 100644 --- a/orchestra/apps/resources/actions.py +++ b/orchestra/apps/resources/actions.py @@ -1,6 +1,8 @@ from django.contrib import messages +from django.core.urlresolvers import reverse from django.db import transaction from django.shortcuts import redirect +from django.utils.safestring import mark_safe from django.utils.translation import ungettext, ugettext_lazy as _ @@ -10,23 +12,33 @@ def run_monitor(modeladmin, request, queryset): if not queryset: modeladmin.message_user(request, _("No resource has been selected,")) return redirect(referer) + async = modeladmin.model.monitor.func_defaults[0] + results = [] for resource in queryset: - resource.monitor() + result = resource.monitor() + if not async: + results += result modeladmin.log_change(request, resource, _("Run monitors")) num = len(queryset) - async = resource.monitor.func_defaults[0] if async: - # TODO schedulet link to celery taskstate page + link = reverse('admin:djcelery_taskstate_changelist') msg = ungettext( - _("One selected resource has been scheduled for monitoring."), - _("%s selected resource have been scheduled for monitoring.") % num, + _("One selected resource has been scheduled for monitoring.") % link, + _("%s selected resource have been scheduled for monitoring.") % (num, link), num) else: - msg = ungettext( - _("One selected resource has been monitored."), - _("%s selected resource have been monitored.") % num, - num) - modeladmin.message_user(request, msg) + if len(results) == 1: + log = results[0].log + link = reverse('admin:orchestration_backendlog_change', args=(log.pk,)) + msg = _("One selected resource has been monitored.") % link + elif len(results) >= 1: + logs = [str(result.log.pk) for result in results] + link = reverse('admin:orchestration_backendlog_changelist') + link += '?id__in=%s' % ','.join(logs) + msg = _("%s selected resources have been monitored.") % (num, link) + else: + msg = _("No related monitors have been executed.") + modeladmin.message_user(request, mark_safe(msg)) if referer: return redirect(referer) run_monitor.url_name = 'monitor' diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index d1e2f433..4b72f52c 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -130,7 +130,7 @@ class Resource(models.Model): def monitor(self, async=True): if async: return tasks.monitor.delay(self.pk, async=async) - tasks.monitor(self.pk, async=async) + return tasks.monitor(self.pk, async=async) class ResourceData(models.Model): diff --git a/orchestra/apps/resources/tasks.py b/orchestra/apps/resources/tasks.py index 3c4c9cd6..196ec182 100644 --- a/orchestra/apps/resources/tasks.py +++ b/orchestra/apps/resources/tasks.py @@ -12,6 +12,7 @@ def monitor(resource_id, ids=None, async=True): resource = Resource.objects.get(pk=resource_id) resource_model = resource.content_type.model_class() + operations = [] # Execute monitors for monitor_name in resource.monitors: backend = ServiceMonitor.get_backend(monitor_name) @@ -23,16 +24,18 @@ def monitor(resource_id, ids=None, async=True): kwargs = { path: ids } - operations = [] # Execute monitor + monitorings = [] for obj in model.objects.filter(**kwargs): - operations.append(Operation.create(backend, obj, Operation.MONITOR)) + op = Operation.create(backend, obj, Operation.MONITOR) + operations.append(op) + monitorings.append(op) # TODO async=TRue only when running with celery - Operation.execute(operations, async=async) + Operation.execute(monitorings, async=async) kwargs = {'id__in': ids} if ids else {} # Update used resources and trigger resource exceeded and revovery - operations = [] + triggers = [] model = resource.content_type.model_class() for obj in model.objects.filter(**kwargs): data = ResourceData.get_or_create(obj, resource) @@ -40,8 +43,9 @@ def monitor(resource_id, ids=None, async=True): if not resource.disable_trigger: if data.used > data.allocated: op = Operation.create(backend, obj, Operation.EXCEED) - operations.append(op) + triggers.append(op) elif data.used < data.allocated: op = Operation.create(backend, obj, Operation.RECOVERY) - operations.append(op) - Operation.execute(operations) + triggers.append(op) + Operation.execute(triggers) + return operations diff --git a/orchestra/apps/systemusers/backends.py b/orchestra/apps/systemusers/backends.py index 92bd8d79..4f720994 100644 --- a/orchestra/apps/systemusers/backends.py +++ b/orchestra/apps/systemusers/backends.py @@ -74,7 +74,7 @@ class SystemUserDisk(ServiceMonitor): verbose_name = _('Systemuser disk') def prepare(self): - """ slower """ + super(SystemUserDisk, self).prepare() self.append(textwrap.dedent("""\ function monitor () { { du -bs "$1" || echo 0; } | awk {'print $1'} @@ -102,6 +102,7 @@ class FTPTraffic(ServiceMonitor): verbose_name = _('Systemuser FTP traffic') def prepare(self): + super(FTPTraffic, self).prepare() current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") self.append(textwrap.dedent("""\ function monitor () { diff --git a/orchestra/apps/webapps/models.py b/orchestra/apps/webapps/models.py index 2f9280e8..3faafe83 100644 --- a/orchestra/apps/webapps/models.py +++ b/orchestra/apps/webapps/models.py @@ -37,7 +37,7 @@ class WebApp(models.Model): return self.name or settings.WEBAPPS_BLANK_NAME def get_fpm_port(self): - return settings.WEBAPPS_FPM_START_PORT + self.account.pk + return settings.WEBAPPS_FPM_START_PORT + self.account_id def get_directive(self): directive = settings.WEBAPPS_TYPES[self.type]['directive'] diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 25eac682..8e7416e7 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -218,6 +218,7 @@ class Apache2Traffic(ServiceMonitor): verbose_name = _("Apache 2 Traffic") def prepare(self): + super(Apache2Traffic, self).prepare() ignore_hosts = '\\|'.join(settings.WEBSITES_TRAFFIC_IGNORE_HOSTS) context = { 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), diff --git a/orchestra/utils/tests.py b/orchestra/utils/tests.py index a802bafb..6fc9f7c1 100644 --- a/orchestra/utils/tests.py +++ b/orchestra/utils/tests.py @@ -99,7 +99,7 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): def admin_login(self): session = SessionStore() - session[SESSION_KEY] = self.account.pk + session[SESSION_KEY] = self.account_id session[BACKEND_SESSION_KEY] = settings.AUTHENTICATION_BACKENDS[0] session.save() ## to set a cookie we need to first visit the domain.