From 55f4b6e88cd107731fdb880903fa9ef4f5ba0a11 Mon Sep 17 00:00:00 2001 From: Marc Date: Sat, 4 Oct 2014 09:29:18 +0000 Subject: [PATCH] fucked up tests --- orchestra/apps/domains/backends.py | 21 +++++++---- orchestra/apps/domains/helpers.py | 37 +++++++++++-------- orchestra/apps/domains/models.py | 29 ++++++--------- orchestra/apps/orchestration/backends.py | 2 +- orchestra/apps/orchestration/manager.py | 10 +++-- orchestra/apps/orchestration/methods.py | 14 +++++-- orchestra/apps/orchestration/models.py | 9 ++++- .../apps/orchestration/tests/test_route.py | 27 +++++++------- orchestra/apps/services/handlers.py | 4 +- orchestra/utils/tests.py | 1 + scripts/services/postfix.md | 13 ++++--- 11 files changed, 97 insertions(+), 70 deletions(-) diff --git a/orchestra/apps/domains/backends.py b/orchestra/apps/domains/backends.py index 51eb58d9..1a266d08 100644 --- a/orchestra/apps/domains/backends.py +++ b/orchestra/apps/domains/backends.py @@ -43,10 +43,11 @@ class Bind9MasterDomainBackend(ServiceController): self.delete_conf(context) def delete_conf(self, context): - self.append('awk -v s=%(name)s \'BEGIN {' - ' RS=""; s="zone \\""s"\\""' - '} $0!~s{ print $0"\\n" }\' %(conf_path)s > %(conf_path)s.tmp' - % context) + self.append(textwrap.dedent(""" + awk -v s=%(name)s 'BEGIN { + RS=""; s="zone \""s"\"" + } $0!~s{ print $0"\n" }' %(conf_path)s > %(conf_path)s.tmp""" % context + )) self.append('diff -I"^\s*//" %(conf_path)s.tmp %(conf_path)s || UPDATED=1' % context) self.append('mv %(conf_path)s.tmp %(conf_path)s' % context) @@ -62,13 +63,16 @@ class Bind9MasterDomainBackend(ServiceController): servers.append(server.get_ip()) return servers + def get_slaves(self, domain): + return self.get_servers(domain, Bind9SlaveDomainBackend) + def get_context(self, domain): context = { 'name': domain.name, 'zone_path': settings.DOMAINS_ZONE_PATH % {'name': domain.name}, 'subdomains': domain.subdomains.all(), 'banner': self.get_banner(), - 'slaves': '; '.join(self.get_servers(domain, Bind9SlaveDomainBackend)), + 'slaves': '; '.join(self.get_slaves(domain)) or 'none', } context.update({ 'conf_path': settings.DOMAINS_MASTERS_PATH, @@ -96,17 +100,20 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): def delete(self, domain): context = self.get_context(domain) self.delete_conf(context) - + def commit(self): """ ideally slave should be restarted after master """ self.append('[[ $UPDATED == 1 ]] && { sleep 1 && service bind9 reload; } &') + def get_masters(self, domain): + return self.get_servers(domain, Bind9MasterDomainBackend) + def get_context(self, domain): context = { 'name': domain.name, 'banner': self.get_banner(), 'subdomains': domain.subdomains.all(), - 'masters': '; '.join(self.get_servers(domain, Bind9MasterDomainBackend)), + 'masters': '; '.join(self.get_masters(domain)) or 'none', } context.update({ 'conf_path': settings.DOMAINS_SLAVES_PATH, diff --git a/orchestra/apps/domains/helpers.py b/orchestra/apps/domains/helpers.py index 14e10a6b..da22c287 100644 --- a/orchestra/apps/domains/helpers.py +++ b/orchestra/apps/domains/helpers.py @@ -1,33 +1,38 @@ import copy +from functools import partial from .models import Domain, Record def domain_for_validation(instance, records): - """ Create a fake zone in order to generate the whole zone file and check it """ + """ + Since the new data is not yet on the database, we update it on the fly, + so when validation calls render_zone() it will use the new provided data + """ domain = copy.copy(instance) - if not domain.pk: - domain.top = domain.get_top() + def get_records(): for data in records: yield Record(type=data['type'], value=data['value']) domain.get_records = get_records - def get_top_subdomains(exclude=None): - subdomains = [] - for subdomain in Domain.objects.filter(name__endswith='.%s' % domain.origin.name): - if exclude != subdomain.pk: - subdomain.top = domain + def get_subdomains(replace=None, make_top=False): + for subdomain in Domain.objects.filter(name__endswith='.%s' % domain.name): + if replace == subdomain.pk: + # domain is a subdomain, yield our copy + yield domain + else: + if make_top: + subdomain.top = domain yield subdomain - domain.get_top_subdomains = get_top_subdomains + if not domain.pk: + # top domain lookup for new domains + domain.top = domain.get_top() if domain.top: - subdomains = domain.get_top_subdomains(exclude=instance.pk) - domain.top.get_subdomains = lambda: list(subdomains) + [domain] + # is a subdomains + domain.top.get_subdomains = partial(get_subdomains, replace=domain.pk) elif not domain.pk: - subdomains = [] - for subdomain in Domain.objects.filter(name__endswith=domain.name): - subdomain.top = domain - subdomains.append(subdomain) - domain.get_subdomains = get_top_subdomains + # is top domain + domain.get_subdomains = partial(get_subdomains, make_top=True) return domain diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index 7cf1947c..356f4c16 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -24,30 +24,35 @@ class Domain(models.Model): @property def origin(self): - # Do not cache return self.top or self @property def is_top(self): - # Do not cache + # don't cache, don't replace by top_id return not bool(self.top) def get_records(self): """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.records.all() - def get_top_subdomains(self): + def get_subdomains(self): """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.origin.subdomains.all() - def get_subdomains(self): - """ proxy method, needed for input validation, see helpers.domain_for_validation """ - return self.get_top_subdomains().filter(name__endswith=r'.%s' % self.name) + def get_top(self): + split = self.name.split('.') + top = None + for i in range(1, len(split)-1): + name = '.'.join(split[i:]) + domain = Domain.objects.filter(name=name) + if domain: + top = domain.get() + return top def render_zone(self): origin = self.origin zone = origin.render_records() - for subdomain in origin.get_top_subdomains(): + for subdomain in origin.get_subdomains(): zone += subdomain.render_records() return zone @@ -134,16 +139,6 @@ class Domain(models.Model): domain.top = self domain.save(update_fields=['top']) self.subdomains.update(account_id=self.account_id) - - def get_top(self): - split = self.name.split('.') - top = None - for i in range(1, len(split)-1): - name = '.'.join(split[i:]) - domain = Domain.objects.filter(name=name) - if domain: - top = domain.get() - return top class Record(models.Model): diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index 73f68a53..a5a092ef 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -71,7 +71,7 @@ class ServiceBackend(plugins.Plugin): def get_banner(self): time = timezone.now().strftime("%h %d, %Y %I:%M:%S") - return "Generated by Orchestra %s" % time + return "Generated by Orchestra at %s" % time def execute(self, server): from .models import BackendLog diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index 80b5254e..f2373302 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -37,9 +37,9 @@ def close_connection(execute): def execute(operations): """ generates and executes the operations on the servers """ router = import_class(settings.ORCHESTRATION_ROUTER) - # Generate scripts per server+backend scripts = {} cache = {} + # Generate scripts per server+backend for operation in operations: logger.debug("Queued %s" % str(operation)) servers = router.get_servers(operation, cache=cache) @@ -50,6 +50,7 @@ def execute(operations): scripts[key][0].prepare() else: scripts[key][1].append(operation) + # Get and call backend action method method = getattr(scripts[key][0], operation.action) method(operation.instance) # Execute scripts on each server @@ -67,12 +68,15 @@ def execute(operations): executions.append((execute, operations)) [ thread.join() for thread in threads ] logs = [] + # collect results for execution, operations in executions: for operation in operations: logger.info("Executed %s" % str(operation)) operation.log = execution.log operation.save() - logger.debug(execution.log.stdout) - logger.debug(execution.log.stderr) + stdout = execution.log.stdout.strip() + stdout and logger.debug('STDOUT %s', stdout) + stderr = execution.log.stderr.strip() + stderr and logger.debug('STDERR %s', stderr) logs.append(execution.log) return logs diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index 056c7570..c30c7b1f 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -1,5 +1,6 @@ import hashlib import json +import logging import os import socket import sys @@ -11,13 +12,16 @@ from celery.datastructures import ExceptionInfo from . import settings +logger = logging.getLogger(__name__) + + def BashSSH(backend, log, server, cmds): from .models import BackendLog script = '\n'.join(['set -e', 'set -o pipefail'] + cmds + ['exit 0']) script = script.replace('\r', '') log.script = script log.save(update_fields=['script']) - + logger.debug('%s is going to be executed on %s' % (backend, server)) try: # Avoid "Argument list too long" on large scripts by genereting a file # and scping it to the remote server @@ -30,15 +34,16 @@ def BashSSH(backend, log, server, cmds): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) addr = server.get_address() try: - ssh.connect(addr, username='root', - key_filename=settings.ORCHESTRATION_SSH_KEY_PATH) + ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH) except socket.error: + logger.error('%s timed out on %s' % (backend, server)) log.state = BackendLog.TIMEOUT log.save(update_fields=['state']) return transport = ssh.get_transport() sftp = paramiko.SFTPClient.from_transport(transport) sftp.put(path, "%s.remote" % path) + logger.debug('%s copied on %s' % (backend, server)) sftp.close() os.remove(path) @@ -55,6 +60,7 @@ def BashSSH(backend, log, server, cmds): channel = transport.open_session() channel.exec_command(cmd) + logger.debug('%s running on %s' % (backend, server)) if True: # TODO if not async log.stdout += channel.makefile('rb', -1).read().decode('utf-8') log.stderr += channel.makefile_stderr('rb', -1).read().decode('utf-8') @@ -71,10 +77,12 @@ def BashSSH(backend, log, server, cmds): break log.exit_code = exit_code = channel.recv_exit_status() log.state = BackendLog.SUCCESS if exit_code == 0 else BackendLog.FAILURE + logger.debug('%s execution state on %s is %s' % (backend, server, log.state)) channel.close() ssh.close() log.save() except: + logger.error('Exception while executing %s on %s' % (backend, server)) log.state = BackendLog.ERROR log.traceback = ExceptionInfo(sys.exc_info()).traceback log.save() diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index cedcb492..2e13a65f 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -176,7 +176,10 @@ class Route(models.Model): for route in cls.objects.filter(is_active=True, backend=backend.get_name()): for action in backend.get_actions(): _key = (route.backend, action) - cache[_key] = [route] + try: + cache[_key].append(route) + except KeyError: + cache[_key] = [route] routes = cache[key] for route in routes: if route.matches(operation.instance): @@ -185,7 +188,9 @@ class Route(models.Model): def matches(self, instance): safe_locals = { - 'instance': instance + 'instance': instance, + 'obj': instance, + instance._meta.model_name: instance, } return eval(self.match, safe_locals) diff --git a/orchestra/apps/orchestration/tests/test_route.py b/orchestra/apps/orchestration/tests/test_route.py index 9a30647d..62efb040 100644 --- a/orchestra/apps/orchestration/tests/test_route.py +++ b/orchestra/apps/orchestration/tests/test_route.py @@ -1,8 +1,7 @@ from orchestra.utils.tests import BaseTestCase -from .. import operations, backends -from ..models import Route, Server -from ..utils import get_backend_choices +from .. import backends +from ..models import Route, Server, BackendOperation as Operation class RouterTests(BaseTestCase): @@ -18,25 +17,25 @@ class RouterTests(BaseTestCase): def test_get_instances(self): - class TestBackend(backends.ServiceBackend): + class TestBackend(backends.ServiceController): verbose_name = 'Route' - models = ['routes.Route',] + models = ['routes.Route'] + + def save(self, instance): + pass - choices = get_backend_choices(backends.ServiceBackend.get_backends()) + choices = backends.ServiceBackend.get_plugin_choices() Route._meta.get_field_by_name('backend')[0]._choices = choices backend = TestBackend.get_name() - route = Route.objects.create(backend=backend, host=self.host, - match='True') - operation = operations.Operation(TestBackend, route, 'commit') + route = Route.objects.create(backend=backend, host=self.host, match='True') + operation = Operation(backend=TestBackend, instance=route, action='save') self.assertEqual(1, len(Route.get_servers(operation))) - + route = Route.objects.create(backend=backend, host=self.host1, - match='instance.backend == "TestBackend"') - operation = operations.Operation(TestBackend, route, 'commit') + match='route.backend == "%s"' % TestBackend.get_name()) self.assertEqual(2, len(Route.get_servers(operation))) route = Route.objects.create(backend=backend, host=self.host2, - match='instance.backend == "something else"') - operation = operations.Operation(TestBackend, route, 'commit') + match='route.backend == "something else"') self.assertEqual(2, len(Route.get_servers(operation))) diff --git a/orchestra/apps/services/handlers.py b/orchestra/apps/services/handlers.py index 5a8c5d80..c666dc9c 100644 --- a/orchestra/apps/services/handlers.py +++ b/orchestra/apps/services/handlers.py @@ -46,7 +46,9 @@ class ServiceHandler(plugins.Plugin): def matches(self, instance): safe_locals = { - instance._meta.model_name: instance + 'instance': instance, + 'obj': instance, + instance._meta.model_name: instance, } return eval(self.match, safe_locals) diff --git a/orchestra/utils/tests.py b/orchestra/utils/tests.py index a89dfffb..aa7ed864 100644 --- a/orchestra/utils/tests.py +++ b/orchestra/utils/tests.py @@ -90,6 +90,7 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): def setUp(self): super(BaseLiveServerTestCase, self).setUp() self.rest = Api(self.live_server_url + '/api/') + self.rest.enable_logging() self.account = self.create_account(superuser=True) def admin_login(self): diff --git a/scripts/services/postfix.md b/scripts/services/postfix.md index 90c94538..9e7f7873 100644 --- a/scripts/services/postfix.md +++ b/scripts/services/postfix.md @@ -1,5 +1,3 @@ -apt-get install postfix - # http://www.postfix.org/VIRTUAL_README.html#virtual_mailbox # https://help.ubuntu.com/community/PostfixVirtualMailBoxClamSmtpHowto @@ -9,15 +7,17 @@ apt-get install postfix apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve -sed -i "s#^mail_location = mbox.*#mail_location = maildir:~/Maildir#" /etc/dovecot/conf.d/10-mail.conf -echo 'auth_username_format = %n' >> /etc/dovecot/conf.d/10-auth.conf -echo 'service lmtp { + +echo 'mail_location = maildir:~/Maildir +mail_plugins = quota +auth_username_format = %n +service lmtp { unix_listener /var/spool/postfix/private/dovecot-lmtp { group = postfix mode = 0600 user = postfix } -}' >> /etc/dovecot/conf.d/10-master.conf +}' > /etc/dovecot/local.conf cat > /etc/apt/sources.list.d/mailscanner.list << 'EOF' @@ -38,6 +38,7 @@ echo 'mailbox_transport = lmtp:unix:private/dovecot-lmtp' >> /etc/postfix/main.c + /etc/init.d/dovecot restart /etc/init.d/postfix restart