From c5abcf6b45161427479c1172172ad5a930eeb69f Mon Sep 17 00:00:00 2001 From: Marc Date: Sat, 4 Oct 2014 13:23:04 +0000 Subject: [PATCH] minor fixes on domains0 --- .../databases/tests/functional_tests/tests.py | 2 +- orchestra/apps/domains/backends.py | 9 +++--- orchestra/apps/domains/helpers.py | 24 ++++++---------- .../domains/tests/functional_tests/tests.py | 15 ++++------ .../mails/tests/functional_tests/tests.py | 28 +++++++++++++++++++ orchestra/apps/orchestration/methods.py | 18 ++++++++---- orchestra/bin/orchestra-admin | 2 +- orchestra/utils/system.py | 2 +- orchestra/utils/tests.py | 6 ++-- 9 files changed, 67 insertions(+), 39 deletions(-) create mode 100644 orchestra/apps/mails/tests/functional_tests/tests.py diff --git a/orchestra/apps/databases/tests/functional_tests/tests.py b/orchestra/apps/databases/tests/functional_tests/tests.py index 7cdaa26e..ec7e8bd3 100644 --- a/orchestra/apps/databases/tests/functional_tests/tests.py +++ b/orchestra/apps/databases/tests/functional_tests/tests.py @@ -1,4 +1,4 @@ -import MySQLdb +#import MySQLdb from functools import partial from django.conf import settings as djsettings diff --git a/orchestra/apps/domains/backends.py b/orchestra/apps/domains/backends.py index 1a266d08..29be0800 100644 --- a/orchestra/apps/domains/backends.py +++ b/orchestra/apps/domains/backends.py @@ -3,6 +3,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController +from orchestra.utils.python import AttrDict from . import settings @@ -45,8 +46,8 @@ class Bind9MasterDomainBackend(ServiceController): def delete_conf(self, context): self.append(textwrap.dedent(""" awk -v s=%(name)s 'BEGIN { - RS=""; s="zone \""s"\"" - } $0!~s{ print $0"\n" }' %(conf_path)s > %(conf_path)s.tmp""" % context + RS=""; s="zone \\""s"\\"" + } $0!~s{ print $0"\\n" }' %(conf_path)s > %(conf_path)s.tmp""" % context )) self.append('diff -I"^\s*//" %(conf_path)s.tmp %(conf_path)s || UPDATED=1' % context) self.append('mv %(conf_path)s.tmp %(conf_path)s' % context) @@ -56,8 +57,8 @@ class Bind9MasterDomainBackend(ServiceController): self.append('[[ $UPDATED == 1 ]] && service bind9 reload') def get_servers(self, domain, backend): - from orchestra.apps.orchestration.models import Route, BackendOperation as Operation - operation = Operation(backend=backend, action='save', instance=domain) + from orchestra.apps.orchestration.models import Route + operation = AttrDict(backend=backend, action='save', instance=domain) servers = [] for server in Route.get_servers(operation): servers.append(server.get_ip()) diff --git a/orchestra/apps/domains/helpers.py b/orchestra/apps/domains/helpers.py index da22c287..ff32046d 100644 --- a/orchestra/apps/domains/helpers.py +++ b/orchestra/apps/domains/helpers.py @@ -10,29 +10,23 @@ def domain_for_validation(instance, records): so when validation calls render_zone() it will use the new provided data """ domain = copy.copy(instance) - def get_records(): for data in records: yield Record(type=data['type'], value=data['value']) domain.get_records = get_records - def get_subdomains(replace=None, make_top=False): - for subdomain in Domain.objects.filter(name__endswith='.%s' % domain.name): - if replace == subdomain.pk: - # domain is a subdomain, yield our copy - yield domain - else: - if make_top: - subdomain.top = domain - yield subdomain - if not domain.pk: # top domain lookup for new domains domain.top = domain.get_top() if domain.top: - # is a subdomains - domain.top.get_subdomains = partial(get_subdomains, replace=domain.pk) + # is a subdomain + subdomains = [sub for sub in domain.top.subdomains.all() if sub.pk != domain.pk] + domain.top.get_subdomains = lambda: subdomains + [domain] elif not domain.pk: - # is top domain - domain.get_subdomains = partial(get_subdomains, make_top=True) + # is a new top domain + subdomains = [] + for subdomain in Domain.objects.filter(name__endswith='.%s' % domain.name): + subdomain.top = domain + subdomains.append(subdomain) + domain.get_subdomains = lambda: subdomains return domain diff --git a/orchestra/apps/domains/tests/functional_tests/tests.py b/orchestra/apps/domains/tests/functional_tests/tests.py index dae75954..979db056 100644 --- a/orchestra/apps/domains/tests/functional_tests/tests.py +++ b/orchestra/apps/domains/tests/functional_tests/tests.py @@ -1,7 +1,7 @@ -import functools import os import time import socket +from functools import partial from django.conf import settings as djsettings from django.core.urlresolvers import reverse @@ -15,7 +15,7 @@ from ... import settings, utils, backends from ...models import Domain, Record -run = functools.partial(run, display=False) +run = partial(run, display=False) class DomainTestMixin(object): @@ -54,13 +54,6 @@ class DomainTestMixin(object): ) self.django_domain_name = 'django%s.lan' % random_ascii(10) - def tearDown(self): - try: - self.delete(self.domain_name) - except Domain.DoesNotExist: - pass - super(DomainTestMixin, self).tearDown() - def add_route(self): raise NotImplementedError @@ -183,8 +176,9 @@ class DomainTestMixin(object): self.add(self.ns1_name, self.ns1_records) self.add(self.ns2_name, self.ns2_records) self.add(self.domain_name, self.domain_records) + self.addCleanup(partial(self.delete, self.domain_name)) self.validate_add(self.MASTER_SERVER_ADDR, self.domain_name) - time.sleep(0.5) + time.sleep(1) self.validate_add(self.SLAVE_SERVER_ADDR, self.domain_name) def test_delete(self): @@ -200,6 +194,7 @@ class DomainTestMixin(object): self.add(self.ns1_name, self.ns1_records) self.add(self.ns2_name, self.ns2_records) self.add(self.domain_name, self.domain_records) + self.addCleanup(partial(self.delete, self.domain_name)) self.update(self.domain_name, self.domain_update_records) self.add(self.www_name, self.www_records) self.validate_update(self.MASTER_SERVER_ADDR, self.domain_name) diff --git a/orchestra/apps/mails/tests/functional_tests/tests.py b/orchestra/apps/mails/tests/functional_tests/tests.py new file mode 100644 index 00000000..4a8448c5 --- /dev/null +++ b/orchestra/apps/mails/tests/functional_tests/tests.py @@ -0,0 +1,28 @@ +#import imaplib +#mail = imaplib.IMAP4_SSL('localhost') +#mail.login('rata', '3') +#('OK', ['Logged in']) + +#>>> mail.list() +#('OK', ['(\\HasNoChildren) "." INBOX']) +#>>> mail.select('INBOX') +#('OK', ['18']) + +#>>> mail.getquota('INBOX') +#imaplib.error: GETQUOTA command error: BAD ['Error in IMAP command GETQUOTA: Unknown command.'] + +#mail.fetch(10, '(RFC822)') +#('OK', [('10 (FLAGS (\\Seen) RFC822 {550}', 'Return-Path: \r\nDelivered-To: \r\nReceived: from test3.orchestra.lan\r\n\tby test3.orchestra.lan (Dovecot) with LMTP id hvDUEAIKL1QlOQAAL4hJug\r\n\tfor ; Fri, 03 Oct 2014 16:41:38 -0400\r\nReceived: by test3.orchestra.lan (Postfix, from userid 0)\r\n\tid 43BB1F94633; Fri, 3 Oct 2014 16:41:38 -0400 (EDT)\r\nTo: rata@orchestra.lan\r\nSubject: hola\r\nMessage-Id: <20141003204138.43BB1F94633@test3.orchestra.lan>\r\nDate: Fri, 3 Oct 2014 16:41:38 -0400 (EDT)\r\nFrom: root@test3.orchestra.lan (root)\r\n\r\n\r\n\r\n'), ')']) +#>>> mail.close() +#('OK', ['Close completed.']) + + +#import poplib +#pop = poplib.POP3('localhost') +#pop.user('rata') +#pop.pass_('3') +#>>> pop.list() +#('+OK 18 messages:', ['1 552', '2 550', '3 550', '4 548', '5 546', '6 546', '7 554', '8 548', '9 550', '10 550', '11 546', '12 546', '13 546', '14 544', '15 548', '16 577', '17 546', '18 546'], 135) +#>>> pop.quit() +#'+OK Logging out.' + diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index c30c7b1f..7771a7e5 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -14,6 +14,8 @@ from . import settings logger = logging.getLogger(__name__) +transports = {} + def BashSSH(backend, log, server, cmds): from .models import BackendLog @@ -29,6 +31,7 @@ def BashSSH(backend, log, server, cmds): path = os.path.join(settings.ORCHESTRATION_TEMP_SCRIPT_PATH, digest) with open(path, 'w') as script_file: script_file.write(script) + # ssh connection ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -41,12 +44,14 @@ def BashSSH(backend, log, server, cmds): log.save(update_fields=['state']) return transport = ssh.get_transport() + + # Copy script to remote server sftp = paramiko.SFTPClient.from_transport(transport) sftp.put(path, "%s.remote" % path) - logger.debug('%s copied on %s' % (backend, server)) sftp.close() os.remove(path) + # Execute it context = { 'path': "%s.remote" % path, 'digest': digest @@ -57,9 +62,10 @@ def BashSSH(backend, log, server, cmds): # TODO "rm -fr %(path)s\n" "exit $RETURN_CODE" % context ) - channel = transport.open_session() channel.exec_command(cmd) + + # Log results logger.debug('%s running on %s' % (backend, server)) if True: # TODO if not async log.stdout += channel.makefile('rb', -1).read().decode('utf-8') @@ -78,14 +84,16 @@ def BashSSH(backend, log, server, cmds): log.exit_code = exit_code = channel.recv_exit_status() log.state = BackendLog.SUCCESS if exit_code == 0 else BackendLog.FAILURE logger.debug('%s execution state on %s is %s' % (backend, server, log.state)) - channel.close() - ssh.close() log.save() except: - logger.error('Exception while executing %s on %s' % (backend, server)) log.state = BackendLog.ERROR log.traceback = ExceptionInfo(sys.exc_info()).traceback + logger.error('Exception while executing %s on %s' % (backend, server)) + logger.debug(log.traceback) log.save() + finally: + channel.close() + ssh.close() def Python(backend, log, server, cmds): diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index a4cb0973..084096ca 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -144,7 +144,7 @@ function install_requirements () { Markdown==2.4 \ django-debug-toolbar==1.2.1 \ djangorestframework==2.3.14 \ - paramiko==1.12.1 \ + paramiko==1.15.1 \ Pygments==1.6 \ django-filter==0.7 \ passlib==1.6.2 \ diff --git a/orchestra/utils/system.py b/orchestra/utils/system.py index d22ae609..da09610a 100644 --- a/orchestra/utils/system.py +++ b/orchestra/utils/system.py @@ -99,7 +99,7 @@ def run(command, display=True, error_codes=[0], silent=False, stdin=''): if display: sys.stderr.write("\n\033[1;31mCommandError: %s %s\033[m\n" % (msg, err)) if not silent: - raise CommandError("%s %s" % (msg, err)) + raise CommandError("%s %s %s" % (msg, err, out)) out.succeeded = not out.failed return out diff --git a/orchestra/utils/tests.py b/orchestra/utils/tests.py index aa7ed864..f68bfa95 100644 --- a/orchestra/utils/tests.py +++ b/orchestra/utils/tests.py @@ -1,4 +1,5 @@ import datetime +import os import string import random from functools import wraps @@ -118,7 +119,8 @@ def snapshot_on_error(test): except: self = args[0] timestamp = datetime.datetime.now().isoformat().replace(':', '') - filename = '/tmp/screenshot_%s_%s.png' % (self.id(), timestamp) - self.selenium.save_screenshot(filename) + filename = 'screenshot_%s_%s.png' % (self.id(), timestamp) + path = '/home/orchestra/snapshots' + self.selenium.save_screenshot(os.path.join(path, filename)) raise return inner