From 9ecfc8d4dd2790d36cebfc751744c1ff3e31c3a1 Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 3 Oct 2014 14:02:11 +0000 Subject: [PATCH] domains app functional tests passing --- ROADMAP.md | 7 +- TODO.md | 15 ++- orchestra/apps/accounts/admin.py | 10 +- orchestra/apps/databases/tests/__init__.py | 0 .../tests/functional_tests/__init__.py | 0 .../databases/tests/functional_tests/tests.py | 76 +++++++++++++ orchestra/apps/domains/backends.py | 6 +- orchestra/apps/domains/forms.py | 21 +--- orchestra/apps/domains/helpers.py | 13 ++- orchestra/apps/domains/models.py | 23 ++-- orchestra/apps/domains/serializers.py | 1 - orchestra/apps/domains/settings.py | 4 +- .../domains/tests/functional_tests/tests.py | 106 +++++++++++------- orchestra/apps/orchestration/helpers.py | 36 ++++-- orchestra/apps/orchestration/middlewares.py | 3 +- orchestra/apps/systemusers/backends.py | 1 - .../tests/functional_tests/tests.py | 53 ++++++--- orchestra/bin/orchestra-admin | 3 +- orchestra/utils/system.py | 5 + orchestra/utils/tests.py | 16 +++ scripts/services/bind9.md | 24 ++++ scripts/services/bind9.sh | 9 -- scripts/services/postfix.md | 18 ++- scripts/services/rssh.md | 2 +- scripts/services/vsftpd.md | 6 +- 25 files changed, 322 insertions(+), 136 deletions(-) create mode 100644 orchestra/apps/databases/tests/__init__.py create mode 100644 orchestra/apps/databases/tests/functional_tests/__init__.py create mode 100644 orchestra/apps/databases/tests/functional_tests/tests.py create mode 100644 scripts/services/bind9.md delete mode 100644 scripts/services/bind9.sh diff --git a/ROADMAP.md b/ROADMAP.md index efeef178..86dcd0a0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -59,7 +59,8 @@ Note `*` _for sustancial progress_ 1. [ ] Integration with third-party service providers, e.g. Gandi 2. [ ] Scheduling of service cancellations and deactivations -1. [ ] Object level permissions system -2. [ ] REST API for superusers -3. [ ] Responsive user interface +1. [ ] Object-level permission system +2. [ ] REST API functionality for superusers +3. [ ] Responsive user interface, based on a JS framework. 4. [ ] Full documentation +5. [ ] [http://www.ansible.com/home](Ansible) orchestration method, which synchronize the whole service config everytime instead of incremental changes. diff --git a/TODO.md b/TODO.md index 48f3ac79..4e5c4a7f 100644 --- a/TODO.md +++ b/TODO.md @@ -141,7 +141,7 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * Redirect junk emails and delete every 30 days? -* Complitely decouples scripts execution, billing, service definition +* DOC: Complitely decouples scripts execution, billing, service definition * Create SystemUser on account creation. username=username, is_main=True, * Exclude is_main=True from queryset filter default is_main=False @@ -149,8 +149,15 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * Unify all users -* backend message with link - -* test fucking user +* backend admin message with link * delete main user -> delete account or prevent delete main user + + +APPS app? + +* https://blog.flameeyes.eu/2011/01/mostly-unknown-openssh-tricks + +* Ansible orchestration *method* (methods.py) +* interdependency user <-> account with the old usermodel + diff --git a/orchestra/apps/accounts/admin.py b/orchestra/apps/accounts/admin.py index 665fa1fa..e58e021e 100644 --- a/orchestra/apps/accounts/admin.py +++ b/orchestra/apps/accounts/admin.py @@ -127,6 +127,7 @@ class AccountAdminMixin(object): filter_by_account_fields = [] change_list_template = 'admin/accounts/account/change_list.html' change_form_template = 'admin/accounts/account/change_form.html' + account = None def account_link(self, instance): account = instance.account if instance.pk else self.account @@ -151,7 +152,7 @@ class AccountAdminMixin(object): """ Filter by account """ formfield = super(AccountAdminMixin, self).formfield_for_dbfield(db_field, **kwargs) if db_field.name in self.filter_by_account_fields: - if hasattr(self, 'account'): + if self.account: # Hack widget render in order to append ?account=id to the add url old_render = formfield.widget.render def render(*args, **kwargs): @@ -161,6 +162,11 @@ class AccountAdminMixin(object): formfield.widget.render = render # Filter related object by account formfield.queryset = formfield.queryset.filter(account=self.account) + elif db_field.name == 'account': + if self.account: + formfield.initial = self.account.pk + elif Account.objects.count() == 1: + formfield.initial = 1 return formfield def get_account_from_preserve_filters(self, request): @@ -215,7 +221,7 @@ class SelectAccountAdminMixin(AccountAdminMixin): """ Provides support for accounts on ModelAdmin """ def get_inline_instances(self, request, obj=None): inlines = super(AccountAdminMixin, self).get_inline_instances(request, obj=obj) - if hasattr(self, 'account'): + if self.account: account = self.account else: account = Account.objects.get(pk=request.GET['account']) diff --git a/orchestra/apps/databases/tests/__init__.py b/orchestra/apps/databases/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/databases/tests/functional_tests/__init__.py b/orchestra/apps/databases/tests/functional_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/databases/tests/functional_tests/tests.py b/orchestra/apps/databases/tests/functional_tests/tests.py new file mode 100644 index 00000000..7cdaa26e --- /dev/null +++ b/orchestra/apps/databases/tests/functional_tests/tests.py @@ -0,0 +1,76 @@ +import MySQLdb +from functools import partial + +from django.conf import settings as djsettings +from django.core.management.base import CommandError +from django.core.urlresolvers import reverse +from selenium.webdriver.support.select import Select + +from orchestra.apps.accounts.models import Account +from orchestra.apps.orchestration.models import Server, Route +from orchestra.utils.system import run +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii + +from ... import backends, settings +from ...models import Satabase + + +class DatabaseTestMixin(object): + MASTER_ADDR = 'localhost' + DEPENDENCIES = ( + 'orchestra.apps.orchestration', + 'orcgestra.apps.databases', + ) + + def setUp(self): + super(SystemUserMixin, self).setUp() + self.add_route() + djsettings.DEBUG = True + + def add_route(self): + raise NotImplementedError + + def save(self): + raise NotImplementedError + + def add(self): + raise NotImplementedError + + def delete(self): + raise NotImplementedError + + def update(self): + raise NotImplementedError + + def disable(self): + raise NotImplementedError + + def add_group(self, username, groupname): + raise NotImplementedError + + def test_add(self): + self.add() + + + + +class MysqlBackendMixin(object): + def add_route(self): + server = Server.objects.create(name=self.MASTER_ADDR) + backend = backends.MysqlBackend.get_name() + Route.objects.create(backend=backend, match="database.type == 'mysql'", host=server) + + def validate_create_table(self, name, username, password): + db = MySQLdb.connect(host=self.MASTER_ADDR, user=username, passwd=password, db=name) + cur = db.cursor() + cur.execute('CREATE TABLE test;') + + def validate_delete(self, name, username, password): + self.asseRaises(MySQLdb.ConnectionError, + MySQLdb.connect(host=self.MASTER_ADDR, user=username, passwd=password, db=name)) + + + +class RESTDatabaseTest(DatabaseTestMixin): + def add(self, dbname): + self.api.databases.create(name=dbname) diff --git a/orchestra/apps/domains/backends.py b/orchestra/apps/domains/backends.py index 460d03e1..23f80a5e 100644 --- a/orchestra/apps/domains/backends.py +++ b/orchestra/apps/domains/backends.py @@ -33,7 +33,7 @@ class Bind9MasterDomainBackend(ServiceController): " { echo -e '%(conf)s' >> %(conf_path)s; UPDATED=1; }" % context) for subdomain in context['subdomains']: context['name'] = subdomain.name - self.delete_conf(context) + self.delete(subdomain) def delete(self, domain): context = self.get_context(domain) @@ -56,7 +56,7 @@ class Bind9MasterDomainBackend(ServiceController): context = { 'name': domain.name, 'zone_path': settings.DOMAINS_ZONE_PATH % {'name': domain.name}, - 'subdomains': domain.get_subdomains(), + 'subdomains': domain.subdomains.all(), 'banner': self.get_banner(), } context.update({ @@ -92,7 +92,7 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): context = { 'name': domain.name, 'masters': '; '.join(settings.DOMAINS_MASTERS), - 'subdomains': domain.get_subdomains() + 'subdomains': domain.subdomains.all() } context.update({ 'conf_path': settings.DOMAINS_SLAVES_PATH, diff --git a/orchestra/apps/domains/forms.py b/orchestra/apps/domains/forms.py index 19e09927..fead09de 100644 --- a/orchestra/apps/domains/forms.py +++ b/orchestra/apps/domains/forms.py @@ -16,8 +16,8 @@ class DomainAdminForm(forms.ModelForm): top = domain.get_top() if not top: # Fake an account to make django validation happy - Account = self.fields['account']._queryset.model - cleaned_data['account'] = Account() + account_model = self.fields['account']._queryset.model + cleaned_data['account'] = account_model() msg = _("An account should be provided for top domain names") raise ValidationError(msg) cleaned_data['account'] = top.account @@ -37,20 +37,3 @@ class RecordInlineFormSet(forms.models.BaseInlineFormSet): records.append(data) domain = domain_for_validation(self.instance, records) validators.validate_zone(domain.render_zone()) - - -class DomainIterator(forms.models.ModelChoiceIterator): - """ Group ticket owner by superusers, ticket.group and regular users """ - def __init__(self, *args, **kwargs): - self.account = kwargs.pop('account') - self.domains = kwargs.pop('domains') - super(forms.models.ModelChoiceIterator, self).__init__(*args, **kwargs) - - def __iter__(self): - yield ('', '---------') - account_domains = self.domains.filter(account=self.account) - account_domains = account_domains.values_list('pk', 'name') - yield (_("Account"), list(account_domains)) - domains = self.domains.exclude(account=self.account) - domains = domains.values_list('pk', 'name') - yield (_("Other"), list(domains)) diff --git a/orchestra/apps/domains/helpers.py b/orchestra/apps/domains/helpers.py index 2e6eac91..14e10a6b 100644 --- a/orchestra/apps/domains/helpers.py +++ b/orchestra/apps/domains/helpers.py @@ -12,13 +12,22 @@ def domain_for_validation(instance, records): for data in records: yield Record(type=data['type'], value=data['value']) domain.get_records = get_records + + def get_top_subdomains(exclude=None): + subdomains = [] + for subdomain in Domain.objects.filter(name__endswith='.%s' % domain.origin.name): + if exclude != subdomain.pk: + subdomain.top = domain + yield subdomain + domain.get_top_subdomains = get_top_subdomains + if domain.top: - subdomains = domain.get_topsubdomains().exclude(pk=instance.pk) + subdomains = domain.get_top_subdomains(exclude=instance.pk) domain.top.get_subdomains = lambda: list(subdomains) + [domain] elif not domain.pk: subdomains = [] for subdomain in Domain.objects.filter(name__endswith=domain.name): subdomain.top = domain subdomains.append(subdomain) - domain.get_subdomains = lambda: subdomains + domain.get_subdomains = get_top_subdomains return domain diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index 571f6d78..63f41b6a 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -14,7 +14,7 @@ class Domain(models.Model): name = models.CharField(_("name"), max_length=256, unique=True, validators=[validate_hostname, validators.validate_allowed_domain]) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), - related_name='domains', blank=True) + related_name='domains', blank=True, help_text=_("Automatically selected for subdomains")) top = models.ForeignKey('domains.Domain', null=True, related_name='subdomains') serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, help_text=_("Serial number")) @@ -22,29 +22,32 @@ class Domain(models.Model): def __unicode__(self): return self.name - @cached_property + @property def origin(self): + # Do not cache return self.top or self - @cached_property + @property def is_top(self): + # Do not cache return not bool(self.top) def get_records(self): - """ proxy method, needed for input validation """ + """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.records.all() - def get_topsubdomains(self): - """ proxy method, needed for input validation """ + def get_top_subdomains(self): + """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.origin.subdomains.all() def get_subdomains(self): - return self.get_topsubdomains().filter(name__regex=r'.%s$' % self.name) + """ proxy method, needed for input validation, see helpers.domain_for_validation """ + return self.get_top_subdomains().filter(name__endswith=r'.%s' % self.name) def render_zone(self): origin = self.origin zone = origin.render_records() - for subdomain in origin.get_topsubdomains(): + for subdomain in origin.get_top_subdomains(): zone += subdomain.render_records() return zone @@ -76,7 +79,7 @@ class Domain(models.Model): records.append( AttrDict(type=record.type, ttl=record.get_ttl(), value=record.value) ) - if not self.top: + if self.is_top: if Record.NS not in types: for ns in settings.DOMAINS_DEFAULT_NS: records.append(AttrDict(type=Record.NS, value=ns)) @@ -129,7 +132,7 @@ class Domain(models.Model): for domain in domains.filter(name__endswith=self.name): domain.top = self domain.save(update_fields=['top']) - self.get_subdomains().update(account=self.account) + self.subdomains.update(account=self.account) def get_top(self): split = self.name.split('.') diff --git a/orchestra/apps/domains/serializers.py b/orchestra/apps/domains/serializers.py index 75550f38..46aee9de 100644 --- a/orchestra/apps/domains/serializers.py +++ b/orchestra/apps/domains/serializers.py @@ -37,4 +37,3 @@ class DomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSeria self._errors = { 'all': err.message } return None return instance - diff --git a/orchestra/apps/domains/settings.py b/orchestra/apps/domains/settings.py index 3c31f571..8aa33afe 100644 --- a/orchestra/apps/domains/settings.py +++ b/orchestra/apps/domains/settings.py @@ -2,10 +2,10 @@ from django.conf import settings DOMAINS_DEFAULT_NAME_SERVER = getattr(settings, 'DOMAINS_DEFAULT_NAME_SERVER', - 'ns.example.com') + 'ns.orchestra.lan') DOMAINS_DEFAULT_HOSTMASTER = getattr(settings, 'DOMAINS_DEFAULT_HOSTMASTER', - 'hostmaster@example.com') + 'hostmaster@orchestra.lan') DOMAINS_DEFAULT_TTL = getattr(settings, 'DOMAINS_DEFAULT_TTL', '1h') diff --git a/orchestra/apps/domains/tests/functional_tests/tests.py b/orchestra/apps/domains/tests/functional_tests/tests.py index 4d578f36..913cb8b8 100644 --- a/orchestra/apps/domains/tests/functional_tests/tests.py +++ b/orchestra/apps/domains/tests/functional_tests/tests.py @@ -1,11 +1,14 @@ import functools import os import time +import socket +from django.conf import settings as djsettings +from django.core.urlresolvers import reverse from selenium.webdriver.support.select import Select from orchestra.apps.orchestration.models import Server, Route -from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error from orchestra.utils.system import run from ... import settings, utils, backends @@ -16,10 +19,15 @@ run = functools.partial(run, display=False) class DomainTestMixin(object): + MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') + SLAVE_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost') + MASTER_SERVER_ADDR = socket.gethostbyname(MASTER_SERVER) + SLAVE_SERVER_ADDR = socket.gethostbyname(SLAVE_SERVER) + def setUp(self): + djsettings.DEBUG = True + settings.DOMAINS_MASTERS = [self.MASTER_SERVER_ADDR] super(DomainTestMixin, self).setUp() - self.MASTER_ADDR = os.environ['ORCHESTRA_DNS_MASTER_ADDR'] - self.SLAVE_ADDR = os.environ['ORCHESTRA_DNS_SLAVE_ADDR'] self.domain_name = 'orchestra%s.lan' % random_ascii(10) self.domain_records = ( (Record.MX, '10 mail.orchestra.lan.'), @@ -33,19 +41,19 @@ class DomainTestMixin(object): (Record.NS, 'ns1.%s.' % self.domain_name), (Record.NS, 'ns2.%s.' % self.domain_name), ) - self.subdomain1_name = 'ns1.%s' % self.domain_name - self.subdomain1_records = ( - (Record.A, '%s' % self.SLAVE_ADDR), + self.ns1_name = 'ns1.%s' % self.domain_name + self.ns1_records = ( + (Record.A, '%s' % self.SLAVE_SERVER_ADDR), ) - self.subdomain2_name = 'ns2.%s' % self.domain_name - self.subdomain2_records = ( - (Record.A, '%s' % self.MASTER_ADDR), + self.ns2_name = 'ns2.%s' % self.domain_name + self.ns2_records = ( + (Record.A, '%s' % self.MASTER_SERVER_ADDR), ) - self.subdomain3_name = 'www.%s' % self.domain_name - self.subdomain3_records = ( + self.www_name = 'www.%s' % self.domain_name + self.www_records = ( (Record.CNAME, 'external.server.org.'), ) - self.second_domain_name = 'django%s.lan' % random_ascii(10) + self.django_domain_name = 'django%s.lan' % random_ascii(10) def tearDown(self): try: @@ -173,42 +181,47 @@ class DomainTestMixin(object): self.assertEqual('external.server.org.', cname[4]) def test_add(self): - self.add(self.subdomain1_name, self.subdomain1_records) - self.add(self.subdomain2_name, self.subdomain2_records) + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) self.add(self.domain_name, self.domain_records) - self.validate_add(self.MASTER_ADDR, self.domain_name) - self.validate_add(self.SLAVE_ADDR, self.domain_name) + self.validate_add(self.MASTER_SERVER_ADDR, self.domain_name) + time.sleep(0.5) + self.validate_add(self.SLAVE_SERVER_ADDR, self.domain_name) def test_delete(self): - self.add(self.subdomain1_name, self.subdomain1_records) - self.add(self.subdomain2_name, self.subdomain2_records) + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) self.add(self.domain_name, self.domain_records) self.delete(self.domain_name) - for name in [self.domain_name, self.subdomain1_name, self.subdomain2_name]: - self.validate_delete(self.MASTER_ADDR, name) - self.validate_delete(self.SLAVE_ADDR, name) + for name in [self.domain_name, self.ns1_name, self.ns2_name]: + self.validate_delete(self.MASTER_SERVER_ADDR, name) + self.validate_delete(self.SLAVE_SERVER_ADDR, name) def test_update(self): - self.add(self.subdomain1_name, self.subdomain1_records) - self.add(self.subdomain2_name, self.subdomain2_records) + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) self.add(self.domain_name, self.domain_records) self.update(self.domain_name, self.domain_update_records) - self.add(self.subdomain3_name, self.subdomain3_records) - self.validate_update(self.MASTER_ADDR, self.domain_name) + self.add(self.www_name, self.www_records) + self.validate_update(self.MASTER_SERVER_ADDR, self.domain_name) time.sleep(5) - self.validate_update(self.SLAVE_ADDR, self.domain_name) + self.validate_update(self.SLAVE_SERVER_ADDR, self.domain_name) def test_add_add_delete_delete(self): - self.add(self.subdomain1_name, self.subdomain1_records) - self.add(self.subdomain2_name, self.subdomain2_records) + self.add(self.ns1_name, self.ns1_records) + self.add(self.ns2_name, self.ns2_records) self.add(self.domain_name, self.domain_records) - self.add(self.second_domain_name, self.domain_records) + self.add(self.django_domain_name, self.domain_records) self.delete(self.domain_name) - self.validate_add(self.MASTER_ADDR, self.second_domain_name) - self.validate_add(self.SLAVE_ADDR, self.second_domain_name) - self.delete(self.second_domain_name) - self.validate_delete(self.MASTER_ADDR, self.second_domain_name) - self.validate_delete(self.SLAVE_ADDR, self.second_domain_name) + self.validate_add(self.MASTER_SERVER_ADDR, self.django_domain_name) + self.validate_add(self.SLAVE_SERVER_ADDR, self.django_domain_name) + self.delete(self.django_domain_name) + self.validate_delete(self.MASTER_SERVER_ADDR, self.django_domain_name) + self.validate_delete(self.SLAVE_SERVER_ADDR, self.django_domain_name) + + def test_bad_creation(self): + self.assertRaises((self.rest.ResponseStatusError, AssertionError), + self.add, self.domain_name, self.domain_records) class AdminDomainMixin(DomainTestMixin): @@ -229,27 +242,38 @@ class AdminDomainMixin(DomainTestMixin): value_input.send_keys(value) return value_input + @snapshot_on_error def add(self, domain_name, records): - # TODO use reverse - url = self.live_server_url + '/admin/domains/domain/add/' + add = reverse('admin:domains_domain_add') + url = self.live_server_url + add self.selenium.get(url) + name = self.selenium.find_element_by_id('id_name') name.send_keys(domain_name) + + account_input = self.selenium.find_element_by_id('id_account') + account_select = Select(account_input) + account_select.select_by_value(str(self.account.pk)) + value_input = self._add_records(records) value_input.submit() self.assertNotEqual(url, self.selenium.current_url) + @snapshot_on_error def delete(self, domain_name): domain = Domain.objects.get(name=domain_name) - url = self.live_server_url + '/admin/domains/domain/%d/delete/' % domain.pk + delete = reverse('admin:domains_domain_delete', args=(domain.pk,)) + url = self.live_server_url + delete self.selenium.get(url) form = self.selenium.find_element_by_name('post') form.submit() self.assertNotEqual(url, self.selenium.current_url) + @snapshot_on_error def update(self, domain_name, records): domain = Domain.objects.get(name=domain_name) - url = self.live_server_url + '/admin/domains/domain/%d/' % domain.pk + change = reverse('admin:domains_domain_change', args=(domain.pk,)) + url = self.live_server_url + change self.selenium.get(url) value_input = self._add_records(records) value_input.submit() @@ -284,10 +308,10 @@ class Bind9BackendMixin(object): ) def add_route(self): - master = Server.objects.create(name=self.MASTER_ADDR) + master = Server.objects.create(name=self.MASTER_SERVER, address=self.MASTER_SERVER_ADDR) backend = backends.Bind9MasterDomainBackend.get_name() Route.objects.create(backend=backend, match=True, host=master) - slave = Server.objects.create(name=self.SLAVE_ADDR) + slave = Server.objects.create(name=self.SLAVE_SERVER, address=self.SLAVE_SERVER_ADDR) backend = backends.Bind9SlaveDomainBackend.get_name() Route.objects.create(backend=backend, match=True, host=slave) @@ -296,5 +320,5 @@ class RESTBind9BackendDomainTest(Bind9BackendMixin, RESTDomainMixin, BaseLiveSer pass -class AdminBind9BackendDomainest(Bind9BackendMixin, AdminDomainMixin, BaseLiveServerTestCase): +class AdminBind9BackendDomainTest(Bind9BackendMixin, AdminDomainMixin, BaseLiveServerTestCase): pass diff --git a/orchestra/apps/orchestration/helpers.py b/orchestra/apps/orchestration/helpers.py index c0cf370c..fb56a008 100644 --- a/orchestra/apps/orchestration/helpers.py +++ b/orchestra/apps/orchestration/helpers.py @@ -1,7 +1,9 @@ from django.contrib import messages from django.core.mail import mail_admins +from django.core.urlresolvers import reverse from django.utils.html import escape -from django.utils.translation import ugettext_lazy as _ +from django.utils.safestring import mark_safe +from django.utils.translation import ungettext, ugettext_lazy as _ def send_report(method, args, log): @@ -32,15 +34,27 @@ def send_report(method, args, log): def message_user(request, logs): - total = len(logs) - successes = [ log for log in logs if log.state == log.SUCCESS ] - successes = len(successes) + total, successes = 0, 0 + ids = [] + for log in logs: + total += 1 + ids.append(log.pk) + if log.state == log.SUCCESS: + successes += 1 errors = total-successes - if errors: - msg = 'backends have' if errors > 1 else 'backend has' - msg = _("%d out of %d {0} fail to execute".format(msg)) - messages.warning(request, msg % (errors, total)) + if total > 1: + url = reverse('admin:orchestration_backendlog_changelist') + url += '?id__in=%s' ','.join(map(str, ids)) else: - msg = 'backends have' if successes > 1 else 'backend has' - msg = _("%d {0} been successfully executed".format(msg)) - messages.success(request, msg % successes) + url = reverse('admin:orchestration_backendlog_change', args=ids) + if errors: + msg = ungettext( + _('{errors} out of {total} banckends has fail to execute.'), + _('{errors} out of {total} banckends have fail to execute.'), + errors) + else: + msg = ungettext( + _('{total} banckend has been executed.'), + _('{total} banckends have been executed.'), + total) + messages.warning(request, mark_safe(msg.format(errors=errors, total=total, url=url))) diff --git a/orchestra/apps/orchestration/middlewares.py b/orchestra/apps/orchestration/middlewares.py index 744bb162..0f8eaffc 100644 --- a/orchestra/apps/orchestration/middlewares.py +++ b/orchestra/apps/orchestration/middlewares.py @@ -1,6 +1,7 @@ import copy from threading import local +from django.core.urlresolvers import resolve from django.db.models.signals import pre_delete, post_save from django.dispatch import receiver from django.http.response import HttpResponseServerError @@ -92,6 +93,6 @@ class OperationsMiddleware(object): operations = type(self).get_pending_operations() if operations: logs = Operation.execute(operations) - if logs: + if logs and resolve(request.path).app_name == 'admin': message_user(request, logs) return response diff --git a/orchestra/apps/systemusers/backends.py b/orchestra/apps/systemusers/backends.py index 633c55fc..54708235 100644 --- a/orchestra/apps/systemusers/backends.py +++ b/orchestra/apps/systemusers/backends.py @@ -132,4 +132,3 @@ class FTPTraffic(ServiceMonitor): 'object_id': user.pk, 'username': user.username, } - diff --git a/orchestra/apps/systemusers/tests/functional_tests/tests.py b/orchestra/apps/systemusers/tests/functional_tests/tests.py index 0e8b7888..ebebf133 100644 --- a/orchestra/apps/systemusers/tests/functional_tests/tests.py +++ b/orchestra/apps/systemusers/tests/functional_tests/tests.py @@ -1,5 +1,7 @@ import ftplib +import os import re +import socket from functools import partial import paramiko @@ -10,19 +12,19 @@ from selenium.webdriver.support.select import Select from orchestra.apps.accounts.models import Account from orchestra.apps.orchestration.models import Server, Route -from orchestra.utils.system import run -from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii +from orchestra.utils.system import run, sshrun +from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error from ... import backends, settings from ...models import SystemUser r = partial(run, silent=True, display=False) +sshr = partial(sshrun, silent=True, display=False) class SystemUserMixin(object): - MASTER_ADDR = 'localhost' - ACCOUNT_USERNAME = '%s_account' % random_ascii(10) + MASTER_SERVER = os.environ.get('ORCHESTRA_MASTER_SERVER', 'localhost') DEPENDENCIES = ( 'orchestra.apps.orchestration', 'orcgestra.apps.systemusers', @@ -34,7 +36,7 @@ class SystemUserMixin(object): djsettings.DEBUG = True def add_route(self): - master = Server.objects.create(name=self.MASTER_ADDR) + master = Server.objects.create(name=self.MASTER_SERVER) backend = backends.SystemUserBackend.get_name() Route.objects.create(backend=backend, match=True, host=master) @@ -57,7 +59,7 @@ class SystemUserMixin(object): raise NotImplementedError def validate_user(self, username): - idcmd = r("id %s" % username) + idcmd = sshr(self.MASTER_SERVER, "id %s" % username) self.assertEqual(0, idcmd.return_code) user = SystemUser.objects.get(username=username) groups = list(user.groups.values_list('username', flat=True)) @@ -68,18 +70,22 @@ class SystemUserMixin(object): def validate_delete(self, username): self.assertRaises(SystemUser.DoesNotExist, SystemUser.objects.get, username=username) - self.assertRaises(CommandError, run, 'id %s' % username, display=False) - self.assertRaises(CommandError, run, 'grep "^%s:" /etc/groups' % username, display=False) - self.assertRaises(CommandError, run, 'grep "^%s:" /etc/passwd' % username, display=False) - self.assertRaises(CommandError, run, 'grep "^%s:" /etc/shadow' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER,'id %s' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/groups' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/passwd' % username, display=False) + self.assertRaises(CommandError, + sshrun, self.MASTER_SERVER, 'grep "^%s:" /etc/shadow' % username, display=False) def validate_ftp(self, username, password): - connection = ftplib.FTP(self.MASTER_ADDR) + connection = ftplib.FTP(self.MASTER_SERVER) connection.login(user=username, passwd=password) connection.close() def validate_sftp(self, username, password): - transport = paramiko.Transport((self.MASTER_ADDR, 22)) + transport = paramiko.Transport((self.MASTER_SERVER, 22)) transport.connect(username=username, password=password) sftp = paramiko.SFTPClient.from_transport(transport) sftp.listdir() @@ -88,14 +94,14 @@ class SystemUserMixin(object): def validate_ssh(self, username, password): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(self.MASTER_ADDR, username=username, password=password) + ssh.connect(self.MASTER_SERVER, username=username, password=password) transport = ssh.get_transport() channel = transport.open_session() channel.exec_command('ls') self.assertEqual(0, channel.recv_exit_status()) channel.close() - def test_create_systemuser(self): + def test_create(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) self.add(username, password) @@ -125,7 +131,7 @@ class SystemUserMixin(object): self.addCleanup(partial(self.delete, username)) self.validate_ssh(username, password) - def test_delete_systemuser(self): + def test_delete(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%sppppP001' % random_ascii(5) self.add(username, password) @@ -133,7 +139,7 @@ class SystemUserMixin(object): self.delete(username) self.validate_delete(username) - def test_add_group_systemuser(self): + def test_add_group(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) self.add(username, password) @@ -150,7 +156,7 @@ class SystemUserMixin(object): self.assertIn(username2, groups) self.validate_user(username) - def test_disable_systemuser(self): + def test_disable(self): username = '%s_systemuser' % random_ascii(10) password = '@!?%spppP001' % random_ascii(5) self.add(username, password, shell='/dev/null') @@ -159,6 +165,10 @@ class SystemUserMixin(object): self.disable(username) self.validate_user(username) self.assertRaises(ftplib.error_perm, self.validate_ftp, username, password) + + def test_change_password(self): + pass + # TODO class RESTSystemUserMixin(SystemUserMixin): @@ -200,6 +210,7 @@ class AdminSystemUserMixin(SystemUserMixin): self.save(self.account.username) self.addCleanup(partial(self.delete, self.account.username)) + @snapshot_on_error def add(self, username, password, shell='/dev/null'): url = self.live_server_url + reverse('admin:systemusers_systemuser_add') self.selenium.get(url) @@ -223,6 +234,7 @@ class AdminSystemUserMixin(SystemUserMixin): username_field.submit() self.assertNotEqual(url, self.selenium.current_url) + @snapshot_on_error def delete(self, username): user = SystemUser.objects.get(username=username) delete = reverse('admin:systemusers_systemuser_delete', args=(user.pk,)) @@ -232,6 +244,7 @@ class AdminSystemUserMixin(SystemUserMixin): confirmation.submit() self.assertNotEqual(url, self.selenium.current_url) + @snapshot_on_error def disable(self, username): user = SystemUser.objects.get(username=username) change = reverse('admin:systemusers_systemuser_change', args=(user.pk,)) @@ -243,6 +256,7 @@ class AdminSystemUserMixin(SystemUserMixin): save.submit() self.assertNotEqual(url, self.selenium.current_url) + @snapshot_on_error def add_group(self, username, groupname): user = SystemUser.objects.get(username=username) change = reverse('admin:systemusers_systemuser_change', args=(user.pk,)) @@ -254,6 +268,7 @@ class AdminSystemUserMixin(SystemUserMixin): save.submit() self.assertNotEqual(url, self.selenium.current_url) + @snapshot_on_error def save(self, username): user = SystemUser.objects.get(username=username) change = reverse('admin:systemusers_systemuser_change', args=(user.pk,)) @@ -269,6 +284,7 @@ class RESTSystemUserTest(RESTSystemUserMixin, BaseLiveServerTestCase): class AdminSystemUserTest(AdminSystemUserMixin, BaseLiveServerTestCase): + @snapshot_on_error def test_create_account(self): url = self.live_server_url + reverse('admin:accounts_account_add') self.selenium.get(url) @@ -298,8 +314,9 @@ class AdminSystemUserTest(AdminSystemUserMixin, BaseLiveServerTestCase): account = Account.objects.get(username=account_username) self.addCleanup(account.delete) self.assertNotEqual(url, self.selenium.current_url) - self.assertEqual(0, r("id %s" % account.username).return_code) + self.assertEqual(0, sshr(self.MASTER_SERVER, "id %s" % account.username).return_code) + @snapshot_on_error def test_delete_account(self): home = self.account.systemusers.get(is_main=True).get_home() diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index 9b4c44dc..8e758d88 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -158,7 +158,8 @@ function install_requirements () { PIP="${PIP} \ selenium \ xvfbwrapper \ - freezegun" + freezegun \ + coverage" fi # Make sure locales are in place before installing postgres diff --git a/orchestra/utils/system.py b/orchestra/utils/system.py index 121a5fa9..d22ae609 100644 --- a/orchestra/utils/system.py +++ b/orchestra/utils/system.py @@ -105,6 +105,11 @@ def run(command, display=True, error_codes=[0], silent=False, stdin=''): return out +def sshrun(addr, command, *args, **kwargs): + cmd = "ssh -o stricthostkeychecking=no root@%s -C '%s'" % (addr, command) + return run(cmd, *args, **kwargs) + + def get_default_celeryd_username(): """ Introspect celeryd defaults file in order to get its username """ user = None diff --git a/orchestra/utils/tests.py b/orchestra/utils/tests.py index 64060a88..a89dfffb 100644 --- a/orchestra/utils/tests.py +++ b/orchestra/utils/tests.py @@ -1,5 +1,7 @@ +import datetime import string import random +from functools import wraps from django.conf import settings from django.contrib.auth import BACKEND_SESSION_KEY, SESSION_KEY, get_user_model @@ -105,3 +107,17 @@ class BaseLiveServerTestCase(AppDependencyMixin, LiveServerTestCase): def rest_login(self): self.rest.login(username=self.account.username, password=self.account_password) + + +def snapshot_on_error(test): + @wraps(test) + def inner(*args, **kwargs): + try: + test(*args, **kwargs) + except: + self = args[0] + timestamp = datetime.datetime.now().isoformat().replace(':', '') + filename = '/tmp/screenshot_%s_%s.png' % (self.id(), timestamp) + self.selenium.save_screenshot(filename) + raise + return inner diff --git a/scripts/services/bind9.md b/scripts/services/bind9.md new file mode 100644 index 00000000..723887f1 --- /dev/null +++ b/scripts/services/bind9.md @@ -0,0 +1,24 @@ +Bind9 Master and Slave +====================== + +1. Install bind9 service as well as some convinient utilities on master and slave servers + ```bash + apt-get update + apt-get install bind9 dnsutils + ``` + +2. create the zone directory on the master server + ```bash + mkdir /etc/bind/master + chown bind.bind /etc/bind/master + ``` + +2. Allow zone transfer on master by adding the following line to `named.conf.options` + ```bash + allow-transfer { slave-ip; }; + ``` + +3. Addlow notifications on the slave server by adding the following line to `named.conf.options` + ```bash + allow-notify { master-ip; }; + ``` diff --git a/scripts/services/bind9.sh b/scripts/services/bind9.sh deleted file mode 100644 index 9aeb303a..00000000 --- a/scripts/services/bind9.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Installs and confingures bind9 to work with Orchestra - - -apt-get update -apt-get install bind9 - -echo "nameserver 127.0.0.1" > /etc/resolv.conf diff --git a/scripts/services/postfix.md b/scripts/services/postfix.md index 51eae45d..90c94538 100644 --- a/scripts/services/postfix.md +++ b/scripts/services/postfix.md @@ -9,6 +9,15 @@ apt-get install postfix apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve +sed -i "s#^mail_location = mbox.*#mail_location = maildir:~/Maildir#" /etc/dovecot/conf.d/10-mail.conf +echo 'auth_username_format = %n' >> /etc/dovecot/conf.d/10-auth.conf +echo 'service lmtp { + unix_listener /var/spool/postfix/private/dovecot-lmtp { + group = postfix + mode = 0600 + user = postfix + } +}' >> /etc/dovecot/conf.d/10-master.conf cat > /etc/apt/sources.list.d/mailscanner.list << 'EOF' @@ -18,16 +27,17 @@ EOF wget -O - http://apt.baruwa.org/baruwa-apt-keys.gpg | apt-key add - - apt-get update apt-get install mailscanner - -apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-sieve apt-get install postfix +echo 'home_mailbox = Maildir/' >> /etc/postfix/main.cf +echo 'mailbox_transport = lmtp:unix:private/dovecot-lmtp' >> /etc/postfix/main.cf -mail_location = maildir:~/Maildir + +/etc/init.d/dovecot restart +/etc/init.d/postfix restart diff --git a/scripts/services/rssh.md b/scripts/services/rssh.md index 9cc2065c..9ff2be8b 100644 --- a/scripts/services/rssh.md +++ b/scripts/services/rssh.md @@ -12,6 +12,6 @@ Restricted Shell for SCP and Rsync 2. Enable the shell ```bash - ln -s /usr/local/bin/rssh /bin/rssh + ln -s /usr/bin/rssh /bin/rssh echo /bin/rssh >> /etc/shells ``` diff --git a/scripts/services/vsftpd.md b/scripts/services/vsftpd.md index 3c3b0d6b..0267c54e 100644 --- a/scripts/services/vsftpd.md +++ b/scripts/services/vsftpd.md @@ -12,9 +12,9 @@ VsFTPd with System Users ```bash sed -i "s/anonymous_enable=YES/anonymous_enable=NO/" /etc/vsftpd.conf sed -i "s/#local_enable=YES/local_enable=YES/" /etc/vsftpd.conf - sed -i "s/#write_enable=YES/write_enable=YES" /etc/vsftpd.conf - sed -i "s/#chroot_local_user=YES/chroot_local_user=YES/" /etc/vsftpd.conf - + sed -i "s/#write_enable=YES/write_enable=YES/" /etc/vsftpd.conf + # sed -i "s/#chroot_local_user=YES/chroot_local_user=YES/" /etc/vsftpd.conf + echo '/dev/null' >> /etc/shells ```