From 6a3e3f637c5159538263aa6347a24fe278f9ef13 Mon Sep 17 00:00:00 2001 From: Marc Date: Fri, 3 Oct 2014 17:37:36 +0000 Subject: [PATCH] Removed slaves domain settings, now generated on the fly --- ROADMAP.md | 4 +- orchestra/apps/domains/backends.py | 45 +++++++++++++------ orchestra/apps/domains/models.py | 3 +- orchestra/apps/domains/settings.py | 15 ++++++- .../domains/tests/functional_tests/tests.py | 1 - orchestra/apps/domains/tests/test_domains.py | 17 +++---- orchestra/apps/orchestration/admin.py | 6 +-- orchestra/apps/orchestration/helpers.py | 3 +- orchestra/apps/orchestration/models.py | 13 ++++++ .../tests/functional_tests/tests.py | 1 - orchestra/bin/orchestra-admin | 3 +- orchestra/core/validators.py | 8 ++++ scripts/services/bind9.md | 10 ----- scripts/services/{mysql.sh => mysql.md} | 3 ++ 14 files changed, 88 insertions(+), 44 deletions(-) rename scripts/services/{mysql.sh => mysql.md} (69%) diff --git a/ROADMAP.md b/ROADMAP.md index 86dcd0a0..e108b261 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -15,7 +15,7 @@ Note `*` _for sustancial progress_ 4. [ ] Data model, crazy input validation, admin and REST interfaces, permissions, unit and functional tests, service management, migration scripts and documentation of: 1. [x] PHP/static Web applications 1. [x] Websites with Apache - 2. [ ] *FTP/rsync/scp/shell system accounts + 2. [x] FTP/rsync/scp/shell system accounts 2. [ ] *Databases and database users with MySQL 1. [ ] *Mail accounts, aliases, forwards with Postfix and Dovecot 1. [x] DNS with Bind @@ -63,4 +63,4 @@ Note `*` _for sustancial progress_ 2. [ ] REST API functionality for superusers 3. [ ] Responsive user interface, based on a JS framework. 4. [ ] Full documentation -5. [ ] [http://www.ansible.com/home](Ansible) orchestration method, which synchronize the whole service config everytime instead of incremental changes. +5. [ ] [Ansible](http://www.ansible.com/home) orchestration method, which synchronizes the whole service config everytime instead of incremental changes. diff --git a/orchestra/apps/domains/backends.py b/orchestra/apps/domains/backends.py index 23f80a5e..51eb58d9 100644 --- a/orchestra/apps/domains/backends.py +++ b/orchestra/apps/domains/backends.py @@ -1,9 +1,11 @@ +import textwrap + from django.utils.translation import ugettext_lazy as _ -from . import settings - from orchestra.apps.orchestration import ServiceController +from . import settings + class Bind9MasterDomainBackend(ServiceController): verbose_name = _("Bind9 master domain") @@ -52,20 +54,31 @@ class Bind9MasterDomainBackend(ServiceController): """ reload bind if needed """ self.append('[[ $UPDATED == 1 ]] && service bind9 reload') + def get_servers(self, domain, backend): + from orchestra.apps.orchestration.models import Route, BackendOperation as Operation + operation = Operation(backend=backend, action='save', instance=domain) + servers = [] + for server in Route.get_servers(operation): + servers.append(server.get_ip()) + return servers + def get_context(self, domain): context = { 'name': domain.name, 'zone_path': settings.DOMAINS_ZONE_PATH % {'name': domain.name}, 'subdomains': domain.subdomains.all(), 'banner': self.get_banner(), + 'slaves': '; '.join(self.get_servers(domain, Bind9SlaveDomainBackend)), } context.update({ 'conf_path': settings.DOMAINS_MASTERS_PATH, - 'conf': 'zone "%(name)s" {\n' - ' // %(banner)s\n' - ' type master;\n' - ' file "%(zone_path)s";\n' - '};\n' % context + 'conf': textwrap.dedent(""" + zone "%(name)s" { + // %(banner)s + type master; + file "%(zone_path)s"; + allow-transfer { %(slaves)s; }; + };""" % context) }) return context @@ -91,15 +104,19 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): def get_context(self, domain): context = { 'name': domain.name, - 'masters': '; '.join(settings.DOMAINS_MASTERS), - 'subdomains': domain.subdomains.all() + 'banner': self.get_banner(), + 'subdomains': domain.subdomains.all(), + 'masters': '; '.join(self.get_servers(domain, Bind9MasterDomainBackend)), } context.update({ 'conf_path': settings.DOMAINS_SLAVES_PATH, - 'conf': 'zone "%(name)s" {\n' - ' type slave;\n' - ' file "%(name)s";\n' - ' masters { %(masters)s; };\n' - '};\n' % context + 'conf': textwrap.dedent(""" + zone "%(name)s" { + // %(banner)s + type slave; + file "%(name)s"; + masters { %(masters)s; }; + allow-notify { %(masters)s; }; + };""" % context) }) return context diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index 63f41b6a..7cf1947c 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -124,6 +124,7 @@ class Domain(models.Model): top = self.get_top() if top: self.top = top + self.account_id = self.account_id or top.account_id else: update = True super(Domain, self).save(*args, **kwargs) @@ -132,7 +133,7 @@ class Domain(models.Model): for domain in domains.filter(name__endswith=self.name): domain.top = self domain.save(update_fields=['top']) - self.subdomains.update(account=self.account) + self.subdomains.update(account_id=self.account_id) def get_top(self): split = self.name.split('.') diff --git a/orchestra/apps/domains/settings.py b/orchestra/apps/domains/settings.py index 8aa33afe..b4dcf916 100644 --- a/orchestra/apps/domains/settings.py +++ b/orchestra/apps/domains/settings.py @@ -4,48 +4,61 @@ from django.conf import settings DOMAINS_DEFAULT_NAME_SERVER = getattr(settings, 'DOMAINS_DEFAULT_NAME_SERVER', 'ns.orchestra.lan') + DOMAINS_DEFAULT_HOSTMASTER = getattr(settings, 'DOMAINS_DEFAULT_HOSTMASTER', 'hostmaster@orchestra.lan') + DOMAINS_DEFAULT_TTL = getattr(settings, 'DOMAINS_DEFAULT_TTL', '1h') + DOMAINS_DEFAULT_REFRESH = getattr(settings, 'DOMAINS_DEFAULT_REFRESH', '1d') + DOMAINS_DEFAULT_RETRY = getattr(settings, 'DOMAINS_DEFAULT_RETRY', '2h') + DOMAINS_DEFAULT_EXPIRATION = getattr(settings, 'DOMAINS_DEFAULT_EXPIRATION', '4w') + DOMAINS_DEFAULT_MIN_CACHING_TIME = getattr(settings, 'DOMAINS_DEFAULT_MIN_CACHING_TIME', '1h') + DOMAINS_ZONE_PATH = getattr(settings, 'DOMAINS_ZONE_PATH', '/etc/bind/master/%(name)s') + DOMAINS_MASTERS_PATH = getattr(settings, 'DOMAINS_MASTERS_PATH', '/etc/bind/named.conf.local') + DOMAINS_SLAVES_PATH = getattr(settings, 'DOMAINS_SLAVES_PATH', '/etc/bind/named.conf.local') -DOMAINS_MASTERS = getattr(settings, 'DOMAINS_MASTERS', ['10.0.3.13']) DOMAINS_CHECKZONE_BIN_PATH = getattr(settings, 'DOMAINS_CHECKZONE_BIN_PATH', '/usr/sbin/named-checkzone -i local') DOMAINS_CHECKZONE_PATH = getattr(settings, 'DOMAINS_CHECKZONE_PATH', '/dev/shm') + DOMAINS_DEFAULT_A = getattr(settings, 'DOMAINS_DEFAULT_A', '10.0.3.13') + DOMAINS_DEFAULT_MX = getattr(settings, 'DOMAINS_DEFAULT_MX', ( '10 mail.orchestra.lan.', '10 mail2.orchestra.lan.', )) + DOMAINS_DEFAULT_NS = getattr(settings, 'DOMAINS_DEFAULT_NS', ( 'ns1.orchestra.lan.', 'ns2.orchestra.lan.', )) + DOMAINS_FORBIDDEN = getattr(settings, 'DOMAINS_FORBIDDEN', # This setting prevents users from providing random domain names, i.e. google.com # You can generate a 5K forbidden domains list from Alexa's top 1M # wget http://s3.amazonaws.com/alexa-static/top-1m.csv.zip -O /tmp/top-1m.csv.zip # unzip -p /tmp/top-1m.csv.zip | head -n 5000 | sed "s/^.*,//" > forbidden_domains.list + # '%(site_root)s/forbidden_domains.list') '') diff --git a/orchestra/apps/domains/tests/functional_tests/tests.py b/orchestra/apps/domains/tests/functional_tests/tests.py index 913cb8b8..dae75954 100644 --- a/orchestra/apps/domains/tests/functional_tests/tests.py +++ b/orchestra/apps/domains/tests/functional_tests/tests.py @@ -26,7 +26,6 @@ class DomainTestMixin(object): def setUp(self): djsettings.DEBUG = True - settings.DOMAINS_MASTERS = [self.MASTER_SERVER_ADDR] super(DomainTestMixin, self).setUp() self.domain_name = 'orchestra%s.lan' % random_ascii(10) self.domain_records = ( diff --git a/orchestra/apps/domains/tests/test_domains.py b/orchestra/apps/domains/tests/test_domains.py index 82acc3ac..f15ac2d3 100644 --- a/orchestra/apps/domains/tests/test_domains.py +++ b/orchestra/apps/domains/tests/test_domains.py @@ -1,17 +1,18 @@ -from django.test import TestCase +from orchestra.utils.tests import BaseTestCase from ..models import Domain -class DomainTests(TestCase): - def setUp(self): - self.domain = Domain.objects.create(name='rostrepalid.org') +class DomainTest(BaseTestCase): + def test_top_relation(self): + account = self.create_account() + domain = Domain.objects.create(name='rostrepalid.org', account=account) Domain.objects.create(name='www.rostrepalid.org') Domain.objects.create(name='mail.rostrepalid.org') - - def test_top_relation(self): - self.assertEqual(2, len(self.domain.subdomains.all())) + self.assertEqual(2, len(domain.subdomains.all())) def test_render_zone(self): - print self.domain.render_zone() + account = self.create_account() + domain = Domain.objects.create(name='rostrepalid.org', account=account) + domain.render_zone() diff --git a/orchestra/apps/orchestration/admin.py b/orchestra/apps/orchestration/admin.py index 862f12d4..fdbbffb6 100644 --- a/orchestra/apps/orchestration/admin.py +++ b/orchestra/apps/orchestration/admin.py @@ -88,18 +88,16 @@ class BackendLogAdmin(admin.ModelAdmin): ) list_display_links = ('id', 'backend') list_filter = ('state', 'backend') - date_hierarchy = 'updated_at' inlines = [BackendOperationInline] fields = [ 'backend', 'server_link', 'state', 'mono_script', 'mono_stdout', 'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created', - 'display_updated', 'execution_time' + 'execution_time' ] readonly_fields = fields server_link = admin_link('server') - display_updated = admin_date('updated_at') - display_created = admin_date('created_at') + display_created = admin_date('created_at', short_description=_("Created")) display_state = admin_colored('state', colors=STATE_COLORS) mono_script = display_mono('script') mono_stdout = display_mono('stdout') diff --git a/orchestra/apps/orchestration/helpers.py b/orchestra/apps/orchestration/helpers.py index fb56a008..01efb6b0 100644 --- a/orchestra/apps/orchestration/helpers.py +++ b/orchestra/apps/orchestration/helpers.py @@ -52,9 +52,10 @@ def message_user(request, logs): _('{errors} out of {total} banckends has fail to execute.'), _('{errors} out of {total} banckends have fail to execute.'), errors) + messages.error(request, mark_safe(msg.format(errors=errors, total=total, url=url))) else: msg = ungettext( _('{total} banckend has been executed.'), _('{total} banckends have been executed.'), total) - messages.warning(request, mark_safe(msg.format(errors=errors, total=total, url=url))) + messages.success(request, mark_safe(msg.format(total=total, url=url))) diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index bfff6ce4..cedcb492 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -1,8 +1,11 @@ +import socket + from django.contrib.contenttypes import generic from django.contrib.contenttypes.models import ContentType from django.db import models from django.utils.translation import ugettext_lazy as _ +from orchestra.core.validators import validate_ip_address, ValidationError from orchestra.models.fields import NullableCharField from orchestra.utils.apps import autodiscover @@ -27,6 +30,16 @@ class Server(models.Model): if self.address: return self.address return self.name + + def get_ip(self): + if self.address: + return self.address + try: + validate_ip_address(self.name) + except ValidationError: + return socket.gethostbyname(self.name) + else: + return self.name class BackendLog(models.Model): diff --git a/orchestra/apps/systemusers/tests/functional_tests/tests.py b/orchestra/apps/systemusers/tests/functional_tests/tests.py index ebebf133..4cbd1b97 100644 --- a/orchestra/apps/systemusers/tests/functional_tests/tests.py +++ b/orchestra/apps/systemusers/tests/functional_tests/tests.py @@ -1,7 +1,6 @@ import ftplib import os import re -import socket from functools import partial import paramiko diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index 8e758d88..a4cb0973 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -159,7 +159,8 @@ function install_requirements () { selenium \ xvfbwrapper \ freezegun \ - coverage" + coverage \ + orchestra-orm==dev" fi # Make sure locales are in place before installing postgres diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index ed1b7ca3..584387dd 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -29,6 +29,14 @@ def validate_ipv6_address(value): raise ValidationError(msg) +def validate_ip_address(value): + msg = _("%s is not a valid IP address") % value + try: + ip = IP(value) + except: + raise ValidationError(msg) + + def validate_name(value): """ A single non-empty line of free-form text with no whitespace. diff --git a/scripts/services/bind9.md b/scripts/services/bind9.md index 723887f1..99dbacb8 100644 --- a/scripts/services/bind9.md +++ b/scripts/services/bind9.md @@ -12,13 +12,3 @@ Bind9 Master and Slave mkdir /etc/bind/master chown bind.bind /etc/bind/master ``` - -2. Allow zone transfer on master by adding the following line to `named.conf.options` - ```bash - allow-transfer { slave-ip; }; - ``` - -3. Addlow notifications on the slave server by adding the following line to `named.conf.options` - ```bash - allow-notify { master-ip; }; - ``` diff --git a/scripts/services/mysql.sh b/scripts/services/mysql.md similarity index 69% rename from scripts/services/mysql.sh rename to scripts/services/mysql.md index 71bfee8c..efc74d0a 100644 --- a/scripts/services/mysql.sh +++ b/scripts/services/mysql.md @@ -1 +1,4 @@ +MySQL +===== + apt-get install mysql-server