From e4f1673de8b8590fe49897ebaeef3f1a23f987a4 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Tue, 5 May 2015 19:42:55 +0000 Subject: [PATCH] Fixes on mailboxes, tasks, and mailer --- TODO.md | 19 +++ orchestra/admin/actions.py | 8 +- orchestra/admin/menu.py | 20 +-- orchestra/bin/orchestra-beat | 152 +++++++++++++++++ orchestra/contrib/domains/backends.py | 23 ++- orchestra/contrib/domains/helpers.py | 4 +- orchestra/contrib/domains/models.py | 21 ++- orchestra/contrib/domains/utils.py | 26 +++ orchestra/contrib/lists/backends.py | 84 ++++++--- orchestra/contrib/mailboxes/actions.py | 8 +- orchestra/contrib/mailboxes/admin.py | 7 +- orchestra/contrib/mailboxes/backends.py | 161 +++++++++++------- orchestra/contrib/mailer/__init__.py | 1 + orchestra/contrib/mailer/admin.py | 41 ++++- orchestra/contrib/mailer/apps.py | 12 ++ orchestra/contrib/mailer/backends.py | 30 ++-- orchestra/contrib/mailer/engine.py | 4 +- orchestra/contrib/mailer/models.py | 19 ++- orchestra/contrib/mailer/settings.py | 5 + orchestra/contrib/mailer/tasks.py | 20 ++- orchestra/contrib/orchestration/admin.py | 1 + orchestra/contrib/orchestration/backends.py | 40 +++-- orchestra/contrib/orchestration/helpers.py | 2 +- orchestra/contrib/orchestration/manager.py | 14 +- orchestra/contrib/orchestration/models.py | 4 +- orchestra/contrib/orchestration/signals.py | 8 + orchestra/contrib/orchestration/tasks.py | 2 +- orchestra/contrib/tasks/bin/orchestra-beat | 84 --------- orchestra/contrib/tasks/decorators.py | 6 +- orchestra/core/__init__.py | 1 + .../management/commands/setupcronbeat.py | 5 +- orchestra/utils/db.py | 30 ---- orchestra/utils/sys.py | 10 ++ scripts/container/deploy.sh | 14 +- setup.py | 2 +- 35 files changed, 602 insertions(+), 286 deletions(-) create mode 100755 orchestra/bin/orchestra-beat create mode 100644 orchestra/contrib/mailer/apps.py delete mode 100755 orchestra/contrib/tasks/bin/orchestra-beat rename orchestra/{contrib/tasks => }/management/commands/setupcronbeat.py (84%) diff --git a/TODO.md b/TODO.md index 5e708b48..dc021233 100644 --- a/TODO.md +++ b/TODO.md @@ -358,3 +358,22 @@ TODO mount the filesystem with "nosuid" option # autoiscover modules on app.ready() # uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask +# SecondaryMailServerBackend and check_origin signal +try: import uwsgi to know its running uwsgi +# avoid cron email errors when failing hard + + +# mailboxes.address settings multiple local domains, not only one? +# backend.context = self.get_context() or save(obj, context=None) + +# smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209') + +# create registered periodic_task on beat execution: and management command: syncperiodictasks + +# MERGE beats and inspect INSTALLED_APPS and get IS_ENABLED + +# make exceptions fot check origin shit + +# rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ? + +# Message last_retry auto_now doesn't work! diff --git a/orchestra/admin/actions.py b/orchestra/admin/actions.py index cbae0579..cbdac596 100644 --- a/orchestra/admin/actions.py +++ b/orchestra/admin/actions.py @@ -37,7 +37,7 @@ class SendEmail(object): raise PermissionDenied initial={ 'email_from': self.default_from, - 'to': ' '.join(self.get_queryset_emails()) + 'to': ' '.join(self.get_email_addresses()) } form = self.form(initial=initial) if request.POST.get('post'): @@ -62,7 +62,7 @@ class SendEmail(object): # Display confirmation page return render(request, self.template, self.context) - def get_queryset_emails(self): + def get_email_addresses(self): return self.queryset.values_list('email', flat=True) def confirm_email(self, request, **options): @@ -74,7 +74,7 @@ class SendEmail(object): if request.POST.get('post') == 'email_confirmation': emails = [] num = 0 - for email in self.get_queryset_emails(): + for email in self.get_email_addresses(): emails.append((subject, message, email_from, [email])) num += 1 if extra_to: @@ -99,7 +99,7 @@ class SendEmail(object): 'content_message': _( "Are you sure you want to send the following message to the following %s?" ) % self.opts.verbose_name_plural, - 'display_objects': ["%s (%s)" % (contact, contact.email) for contact in self.queryset], + 'display_objects': ["%s (%s)" % (contact, email) for contact, email in zip(self.queryset, self.get_email_addresses())], 'form': form, 'subject': subject, 'message': message, diff --git a/orchestra/admin/menu.py b/orchestra/admin/menu.py index c5492253..27d3b4df 100644 --- a/orchestra/admin/menu.py +++ b/orchestra/admin/menu.py @@ -3,7 +3,7 @@ from django.core.urlresolvers import reverse from django.utils.text import capfirst from django.utils.translation import ugettext_lazy as _ -from orchestra.core import services, accounts +from orchestra.core import services, accounts, administration from orchestra.utils.apps import isinstalled @@ -27,15 +27,20 @@ def api_link(context): return reverse('api-root') -def get_services(): +def process_registered_models(register): childrens = [] - for model, options in services.get().items(): + for model, options in register.get().items(): if options.get('menu', True): opts = model._meta url = reverse('admin:{}_{}_changelist'.format( opts.app_label, opts.model_name)) name = capfirst(options.get('verbose_name_plural')) childrens.append(items.MenuItem(name, url)) + return childrens + + +def get_services(): + childrens = process_registered_models(services) return sorted(childrens, key=lambda i: i.title) @@ -47,13 +52,7 @@ def get_accounts(): if isinstalled('orchestra.contrib.issues'): url = reverse('admin:issues_ticket_changelist') childrens.append(items.MenuItem(_("Tickets"), url)) - for model, options in accounts.get().items(): - if options.get('menu', True): - opts = model._meta - url = reverse('admin:{}_{}_changelist'.format( - opts.app_label, opts.model_name)) - name = capfirst(options.get('verbose_name_plural')) - childrens.append(items.MenuItem(name, url)) + childrens.extend(process_registered_models(accounts)) return sorted(childrens, key=lambda i: i.title) @@ -100,6 +99,7 @@ def get_administration_items(): items.MenuItem(_("Periodic tasks"), periodic), items.MenuItem(_("Workers"), worker), ])) + childrens.extend(process_registered_models(administration)) return childrens diff --git a/orchestra/bin/orchestra-beat b/orchestra/bin/orchestra-beat new file mode 100755 index 00000000..5bed7c52 --- /dev/null +++ b/orchestra/bin/orchestra-beat @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +# High performance alternative to beat management command +# Looks for pending work before firing up all the Django machinery on separate processes +# +# Handles orchestra.contrib.tasks periodic_tasks and orchestra.contrib.mailer queued mails +# +# USAGE: beat /path/to/project/manage.py + + +import json +import os +import re +import sys +from datetime import datetime, timedelta + +from celery.schedules import crontab_parser as CrontabParser + +from orchestra.utils.sys import run, join + + +class Setting(object): + def __init__(self, manage): + self.manage = manage + self.settings_file = self.get_settings_file(manage) + + def get_settings(self): + """ get db settings from settings.py file without importing """ + settings = {'__file__': self.settings_file} + with open(self.settings_file) as f: + __file__ = 'rata' + exec(f.read(), settings) + return settings + + def get_settings_file(self, manage): + with open(manage, 'r') as handler: + regex = re.compile(r'"DJANGO_SETTINGS_MODULE"\s*,\s*"([^"]+)"') + for line in handler.readlines(): + match = regex.search(line) + if match: + settings_module = match.groups()[0] + settings_file = os.path.join(*settings_module.split('.')) + '.py' + settings_file = os.path.join(os.path.dirname(manage), settings_file) + return settings_file + raise ValueError("settings module not found in %s" % manage) + + +class DB(object): + def __init__(self, settings): + self.settings = settings['DATABASES']['default'] + + def connect(self): + if self.settings['ENGINE'] == 'django.db.backends.sqlite3': + import sqlite3 + self.conn = sqlite3.connect(self.settings['NAME']) + elif self.settings['ENGINE'] == 'django.db.backends.postgresql_psycopg2': + import psycopg2 + self.conn = psycopg2.connect("dbname='{NAME}' user='{USER}' host='{HOST}' password='{PASSWORD}'".format(**self.settings)) + else: + raise ValueError("%s engine not supported." % self.settings['ENGINE']) + + def query(self, query): + cur = self.conn.cursor() + try: + cur.execute(query) + result = cur.fetchall() + finally: + cur.close() + return result + + def close(self): + self.conn.close() + + +def fire_pending_tasks(manage, db): + def get_tasks(db): + enabled = 1 if 'sqlite' in db.settings['ENGINE'] else True + query = ( + "SELECT c.minute, c.hour, c.day_of_week, c.day_of_month, c.month_of_year, p.id " + "FROM djcelery_periodictask as p, djcelery_crontabschedule as c " + "WHERE p.crontab_id = c.id AND p.enabled = {}" + ).format(enabled) + return db.query(query) + + def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): + n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now + return ( + n_minute in CrontabParser(60).parse(minute) and + n_hour in CrontabParser(24).parse(hour) and + n_day_of_week in CrontabParser(7).parse(day_of_week) and + n_day_of_month in CrontabParser(31, 1).parse(day_of_month) and + n_month_of_year in CrontabParser(12, 1).parse(month_of_year) + ) + + now = datetime.utcnow() + now = tuple(map(int, now.strftime("%M %H %w %d %m").split())) + for minute, hour, day_of_week, day_of_month, month_of_year, task_id in get_tasks(db): + if is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): + command = 'python3 -W ignore::DeprecationWarning {manage} runtask {task_id}'.format( + manage=manage, task_id=task_id) + proc = run(command, async=True) + yield proc + + +def fire_pending_messages(settings, db): + def has_pending_messages(settings, db): + MAILER_DEFERE_SECONDS = settings.get('MAILER_DEFERE_SECONDS', (300, 600, 60*60, 60*60*24)) + now = datetime.utcnow() + query_or = [] + + for num, seconds in enumerate(MAILER_DEFERE_SECONDS): + delta = timedelta(seconds=seconds) + epoch = now-delta + query_or.append("""(mailer_message.retries = 0 AND mailer_message.last_retry <= '%s')""" + % epoch.isoformat().replace('T', ' ')) + query = """\ + SELECT 1 FROM mailer_message + WHERE (mailer_message.state = 'QUEUED' + OR (mailer_message.state = 'DEFERRED' AND (%s))) LIMIT 1""" % ' OR '.join(query_or) + return bool(db.query(query)) + + if has_pending_messages(settings, db): + command = 'python3 -W ignore::DeprecationWarning {manage} send'.format(manage=manage) + proc = run(command, async=True) + yield proc + + +if __name__ == "__main__": + # TODO aquire lock + manage = sys.argv[1] + procs = [] + settings = Setting(manage).get_settings() + db = DB(settings) + db.connect() + try: + if 'orchestra.contrib.tasks' in settings['INSTALLED_APPS']: + if settings.get('TASKS_BACKEND', 'thread') in ('thread', 'process'): + for proc in fire_pending_tasks(manage, db): + procs.append(proc) + if 'orchestra.contrib.mailer' in settings['INSTALLED_APPS']: + for proc in fire_pending_messages(settings, db): + procs.append(proc) + exit_code = 0 + for proc in procs: + result = join(proc) + sys.stdout.write(result.stdout.decode('utf8')) + sys.stderr.write(result.stderr.decode('utf8')) + if result.return_code != 0: + exit_code = result.return_code + finally: + db.close() + sys.exit(exit_code) diff --git a/orchestra/contrib/domains/backends.py b/orchestra/contrib/domains/backends.py index 983bdbf7..64de9789 100644 --- a/orchestra/contrib/domains/backends.py +++ b/orchestra/contrib/domains/backends.py @@ -102,7 +102,7 @@ class Bind9MasterDomainBackend(ServiceController): servers.append(server.get_ip()) return servers - def get_masters(self, domain): + def get_masters_ips(self, domain): ips = list(settings.DOMAINS_MASTERS) if not ips: ips += self.get_servers(domain, Bind9MasterDomainBackend) @@ -110,24 +110,23 @@ class Bind9MasterDomainBackend(ServiceController): def get_slaves(self, domain): ips = [] - masters = self.get_masters(domain) - ns_queryset = domain.records.filter(type=Record.NS).values_list('value', flat=True) - ns_records = ns_queryset or settings.DOMAINS_DEFAULT_NS - for ns in ns_records: - hostname = ns.rstrip('.') + masters_ips = self.get_masters_ips(domain) + records = domain.get_records() + for record in records.by_type(Record.NS): + hostname = record.value.rstrip('.') # First try with a DNS query, a more reliable source try: addr = socket.gethostbyname(hostname) except socket.gaierror: - # check if domain is declared + # check if hostname is declared try: - domain = Domain.objects.get(name=ns) + domain = Domain.objects.get(name=hostname) except Domain.DoesNotExist: continue else: - a_record = domain.records.filter(name=Record.A) or [settings.DOMAINS_DEFAULT_A] - addr = a_record[0] - if addr not in masters: + # default to domain A record address + addr = records.by_type(Record.A)[0].value + if addr not in masters_ips: ips.append(addr) return OrderedSet(sorted(ips)) @@ -185,7 +184,7 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): 'name': domain.name, 'banner': self.get_banner(), 'subdomains': domain.subdomains.all(), - 'masters': '; '.join(self.get_masters(domain)) or 'none', + 'masters': '; '.join(self.get_masters_ips(domain)) or 'none', 'conf_path': self.CONF_PATH, } context['conf'] = textwrap.dedent(""" diff --git a/orchestra/contrib/domains/helpers.py b/orchestra/contrib/domains/helpers.py index f1e1842b..a161fb84 100644 --- a/orchestra/contrib/domains/helpers.py +++ b/orchestra/contrib/domains/helpers.py @@ -9,10 +9,10 @@ def domain_for_validation(instance, records): so when validation calls render_zone() it will use the new provided data """ domain = copy.copy(instance) - def get_records(records=records): + def get_declared_records(records=records): for data in records: yield Record(type=data['type'], value=data['value']) - domain.get_records = get_records + domain.get_declared_records = get_declared_records if not domain.pk: # top domain lookup for new domains diff --git a/orchestra/contrib/domains/models.py b/orchestra/contrib/domains/models.py index 4fac16bf..5b44e476 100644 --- a/orchestra/contrib/domains/models.py +++ b/orchestra/contrib/domains/models.py @@ -85,7 +85,7 @@ class Domain(models.Model): def get_absolute_url(self): return 'http://%s' % self.name - def get_records(self): + def get_declared_records(self): """ proxy method, needed for input validation, see helpers.domain_for_validation """ return self.records.all() @@ -122,10 +122,10 @@ class Domain(models.Model): self.serial = serial self.save(update_fields=['serial']) - def render_records(self): + def get_records(self): types = {} - records = [] - for record in self.get_records(): + records = utils.RecordStorage() + for record in self.get_declared_records(): types[record.type] = True if record.type == record.SOA: # Update serial and insert at 0 @@ -183,8 +183,11 @@ class Domain(models.Model): type=Record.AAAA, value=default_aaaa )) + return records + + def render_records(self): result = '' - for record in records: + for record in self.get_records(): name = '{name}.{spaces}'.format( name=self.name, spaces=' ' * (37-len(self.name)) @@ -205,6 +208,14 @@ class Domain(models.Model): value=record.value ) return result + + def has_default_mx(self): + records = self.get_records() + for record in records.by_type('MX'): + for default in settings.DOMAINS_DEFAULT_MX: + if record.value.endswith(' %s' % default.split()[-1]): + return True + return False class Record(models.Model): diff --git a/orchestra/contrib/domains/utils.py b/orchestra/contrib/domains/utils.py index c67efdb1..e644729f 100644 --- a/orchestra/contrib/domains/utils.py +++ b/orchestra/contrib/domains/utils.py @@ -1,6 +1,32 @@ +from collections import defaultdict + from django.utils import timezone +class RecordStorage(object): + """ + list-dict implementation for fast lookups of record types + """ + + def __init__(self, *args): + self.records = list(*args) + self.type = defaultdict(list) + + def __iter__(self): + return iter(self.records) + + def append(self, record): + self.records.append(record) + self.type[record['type']].append(record) + + def insert(self, ix, record): + self.records.insert(ix, record) + self.type[record['type']].insert(ix, record) + + def by_type(self, type): + return self.type[type] + + def generate_zone_serial(): today = timezone.now() return int("%.4d%.2d%.2d%.2d" % (today.year, today.month, today.day, 0)) diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py index 763b8c6d..c4174644 100644 --- a/orchestra/contrib/lists/backends.py +++ b/orchestra/contrib/lists/backends.py @@ -1,3 +1,4 @@ +import re import textwrap from django.utils.translation import ugettext_lazy as _ @@ -10,12 +11,73 @@ from . import settings from .models import List -class MailmanBackend(ServiceController): +class MailmanVirtualDomainBackend(ServiceController): + """ + Only syncs virtualdomains used on mailman addresses + """ + verbose_name = _("Mailman virtdomain-only") + model = 'lists.List' + doc_settings = (settings, + ('LISTS_VIRTUAL_ALIAS_DOMAINS_PATH',) + ) + + def is_local_domain(self, domain): + """ whether or not domain MX points to this server """ + return domain.has_default_mx() + + def include_virtual_alias_domain(self, context): + domain = context['address_domain'] + if domain and self.is_local_domain(domain): + self.append(textwrap.dedent(""" + [[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || { + echo '%(address_domain)s' >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + }""") % self.context + ) + + def is_last_domain(self, domain): + return not List.objects.filter(address_domain=domain).exists() + + def exclude_virtual_alias_domain(self, context): + domain = context['address_domain'] + if domain and self.is_last_domain(domain): + self.append("sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s" % context) + + def save(self, mail_list): + context = self.get_context(mail_list) + self.include_virtual_alias_domain(context) + + def delete(self, mail_list): + context = self.get_context(mail_list) + self.include_virtual_alias_domain(context) + + def commit(self): + context = self.get_context_files() + self.append(textwrap.dedent(""" + if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then + service postfix reload + fi""") % context + ) + + def get_context_files(self): + return { + 'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH, + } + + def get_context(self, mail_list): + context = self.get_context_files() + context.update({ + 'address_domain': mail_list.address_domain, + }) + return replace(context, "'", '"') + + +class MailmanBackend(MailmanVirtualDomainBackend): """ Mailman 2 backend based on newlist, it handles custom domains. + Includes MailmanVirtualDomainBackend """ verbose_name = "Mailman" - model = 'lists.List' addresses = [ '', '-admin', @@ -35,23 +97,6 @@ class MailmanBackend(ServiceController): 'LISTS_MAILMAN_ROOT_DIR' )) - def include_virtual_alias_domain(self, context): - if context['address_domain']: - # Check if the domain is hosted on this mail server - # TODO this is dependent on the domain model - if Domain.objects.filter(records__type=Record.MX, name=context['address_domain']).exists(): - self.append(textwrap.dedent(""" - [[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || { - echo '%(address_domain)s' >> %(virtual_alias_domains)s - UPDATED_VIRTUAL_ALIAS_DOMAINS=1 - }""") % context - ) - - def exclude_virtual_alias_domain(self, context): - address_domain = context['address_domain'] - if not List.objects.filter(address_domain=address_domain).exists(): - self.append("sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s" % context) - def get_virtual_aliases(self, context): aliases = ['# %(banner)s' % context] for address in self.addresses: @@ -163,7 +208,6 @@ class MailmanBackend(ServiceController): return replace(context, "'", '"') - class MailmanTraffic(ServiceMonitor): """ Parses mailman log file looking for email size and multiples it by list_members count. diff --git a/orchestra/contrib/mailboxes/actions.py b/orchestra/contrib/mailboxes/actions.py index 81b9150a..d5ea0fa7 100644 --- a/orchestra/contrib/mailboxes/actions.py +++ b/orchestra/contrib/mailboxes/actions.py @@ -2,6 +2,12 @@ from orchestra.admin.actions import SendEmail class SendMailboxEmail(SendEmail): - def get_queryset_emails(self): + def get_email_addresses(self): for mailbox in self.queryset.all(): yield mailbox.get_local_address() + + +class SendAddressEmail(SendEmail): + def get_email_addresses(self): + for address in self.queryset.all(): + yield address.emails diff --git a/orchestra/contrib/mailboxes/admin.py b/orchestra/contrib/mailboxes/admin.py index 221f19c7..9a53688f 100644 --- a/orchestra/contrib/mailboxes/admin.py +++ b/orchestra/contrib/mailboxes/admin.py @@ -13,7 +13,7 @@ from orchestra.contrib.accounts.admin import SelectAccountAdminMixin from orchestra.contrib.accounts.filters import IsActiveListFilter from . import settings -from .actions import SendMailboxEmail +from .actions import SendMailboxEmail, SendAddressEmail from .filters import HasMailboxListFilter, HasForwardListFilter, HasAddressListFilter from .forms import MailboxCreationForm, MailboxChangeForm, AddressForm from .models import Mailbox, Address, Autoresponse @@ -84,9 +84,9 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo def get_actions(self, request): if settings.MAILBOXES_LOCAL_ADDRESS_DOMAIN: - self.actions = (SendMailboxEmail(),) + type(self).actions = (SendMailboxEmail(),) else: - self.actions = () + type(self).actions = () return super(MailboxAdmin, self).get_actions(request) def formfield_for_dbfield(self, db_field, **kwargs): @@ -127,6 +127,7 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): inlines = [AutoresponseInline] search_fields = ('forward', 'mailboxes__name', 'account__username', 'computed_email') readonly_fields = ('account_link', 'domain_link', 'email_link') + actions = (SendAddressEmail(),) filter_by_account_fields = ('domain', 'mailboxes') filter_horizontal = ['mailboxes'] form = AddressForm diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 340e5022..30c48aeb 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -203,88 +203,64 @@ class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceControl return replace(context, "'", '"') -class PostfixAddressBackend(ServiceController): +class PostfixAddressVirtualDomainBackend(ServiceController): """ - Addresses based on Postfix virtual alias domains. + Secondary SMTP server without mailboxes in it, only syncs virtual domains. """ - verbose_name = _("Postfix address") + verbose_name = _("Postfix address virtdomain-only") model = 'mailboxes.Address' related_models = ( ('mailboxes.Mailbox', 'addresses'), ) doc_settings = (settings, - ('MAILBOXES_LOCAL_DOMAIN', 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH', 'MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH',) + ('MAILBOXES_LOCAL_DOMAIN', 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH') ) + + def is_local_domain(self, domain): + """ whether or not domain MX points to this server """ + return domain.has_default_mx() + def include_virtual_alias_domain(self, context): - if context['domain'] != context['local_domain']: - # Check if the domain is hosted on this mail server - # TODO this is dependent on the domain model - if Domain.objects.filter(records__type=Record.MX, name=context['domain']).exists(): - self.append(textwrap.dedent(""" - [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || { - echo '%(domain)s' >> %(virtual_alias_domains)s - UPDATED_VIRTUAL_ALIAS_DOMAINS=1 - }""") % context - ) + domain = context['domain'] + if domain.name != context['local_domain'] and self.is_local_domain(domain): + self.append(textwrap.dedent(""" + [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || { + echo '%(domain)s' >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + }""") % context + ) + + def is_last_domain(self, domain): + return not Address.objects.filter(domain=domain).exists() def exclude_virtual_alias_domain(self, context): domain = context['domain'] - if not Address.objects.filter(domain=domain).exists(): - self.append("sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s" % context) - - def update_virtual_alias_maps(self, address, context): - # Virtual mailbox stuff -# destination = [] -# for mailbox in address.get_mailboxes(): -# context['mailbox'] = mailbox -# destination.append("%(mailbox)s@%(local_domain)s" % context) -# for forward in address.forward: -# if '@' in forward: -# destination.append(forward) - destination = address.destination - if destination: - context['destination'] = destination - self.append(textwrap.dedent(""" - LINE='%(email)s\t%(destination)s' - if [[ ! $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then - echo "${LINE}" >> %(virtual_alias_maps)s - UPDATED_VIRTUAL_ALIAS_MAPS=1 - else - if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then - sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s - UPDATED_VIRTUAL_ALIAS_MAPS=1 - fi - fi""") % context) - else: - logger.warning("Address %i is empty" % address.pk) - self.append("sed -i '/^%(email)s\s/d' %(virtual_alias_maps)s" % context) - self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1') - - def exclude_virtual_alias_maps(self, context): - self.append(textwrap.dedent(""" - if [[ $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then - sed -i '/^%(email)s\s.*$/d' %(virtual_alias_maps)s - UPDATED_VIRTUAL_ALIAS_MAPS=1 - fi""") % context) + if self.is_last_domain(domain): + self.append(textwrap.dedent("""\ + sed -i '/^%(domain)s\s*/d;{!q0;q1}' %(virtual_alias_domains)s && \\ + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + """) % context + ) def save(self, address): context = self.get_context(address) self.include_virtual_alias_domain(context) - self.update_virtual_alias_maps(address, context) + return context def delete(self, address): context = self.get_context(address) self.exclude_virtual_alias_domain(context) - self.exclude_virtual_alias_maps(context) + return context def commit(self): context = self.get_context_files() - self.append(textwrap.dedent(""" - [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; } - [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { service postfix reload; } + self.append(textwrap.dedent("""\ + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { + service postfix reload + } + exit $exit_code """) % context ) - self.append('exit 0') def get_context_files(self): return { @@ -302,6 +278,77 @@ class PostfixAddressBackend(ServiceController): return replace(context, "'", '"') +class PostfixAddressBackend(PostfixAddressVirtualDomainBackend): + """ + Addresses based on Postfix virtual alias domains, includes PostfixAddressVirtualDomainBackend. + """ + verbose_name = _("Postfix address") + doc_settings = (settings, + ('MAILBOXES_LOCAL_DOMAIN', 'MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH', 'MAILBOXES_VIRTUAL_ALIAS_MAPS_PATH') + ) + + def update_virtual_alias_maps(self, address, context): + destination = address.destination + if destination: + context['destination'] = destination + self.append(textwrap.dedent(""" + LINE='%(email)s\t%(destination)s' + if [[ ! $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then + # Add new line + echo "${LINE}" >> %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + else + # Update existing line, if needed + if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then + sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s + UPDATED_VIRTUAL_ALIAS_MAPS=1 + fi + fi""") % context) + else: + logger.warning("Address %i is empty" % address.pk) + self.append(textwrap.dedent(""" + sed -i '/^%(email)s\s/d;{!q0;q1}' %(virtual_alias_maps)s && \\ + UPDATED_VIRTUAL_ALIAS_MAPS=1 + """) % context + ) + # Virtual mailbox stuff +# destination = [] +# for mailbox in address.get_mailboxes(): +# context['mailbox'] = mailbox +# destination.append("%(mailbox)s@%(local_domain)s" % context) +# for forward in address.forward: +# if '@' in forward: +# destination.append(forward) + + def exclude_virtual_alias_maps(self, context): + self.append(textwrap.dedent(""" + sed -i '/^%(email)s\s.*$/d;{!q0;q1}' %(virtual_alias_maps)s && \\ + UPDATED_VIRTUAL_ALIAS_MAPS=1 + """) % context + ) + + def save(self, address): + context = super(PostfixAddressBackend, self).save(address) + self.update_virtual_alias_maps(address, context) + + def delete(self, address): + context = super(PostfixAddressBackend, self).save(address) + self.exclude_virtual_alias_maps(context) + + def commit(self): + context = self.get_context_files() + self.append(textwrap.dedent("""\ + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { + service postfix reload + } + [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { + postmap %(virtual_alias_maps)s + } + exit $exit_code + """) % context + ) + + class AutoresponseBackend(ServiceController): """ WARNING: not implemented diff --git a/orchestra/contrib/mailer/__init__.py b/orchestra/contrib/mailer/__init__.py index e69de29b..335e52d3 100644 --- a/orchestra/contrib/mailer/__init__.py +++ b/orchestra/contrib/mailer/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.mailer.apps.MailerConfig' diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py index 70706e31..a884360d 100644 --- a/orchestra/contrib/mailer/admin.py +++ b/orchestra/contrib/mailer/admin.py @@ -1,22 +1,57 @@ from django.contrib import admin +from django.core.urlresolvers import reverse +from django.db.models import Count +from django.utils.translation import ugettext_lazy as _ -from orchestra.admin.utils import admin_link +from orchestra.admin.utils import admin_link, admin_colored, admin_date from .models import Message, SMTPLog +COLORS = { + Message.QUEUED: 'purple', + Message.SENT: 'green', + Message.DEFERRED: 'darkorange', + Message.FAILED: 'red', + SMTPLog.SUCCESS: 'green', + SMTPLog.FAILURE: 'red', +} + + class MessageAdmin(admin.ModelAdmin): list_display = ( - 'id', 'state', 'priority', 'to_address', 'from_address', 'created_at', 'retries', 'last_retry' + 'id', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta', + 'retries', 'last_retry_delta', 'num_logs', ) + list_filter = ('state', 'priority', 'retries') + + colored_state = admin_colored('state', colors=COLORS) + created_at_delta = admin_date('created_at') + last_retry_delta = admin_date('last_retry') + + def num_logs(self, instance): + num = instance.logs__count + url = reverse('admin:mailer_smtplog_changelist') + url += '?&message=%i' % instance.pk + return '%d' % (url, num) + num_logs.short_description = _("Logs") + num_logs.admin_order_field = 'logs__count' + num_logs.allow_tags = True + + def get_queryset(self, request): + qs = super(MessageAdmin, self).get_queryset(request) + return qs.annotate(Count('logs')) class SMTPLogAdmin(admin.ModelAdmin): list_display = ( - 'id', 'message_link', 'result', 'date', 'log_message' + 'id', 'message_link', 'colored_result', 'date_delta', 'log_message' ) + list_filter = ('result',) message_link = admin_link('message') + colored_result = admin_colored('result', colors=COLORS, bold=False) + date_delta = admin_date('date') admin.site.register(Message, MessageAdmin) diff --git a/orchestra/contrib/mailer/apps.py b/orchestra/contrib/mailer/apps.py new file mode 100644 index 00000000..fd8cd958 --- /dev/null +++ b/orchestra/contrib/mailer/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + +from orchestra.core import administration + + +class MailerConfig(AppConfig): + name = 'orchestra.contrib.mailer' + verbose_name = "Mailer" + + def ready(self): + from .models import Message + administration.register(Message) diff --git a/orchestra/contrib/mailer/backends.py b/orchestra/contrib/mailer/backends.py index cfa8748a..e7cc46a8 100644 --- a/orchestra/contrib/mailer/backends.py +++ b/orchestra/contrib/mailer/backends.py @@ -5,27 +5,27 @@ from .tasks import send_message class EmailBackend(BaseEmailBackend): - ''' + """ A wrapper that manages a queued SMTP system. - ''' + """ def send_messages(self, email_messages): if not email_messages: return num_sent = 0 - # TODO if multiple messages queue, else async? + is_bulk = len(email_messages) > 1 for message in email_messages: priority = message.extra_headers.get('X-Mail-Priority', Message.NORMAL) - if priority == Message.CRITICAL: - send_message(message).apply_async() - else: - content = message.message().as_string() - for to_email in message.recipients(): - message = Message.objects.create( - priority=priority, - to_address=to_email, - from_address=message.from_email, - subject=message.subject, - content=content, - ) + content = message.message().as_string() + for to_email in message.recipients(): + message = Message.objects.create( + priority=priority, + to_address=to_email, + from_address=message.from_email, + subject=message.subject, + content=content, + ) + if not is_bulk or priority == Message.CRITICAL: + # send immidiately + send_message.apply_async(message) num_sent += 1 return num_sent diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py index 62b9e40c..8e8e2fe3 100644 --- a/orchestra/contrib/mailer/engine.py +++ b/orchestra/contrib/mailer/engine.py @@ -7,7 +7,7 @@ from django.utils.encoding import smart_str from .models import Message -def send_message(message, num, connection, bulk): +def send_message(message, num=0, connection=None, bulk=100): if num >= bulk: connection.close() connection = None @@ -34,6 +34,7 @@ def send_pending(bulk=100): num = 0 for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'): send_message(message, num, connection, bulk) + num += 1 from django.utils import timezone from . import settings from datetime import timedelta @@ -48,4 +49,3 @@ def send_pending(bulk=100): send_message(message, num, connection, bulk) if connection is not None: connection.close() - diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py index fc75969a..6a943f47 100644 --- a/orchestra/contrib/mailer/models.py +++ b/orchestra/contrib/mailer/models.py @@ -16,10 +16,10 @@ class Message(models.Model): (FAILED, _("Failes")), ) - CRITICAL = '0' - HIGH = '1' - NORMAL = '2' - LOW = '3' + CRITICAL = 0 + HIGH = 1 + NORMAL = 2 + LOW = 3 PRIORITIES = ( (CRITICAL, _("Critical (not queued)")), (HIGH, _("High")), @@ -31,11 +31,12 @@ class Message(models.Model): priority = models.PositiveIntegerField(_("Priority"), choices=PRIORITIES, default=NORMAL) to_address = models.CharField(max_length=256) from_address = models.CharField(max_length=256) - subject = models.CharField(max_length=256) - content = models.TextField() - created_at = models.DateTimeField(auto_now_add=True) - retries = models.PositiveIntegerField(default=0) - last_retry = models.DateTimeField(auto_now=True) + subject = models.CharField(_("subject"), max_length=256) + content = models.TextField(_("content")) + created_at = models.DateTimeField(_("created at"), auto_now_add=True) + retries = models.PositiveIntegerField(_("retries"), default=0) + # TODO rename to last_try + last_retry = models.DateTimeField(_("last try"), auto_now=True) def defer(self): self.state = self.DEFERRED diff --git a/orchestra/contrib/mailer/settings.py b/orchestra/contrib/mailer/settings.py index 7c2290ee..c001c978 100644 --- a/orchestra/contrib/mailer/settings.py +++ b/orchestra/contrib/mailer/settings.py @@ -4,3 +4,8 @@ from orchestra.contrib.settings import Setting MAILER_DEFERE_SECONDS = Setting('MAILER_DEFERE_SECONDS', (300, 600, 60*60, 60*60*24), ) + + +MAILER_MESSAGES_CLEANUP_DAYS = Setting('MAILER_MESSAGES_CLEANUP_DAYS', + 10 +) diff --git a/orchestra/contrib/mailer/tasks.py b/orchestra/contrib/mailer/tasks.py index b5a4e70c..651a0c96 100644 --- a/orchestra/contrib/mailer/tasks.py +++ b/orchestra/contrib/mailer/tasks.py @@ -1,6 +1,20 @@ -def send_message(): - pass +from django.utils import timezone +from celery.task.schedules import crontab + +from orchestra.contrib.tasks import task, periodic_task + +from . import engine +@task +def send_message(message): + engine.send_message(message) + + +@periodic_task(run_every=crontab(hour=7, minute=30)) def cleanup_messages(): - pass + from .models import Message + delta = timedelta(days=settings.MAILER_MESSAGES_CLEANUP_DAYS) + now = timezone.now() + epoch = (now-delta) + Message.objects.filter(state=Message.SENT, last_retry__lt=epoc).delete() diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index fd18fb74..53020523 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -19,6 +19,7 @@ STATE_COLORS = { BackendLog.FAILURE: 'red', BackendLog.ERROR: 'red', BackendLog.REVOKED: 'magenta', + BackendLog.NOTHING: 'green', } diff --git a/orchestra/contrib/orchestration/backends.py b/orchestra/contrib/orchestration/backends.py index 02bb96f7..5b8f298b 100644 --- a/orchestra/contrib/orchestration/backends.py +++ b/orchestra/contrib/orchestration/backends.py @@ -46,6 +46,9 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): # Force the backend manager to block in multiple backend executions executing them synchronously block = False doc_settings = None + # By default backend will not run if actions do not generate insctructions, + # If your backend uses prepare() or commit() only then you should set force_empty_action_execution = True + force_empty_action_execution = False def __str__(self): return type(self).__name__ @@ -64,16 +67,29 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): 'tail', 'content', 'script_method', - 'function_method' + 'function_method', + 'set_head', + 'set_tail', + 'set_content', + 'actions', ) if attr == 'prepare': - self.cmd_section = self.head + self.set_head() elif attr == 'commit': - self.cmd_section = self.tail - elif attr not in IGNORE_ATTRS: - self.cmd_section = self.content + self.set_tail() + elif attr not in IGNORE_ATTRS and attr in self.actions: + self.set_content() return super(ServiceBackend, self).__getattribute__(attr) + def set_head(self): + self.cmd_section = self.head + + def set_tail(self): + self.cmd_section = self.tail + + def set_content(self): + self.cmd_section = self.content + @classmethod def get_actions(cls): return [ action for action in cls.actions if action in dir(cls) ] @@ -148,13 +164,15 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount): from .models import BackendLog scripts = self.scripts state = BackendLog.STARTED - if not scripts: - state = BackendLog.SUCCESS + run = bool(scripts) or (self.force_empty_action_execution or bool(self.content)) + if not run: + state = BackendLog.NOTHING log = BackendLog.objects.create(backend=self.get_name(), state=state, server=server) - for method, commands in scripts: - method(log, server, commands, async) - if log.state != BackendLog.SUCCESS: - break + if run: + for method, commands in scripts: + method(log, server, commands, async) + if log.state != BackendLog.SUCCESS: + break return log def append(self, *cmd): diff --git a/orchestra/contrib/orchestration/helpers.py b/orchestra/contrib/orchestration/helpers.py index 98c941c5..85e9c191 100644 --- a/orchestra/contrib/orchestration/helpers.py +++ b/orchestra/contrib/orchestration/helpers.py @@ -78,7 +78,7 @@ def message_user(request, logs): if log.state != log.EXCEPTION: # EXCEPTION logs are not stored on the database ids.append(log.pk) - if log.state == log.SUCCESS: + if log.state in (log.SUCCESS, log.NOTHING): successes += 1 errors = total-successes if len(ids) == 1: diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index 1b5f8b24..e177f4e3 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -12,7 +12,7 @@ from . import settings, Operation from .backends import ServiceBackend from .helpers import send_report from .models import BackendLog -from .signals import pre_action, post_action +from .signals import pre_action, post_action, pre_commit, post_commit, pre_prepare, post_prepare logger = logging.getLogger(__name__) @@ -54,8 +54,12 @@ def generate(operations): for server in operation.servers: key = (server, operation.backend) if key not in scripts: - scripts[key] = (operation.backend(), [operation]) - scripts[key][0].prepare() + backend, operations = (operation.backend(), [operation]) + scripts[key] = (backend, operations) + backend.set_head() + pre_prepare.send(sender=backend.__class__, backend=backend) + backend.prepare() + post_prepare.send(sender=backend.__class__, backend=backend) else: scripts[key][1].append(operation) # Get and call backend action method @@ -67,6 +71,7 @@ def generate(operations): 'instance': operation.instance, 'action': operation.action, } + backend.set_content() pre_action.send(**kwargs) method(operation.instance) post_action.send(**kwargs) @@ -74,7 +79,10 @@ def generate(operations): block = True for value in scripts.values(): backend, operations = value + backend.set_tail() + pre_commit.send(sender=backend.__class__, backend=backend) backend.commit() + post_commit.send(sender=backend.__class__, backend=backend) return scripts, block diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py index fbc90b68..1a070c7f 100644 --- a/orchestra/contrib/orchestration/models.py +++ b/orchestra/contrib/orchestration/models.py @@ -26,7 +26,7 @@ class Server(models.Model): default=settings.ORCHESTRATION_DEFAULT_OS) def __str__(self): - return self.name + return self.name or str(self.address) def get_address(self): if self.address: @@ -53,6 +53,7 @@ class BackendLog(models.Model): ERROR = 'ERROR' REVOKED = 'REVOKED' ABORTED = 'ABORTED' + NOTHING = 'NOTHING' # Special state for mocked backendlogs EXCEPTION = 'EXCEPTION' @@ -65,6 +66,7 @@ class BackendLog(models.Model): (ERROR, ERROR), (ABORTED, ABORTED), (REVOKED, REVOKED), + (NOTHING, NOTHING), ) backend = models.CharField(_("backend"), max_length=256) diff --git a/orchestra/contrib/orchestration/signals.py b/orchestra/contrib/orchestration/signals.py index 6f8dc6e3..ab26858b 100644 --- a/orchestra/contrib/orchestration/signals.py +++ b/orchestra/contrib/orchestration/signals.py @@ -4,3 +4,11 @@ import django.dispatch pre_action = django.dispatch.Signal(providing_args=['backend', 'instance', 'action']) post_action = django.dispatch.Signal(providing_args=['backend', 'instance', 'action']) + +pre_prepare = django.dispatch.Signal(providing_args=['backend']) + +post_prepare = django.dispatch.Signal(providing_args=['backend']) + +pre_commit = django.dispatch.Signal(providing_args=['backend']) + +post_commit = django.dispatch.Signal(providing_args=['backend']) diff --git a/orchestra/contrib/orchestration/tasks.py b/orchestra/contrib/orchestration/tasks.py index 3884d33e..4c06b794 100644 --- a/orchestra/contrib/orchestration/tasks.py +++ b/orchestra/contrib/orchestration/tasks.py @@ -8,7 +8,7 @@ from orchestra.contrib.tasks import periodic_task from .models import BackendLog -@periodic_task(run_every=crontab(hour=7, minute=30, day_of_week=1)) +@periodic_task(run_every=crontab(hour=7, minute=0)) def backend_logs_cleanup(): days = settings.ORCHESTRATION_BACKEND_CLEANUP_DAYS epoch = timezone.now()-timedelta(days=days) diff --git a/orchestra/contrib/tasks/bin/orchestra-beat b/orchestra/contrib/tasks/bin/orchestra-beat deleted file mode 100755 index d1ac3f13..00000000 --- a/orchestra/contrib/tasks/bin/orchestra-beat +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 - -# High performance alternative to beat management command -# -# USAGE: beat /path/to/project/manage.py - - -import json -import os -import re -import sys -from datetime import datetime - -from orchestra.utils import db -from orchestra.utils.python import import_class -from orchestra.utils.sys import run, join - -from celery.schedules import crontab_parser as CrontabParser - - -def get_settings_file(manage): - with open(manage, 'r') as handler: - regex = re.compile(r'"DJANGO_SETTINGS_MODULE"\s*,\s*"([^"]+)"') - for line in handler.readlines(): - match = regex.search(line) - if match: - settings_module = match.groups()[0] - settings_file = os.path.join(*settings_module.split('.')) + '.py' - settings_file = os.path.join(os.path.dirname(manage), settings_file) - return settings_file - raise ValueError("settings module not found in %s" % manage) - - -def get_tasks(manage): - settings_file = get_settings_file(manage) - settings = db.get_settings(settings_file) - try: - conn = db.get_connection(settings) - except: - sys.stdout.write("ERROR") - sys.stderr.write("I am unable to connect to the database\n") - sys.exit(1) - script, settings_file = sys.argv[:2] - enabled = 1 if 'sqlite' in settings['ENGINE'] else True - query = ( - "SELECT c.minute, c.hour, c.day_of_week, c.day_of_month, c.month_of_year, p.id " - "FROM djcelery_periodictask as p, djcelery_crontabschedule as c " - "WHERE p.crontab_id = c.id AND p.enabled = {}" - ).format(enabled) - tasks = db.run_query(conn, query) - conn.close() - return tasks - - -def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): - n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now - return ( - n_minute in CrontabParser(60).parse(minute) and - n_hour in CrontabParser(24).parse(hour) and - n_day_of_week in CrontabParser(7).parse(day_of_week) and - n_day_of_month in CrontabParser(31, 1).parse(day_of_month) and - n_month_of_year in CrontabParser(12, 1).parse(month_of_year) - ) - - -if __name__ == "__main__": - manage = sys.argv[1] - now = datetime.utcnow() - now = tuple(map(int, now.strftime("%M %H %w %d %m").split())) - procs = [] - for minute, hour, day_of_week, day_of_month, month_of_year, task_id in get_tasks(manage): - if is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): - command = 'python3 -W ignore::DeprecationWarning {manage} runtask {task_id}'.format( - manage=manage, task_id=task_id) - proc = run(command, async=True) - procs.append(proc) - code = 0 - for proc in procs: - result = join(proc) - sys.stdout.write(result.stdout.decode('utf8')) - sys.stderr.write(result.stderr.decode('utf8')) - if result.return_code != 0: - code = result.return_code - sys.exit(code) diff --git a/orchestra/contrib/tasks/decorators.py b/orchestra/contrib/tasks/decorators.py index 9deb77cb..c08af43e 100644 --- a/orchestra/contrib/tasks/decorators.py +++ b/orchestra/contrib/tasks/decorators.py @@ -76,9 +76,9 @@ def task(fn=None, **kwargs): return decorator else: return celery_shared_task(**kwargs) - fn = update_wraper(partial(celery_shared_task, fn)) + fn = celery_shared_task(fn) if settings.TASKS_BACKEND in ('thread', 'process'): - fn = update_wrapper(apply_async(fn), fn) + fn = apply_async(fn) return fn @@ -93,7 +93,7 @@ def periodic_task(fn=None, **kwargs): return decorator else: return celery_periodic_task(**kwargs) - fn = update_wraper(celery_periodic_task(fn), fn) + fn = celery_periodic_task(fn) if settings.TASKS_BACKEND in ('thread', 'process'): name = kwargs.pop('name', None) fn = update_wrapper(apply_async(fn, name), fn) diff --git a/orchestra/core/__init__.py b/orchestra/core/__init__.py index 7772dad5..0badee82 100644 --- a/orchestra/core/__init__.py +++ b/orchestra/core/__init__.py @@ -30,3 +30,4 @@ class Register(object): services = Register() # TODO rename to something else accounts = Register() +administration = Register() diff --git a/orchestra/contrib/tasks/management/commands/setupcronbeat.py b/orchestra/management/commands/setupcronbeat.py similarity index 84% rename from orchestra/contrib/tasks/management/commands/setupcronbeat.py rename to orchestra/management/commands/setupcronbeat.py index d4857870..9b2f8009 100644 --- a/orchestra/contrib/tasks/management/commands/setupcronbeat.py +++ b/orchestra/management/commands/setupcronbeat.py @@ -3,12 +3,13 @@ import os from django.core.management.base import BaseCommand, CommandError from orchestra.utils.paths import get_site_dir -from orchestra.utils.sys import run +from orchestra.utils.sys import run, check_non_root class Command(BaseCommand): - help = 'Runs periodic tasks.' + help = 'Confingure crontab to run periodic tasks and mailer with orchestra-beat.' + @check_non_root def handle(self, *args, **options): context = { 'site_dir': get_site_dir(), diff --git a/orchestra/utils/db.py b/orchestra/utils/db.py index f2222efd..11c33c13 100644 --- a/orchestra/utils/db.py +++ b/orchestra/utils/db.py @@ -13,33 +13,3 @@ def close_connection(execute): finally: db.connection.close() return wrapper - - -def get_settings(settings_file): - """ get db settings from settings.py file without importing """ - settings = {'__file__': settings_file} - with open(settings_file) as f: - __file__ = 'rata' - exec(f.read(), settings) - settings = settings['DATABASES']['default'] - if settings['ENGINE'] not in ('django.db.backends.sqlite3', 'django.db.backends.postgresql_psycopg2'): - raise ValueError("%s engine not supported." % settings['ENGINE']) - return settings - - -def get_connection(settings): - if settings['ENGINE'] == 'django.db.backends.sqlite3': - import sqlite3 - return sqlite3.connect(settings['NAME']) - elif settings['ENGINE'] == 'django.db.backends.postgresql_psycopg2': - import psycopg2 - return psycopg2.connect("dbname='{NAME}' user='{USER}' host='{HOST}' password='{PASSWORD}'".format(**settings)) - return conn - - -def run_query(conn, query): - cur = conn.cursor() - cur.execute(query) - result = cur.fetchall() - cur.close() - return result diff --git a/orchestra/utils/sys.py b/orchestra/utils/sys.py index 1cbce5c7..3c5da067 100644 --- a/orchestra/utils/sys.py +++ b/orchestra/utils/sys.py @@ -21,6 +21,16 @@ def check_root(func): return wrapped +def check_non_root(func): + """ Function decorator that checks if user not has root permissions """ + def wrapped(*args, **kwargs): + if getpass.getuser() == 'root': + cmd_name = func.__module__.split('.')[-1] + raise CommandError("Sorry, you don't want to execute '%s' as superuser (root)." % cmd_name) + return func(*args, **kwargs) + return wrapped + + class _Attribute(object): """ Simple string subclass to allow arbitrary attribute access. """ def __init__(self, stdout): diff --git a/scripts/container/deploy.sh b/scripts/container/deploy.sh index 0dcae250..1fb8c601 100755 --- a/scripts/container/deploy.sh +++ b/scripts/container/deploy.sh @@ -21,6 +21,8 @@ HOME="/home/$USER" PROJECT_NAME='panel' BASE_DIR="$HOME/$PROJECT_NAME" PYTHON_BIN="python3" +CELERY=false + surun () { echo " ${bold}\$ su $USER -c \"${@}\"${normal}" @@ -93,17 +95,23 @@ fi run "$PYTHON_BIN $MANAGE migrate --noinput accounts" run "$PYTHON_BIN $MANAGE migrate --noinput" -sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2 +if [[ $CELERY == true ]]; then + run apt-get install -y rabbitmq + sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2 +else + run "$PYTHON_BIN $MANAGE setupcronbeat" +fi + # Install and configure Nginx+uwsgi web services surun "mkdir -p $BASE_DIR/static" surun "$PYTHON_BIN $MANAGE collectstatic --noinput" run "apt-get install -y nginx uwsgi uwsgi-plugin-python3" -run "$PYTHON_BIN $MANAGE setupnginx" +run "$PYTHON_BIN $MANAGE setupnginx --user $USER --noinput" run "service nginx start" # Apply changes on related services -run "$PYTHON_BIN $MANAGE restartservices" +run "$PYTHON_BIN $MANAGE reloadservices" # Create orchestra superuser cat <<- EOF | $PYTHON_BIN $MANAGE shell diff --git a/setup.py b/setup.py index 804e8337..7e7cdf9e 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( include_package_data = True, scripts=[ 'orchestra/bin/orchestra-admin', - 'orchestra/contrib/tasks/bin/orchestra-beat', + 'orchestra//bin/orchestra-beat', ], packages = packages, classifiers = [