From 524b1ce15fbb437c65fee4d90ff83f46a86a63d4 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Wed, 6 May 2015 19:30:13 +0000 Subject: [PATCH] Replace custom sql on domains and imporve performance of orchestra-beat --- README.md | 2 +- TODO.md | 13 ++- orchestra/bin/orchestra-beat | 97 ++++++++++++++++++++-- orchestra/contrib/accounts/models.py | 2 +- orchestra/contrib/domains/admin.py | 11 +-- orchestra/contrib/issues/models.py | 2 +- orchestra/contrib/mailboxes/actions.py | 2 +- orchestra/contrib/orchestration/models.py | 2 +- orchestra/contrib/orders/apps.py | 2 +- orchestra/contrib/resources/admin.py | 2 +- orchestra/contrib/resources/apps.py | 2 +- orchestra/contrib/resources/serializers.py | 2 +- orchestra/contrib/services/__init__.py | 2 +- orchestra/contrib/services/apps.py | 6 ++ orchestra/contrib/services/models.py | 1 - orchestra/contrib/websites/apps.py | 2 +- orchestra/utils/__init__.py | 1 - orchestra/utils/db.py | 18 ++++ orchestra/utils/{options.py => mail.py} | 28 ------- 19 files changed, 131 insertions(+), 66 deletions(-) create mode 100644 orchestra/contrib/services/apps.py rename orchestra/utils/{options.py => mail.py} (60%) diff --git a/README.md b/README.md index 2df969b9..d47a9226 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ pip3 install django-orchestra==dev \ pip3 install -r \ https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt -# Create an new Orchestra site +# Create a new Orchestra site orchestra-admin startproject panel python3 panel/manage.py migrate accounts python3 panel/manage.py migrate diff --git a/TODO.md b/TODO.md index 0189536d..d6bf4848 100644 --- a/TODO.md +++ b/TODO.md @@ -333,9 +333,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip # all signals + accouns.register() services.register() on apps.py -# if backend.async: don't join. -# RELATED: domains.sync to ns3 make it async backend rather than cron based ? - from orchestra.contrib.tasks import task import time, sys @task(name='rata') @@ -353,13 +350,11 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix TODO mount the filesystem with "nosuid" option -# execute Make after postfix update # wkhtmltopdf -> reportlab -# autoiscover modules on app.ready() +# autoiscover modules on app.ready() ? lazy choices on models for plugins +# ModelTranslation.register on app.ready() # uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask -# avoid cron email errors when failing hard - # mailboxes.address settings multiple local domains, not only one? # backend.context = self.get_context() or save(obj, context=None) @@ -377,3 +372,7 @@ TODO mount the filesystem with "nosuid" option # don't block on beat, and --report periodic tasks # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery) + +# orchestrate async stdout stderr (inspired on pangea managemengt commands) + +# orchestra-beat support for uwsgi cron diff --git a/orchestra/bin/orchestra-beat b/orchestra/bin/orchestra-beat index 26b8a5d5..535b954a 100755 --- a/orchestra/bin/orchestra-beat +++ b/orchestra/bin/orchestra-beat @@ -14,11 +14,86 @@ import re import sys from datetime import datetime, timedelta -from celery.schedules import crontab_parser as CrontabParser - from orchestra.utils.sys import run, join, LockFile +class crontab_parser(object): + """ + from celery.schedules import crontab_parser + Too expensive to import celery + """ + ParseException = ValueError + + _range = r'(\w+?)-(\w+)' + _steps = r'/(\w+)?' + _star = r'\*' + + def __init__(self, max_=60, min_=0): + self.max_ = max_ + self.min_ = min_ + self.pats = ( + (re.compile(self._range + self._steps), self._range_steps), + (re.compile(self._range), self._expand_range), + (re.compile(self._star + self._steps), self._star_steps), + (re.compile('^' + self._star + '$'), self._expand_star), + ) + + def parse(self, spec): + acc = set() + for part in spec.split(','): + if not part: + raise self.ParseException('empty part') + acc |= set(self._parse_part(part)) + return acc + + def _parse_part(self, part): + for regex, handler in self.pats: + m = regex.match(part) + if m: + return handler(m.groups()) + return self._expand_range((part, )) + + def _expand_range(self, toks): + fr = self._expand_number(toks[0]) + if len(toks) > 1: + to = self._expand_number(toks[1]) + if to < fr: # Wrap around max_ if necessary + return (list(range(fr, self.min_ + self.max_)) + + list(range(self.min_, to + 1))) + return list(range(fr, to + 1)) + return [fr] + + def _range_steps(self, toks): + if len(toks) != 3 or not toks[2]: + raise self.ParseException('empty filter') + return self._expand_range(toks[:2])[::int(toks[2])] + + def _star_steps(self, toks): + if not toks or not toks[0]: + raise self.ParseException('empty filter') + return self._expand_star()[::int(toks[0])] + def _expand_star(self, *args): + return list(range(self.min_, self.max_ + self.min_)) + + def _expand_number(self, s): + if isinstance(s, str) and s[0] == '-': + raise self.ParseException('negative numbers not supported') + try: + i = int(s) + except ValueError: + try: + i = weekday(s) + except KeyError: + raise ValueError('Invalid weekday literal {0!r}.'.format(s)) + max_val = self.min_ + self.max_ - 1 + if i > max_val: + raise ValueError( + 'Invalid end range: {0} > {1}.'.format(i, max_val)) + if i < self.min_: + raise ValueError( + 'Invalid beginning range: {0} < {1}.'.format(i, self.min_)) + return i + class Setting(object): def __init__(self, manage): self.manage = manage @@ -28,8 +103,12 @@ class Setting(object): """ get db settings from settings.py file without importing """ settings = {'__file__': self.settings_file} with open(self.settings_file) as f: - __file__ = 'rata' - exec(f.read(), settings) + content = '' + for line in f.readlines(): + # This is very costly, skip + if not line.startswith(('import djcelery', 'djcelery.setup_loader()')): + content += line + exec(content, settings) return settings def get_settings_file(self, manage): @@ -85,11 +164,11 @@ def fire_pending_tasks(manage, db): def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year): n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now return ( - n_minute in CrontabParser(60).parse(minute) and - n_hour in CrontabParser(24).parse(hour) and - n_day_of_week in CrontabParser(7).parse(day_of_week) and - n_day_of_month in CrontabParser(31, 1).parse(day_of_month) and - n_month_of_year in CrontabParser(12, 1).parse(month_of_year) + n_minute in crontab_parser(60).parse(minute) and + n_hour in crontab_parser(24).parse(hour) and + n_day_of_week in crontab_parser(7).parse(day_of_week) and + n_day_of_month in crontab_parser(31, 1).parse(day_of_month) and + n_month_of_year in crontab_parser(12, 1).parse(month_of_year) ) now = datetime.utcnow() diff --git a/orchestra/contrib/accounts/models.py b/orchestra/contrib/accounts/models.py index 5033eeed..8bdd9d1b 100644 --- a/orchestra/contrib/accounts/models.py +++ b/orchestra/contrib/accounts/models.py @@ -7,7 +7,7 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.orchestration.middlewares import OperationsMiddleware from orchestra.contrib.orchestration import Operation -from orchestra.utils import send_email_template +from orchestra.utils.mail import send_email_template from . import settings diff --git a/orchestra/contrib/domains/admin.py b/orchestra/contrib/domains/admin.py index bd0fbb4e..78ff706f 100644 --- a/orchestra/contrib/domains/admin.py +++ b/orchestra/contrib/domains/admin.py @@ -2,6 +2,7 @@ import re from django import forms from django.contrib import admin +from django.db.models.functions import Concat from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin @@ -103,16 +104,8 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): """ Order by structured name and imporve performance """ qs = super(DomainAdmin, self).get_queryset(request) qs = qs.select_related('top', 'account') - # Order by structured name if request.method == 'GET': - # For some reason if we do this we know for sure that join table will be called T4 - query = str(qs.query) - table = re.findall(r'(T\d+)\."account_id"', query)[0] - qs = qs.extra( - select={ - 'structured_name': 'CONCAT({table}.name, domains_domain.name)'.format(table=table) - }, - ).order_by('structured_name') + qs = qs.annotate(structured_name=Concat('top__name', 'name')).order_by('structured_name') if apps.isinstalled('orchestra.contrib.websites'): qs = qs.prefetch_related('websites') return qs diff --git a/orchestra/contrib/issues/models.py b/orchestra/contrib/issues/models.py index 645521b0..62bf22dd 100644 --- a/orchestra/contrib/issues/models.py +++ b/orchestra/contrib/issues/models.py @@ -6,7 +6,7 @@ from orchestra.contrib.contacts import settings as contacts_settings from orchestra.contrib.contacts.models import Contact from orchestra.core.translations import ModelTranslation from orchestra.models.fields import MultiSelectField -from orchestra.utils import send_email_template +from orchestra.utils.mail import send_email_template from . import settings diff --git a/orchestra/contrib/mailboxes/actions.py b/orchestra/contrib/mailboxes/actions.py index d5ea0fa7..bef631da 100644 --- a/orchestra/contrib/mailboxes/actions.py +++ b/orchestra/contrib/mailboxes/actions.py @@ -10,4 +10,4 @@ class SendMailboxEmail(SendEmail): class SendAddressEmail(SendEmail): def get_email_addresses(self): for address in self.queryset.all(): - yield address.emails + yield address.email diff --git a/orchestra/contrib/orchestration/models.py b/orchestra/contrib/orchestration/models.py index 0b79680b..4090fc7a 100644 --- a/orchestra/contrib/orchestration/models.py +++ b/orchestra/contrib/orchestration/models.py @@ -135,7 +135,7 @@ class Route(models.Model): "instance referes to the current object.")) async = models.BooleanField(default=False, help_text=_("Whether or not block the request/response cycle waitting this backend to " - "finish its execution.")) + "finish its execution. Usually you want slave servers to run asynchronously.")) # method = models.CharField(_("method"), max_lenght=32, choices=method_choices, # default=MethodBackend.get_default()) is_active = models.BooleanField(_("active"), default=True) diff --git a/orchestra/contrib/orders/apps.py b/orchestra/contrib/orders/apps.py index 39934323..733d281b 100644 --- a/orchestra/contrib/orders/apps.py +++ b/orchestra/contrib/orders/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from orchestra.core import accounts -from orchestra.utils import database_ready +from orchestra.utils.db import database_ready class OrdersConfig(AppConfig): diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py index 13e02f99..060c21d6 100644 --- a/orchestra/contrib/resources/admin.py +++ b/orchestra/contrib/resources/admin.py @@ -11,7 +11,7 @@ from orchestra.admin import ExtendedModelAdmin from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date from orchestra.contrib.orchestration.models import Route from orchestra.core import services -from orchestra.utils import database_ready +from orchestra.utils.db import database_ready from orchestra.utils.functional import cached from .actions import run_monitor diff --git a/orchestra/contrib/resources/apps.py b/orchestra/contrib/resources/apps.py index aa49c144..ca3c4215 100644 --- a/orchestra/contrib/resources/apps.py +++ b/orchestra/contrib/resources/apps.py @@ -1,7 +1,7 @@ from django import db from django.apps import AppConfig -from orchestra.utils import database_ready +from orchestra.utils.db import database_ready class ResourcesConfig(AppConfig): diff --git a/orchestra/contrib/resources/serializers.py b/orchestra/contrib/resources/serializers.py index 5819692f..a2cde99e 100644 --- a/orchestra/contrib/resources/serializers.py +++ b/orchestra/contrib/resources/serializers.py @@ -1,7 +1,7 @@ from rest_framework import serializers from orchestra.api import router -from orchestra.utils import database_ready +from orchestra.utils.db import database_ready from .models import Resource, ResourceData diff --git a/orchestra/contrib/services/__init__.py b/orchestra/contrib/services/__init__.py index 8b137891..cafed625 100644 --- a/orchestra/contrib/services/__init__.py +++ b/orchestra/contrib/services/__init__.py @@ -1 +1 @@ - +default_app_config = 'orchestra.contrib.services.apps.ServicesConfig' diff --git a/orchestra/contrib/services/apps.py b/orchestra/contrib/services/apps.py new file mode 100644 index 00000000..51f56168 --- /dev/null +++ b/orchestra/contrib/services/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ServicesConfig(AppConfig): + name = 'orchestra.contrib.services' + verbose_name = 'Services' diff --git a/orchestra/contrib/services/models.py b/orchestra/contrib/services/models.py index 3ebce599..27daf1ab 100644 --- a/orchestra/contrib/services/models.py +++ b/orchestra/contrib/services/models.py @@ -20,7 +20,6 @@ autodiscover_modules('handlers') rate_class = import_class(settings.SERVICES_RATE_CLASS) - class Service(models.Model): NEVER = '' # DAILY = 'DAILY' diff --git a/orchestra/contrib/websites/apps.py b/orchestra/contrib/websites/apps.py index bb4fcf97..7451c44f 100644 --- a/orchestra/contrib/websites/apps.py +++ b/orchestra/contrib/websites/apps.py @@ -1,7 +1,7 @@ from django.apps import AppConfig from django.contrib.contenttypes.fields import GenericRelation -from orchestra.utils import database_ready +from orchestra.utils.db import database_ready class WebsiteConfig(AppConfig): diff --git a/orchestra/utils/__init__.py b/orchestra/utils/__init__.py index 921dfe14..e69de29b 100644 --- a/orchestra/utils/__init__.py +++ b/orchestra/utils/__init__.py @@ -1 +0,0 @@ -from .options import * diff --git a/orchestra/utils/db.py b/orchestra/utils/db.py index 83f0a449..679f369e 100644 --- a/orchestra/utils/db.py +++ b/orchestra/utils/db.py @@ -1,6 +1,24 @@ +import sys + from django import db +def running_syncdb(): + return 'migrate' in sys.argv or 'syncdb' in sys.argv or 'makemigrations' in sys.argv + + +def database_ready(): + return ( + not running_syncdb() and + 'setuppostgres' not in sys.argv and + 'test' not in sys.argv and + # Celerybeat has yet to stablish a connection at AppConf.ready() + 'celerybeat' not in sys.argv and + # Allow to run python manage.py without a database + sys.argv != ['manage.py'] and '--help' not in sys.argv + ) + + def close_connection(execute): """ Threads have their own connection pool, closing it when finishing """ def wrapper(*args, **kwargs): diff --git a/orchestra/utils/options.py b/orchestra/utils/mail.py similarity index 60% rename from orchestra/utils/options.py rename to orchestra/utils/mail.py index e156f11b..34a712f5 100644 --- a/orchestra/utils/options.py +++ b/orchestra/utils/mail.py @@ -1,4 +1,3 @@ -import sys from urllib.parse import urlparse from django.core.mail import EmailMultiAlternatives @@ -38,30 +37,3 @@ def send_email_template(template, context, to, email_from=None, html=None, attac msg.attach_alternative(html_message, "text/html") msg.send() - -def running_syncdb(): - return 'migrate' in sys.argv or 'syncdb' in sys.argv or 'makemigrations' in sys.argv - - -def database_ready(): - return (not running_syncdb() and - 'setuppostgres' not in sys.argv and - 'test' not in sys.argv and - # Celerybeat has yet to stablish a connection at AppConf.ready() - 'celerybeat' not in sys.argv and - # Allow to run python manage.py without a database - sys.argv != ['manage.py'] and '--help' not in sys.argv) - - -def dict_setting_to_choices(choices): - return sorted( - [ (name, opt.get('verbose_name', 'name')) for name, opt in choices.items() ], - key=lambda e: e[0] - ) - - -def tuple_setting_to_choices(choices): - return sorted( - tuple((name, opt[0]) for name, opt in choices.items()), - key=lambda e: e[0] - )