Replace custom sql on domains and imporve performance of orchestra-beat

README.md

@@ -47,7 +47,7 @@ pip3 install django-orchestra==dev \
 pip3 install -r \
   https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt
 
-# Create an new Orchestra site
+# Create a new Orchestra site
 orchestra-admin startproject panel
 python3 panel/manage.py migrate accounts
 python3 panel/manage.py migrate

TODO.md

@@ -333,9 +333,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
 
 # all signals + accouns.register() services.register() on apps.py
 
-# if backend.async: don't join.
-# RELATED: domains.sync to ns3 make it async backend rather than cron based ?
-
         from orchestra.contrib.tasks import task
         import time, sys
         @task(name='rata')
@@ -353,13 +350,11 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
 TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
 TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
 TODO mount the filesystem with "nosuid" option
-# execute Make after postfix update
 # wkhtmltopdf -> reportlab
-# autoiscover modules on app.ready()
+# autoiscover modules on app.ready() ? lazy choices on models for plugins
+# ModelTranslation.register on app.ready()
 # uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
 
-# avoid cron email errors when failing hard
-
 # mailboxes.address settings multiple local domains, not only one?
 # backend.context = self.get_context() or save(obj, context=None)
 
@@ -377,3 +372,7 @@ TODO mount the filesystem with "nosuid" option
 # don't block on beat, and --report periodic tasks
 
 # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
+
+# orchestrate async stdout stderr (inspired on pangea managemengt commands)
+
+# orchestra-beat support for uwsgi cron

orchestra/bin/orchestra-beat

@@ -14,11 +14,86 @@ import re
 import sys
 from datetime import datetime, timedelta
 
-from celery.schedules import crontab_parser as CrontabParser
-
 from orchestra.utils.sys import run, join, LockFile
 
 
+class crontab_parser(object):
+    """
+    from celery.schedules import crontab_parser
+    Too expensive to import celery
+    """
+    ParseException = ValueError
+
+    _range = r'(\w+?)-(\w+)'
+    _steps = r'/(\w+)?'
+    _star = r'\*'
+    
+    def __init__(self, max_=60, min_=0):
+        self.max_ = max_
+        self.min_ = min_
+        self.pats = (
+            (re.compile(self._range + self._steps), self._range_steps),
+            (re.compile(self._range), self._expand_range),
+            (re.compile(self._star + self._steps), self._star_steps),
+            (re.compile('^' + self._star + '$'), self._expand_star),
+        )
+
+    def parse(self, spec):
+        acc = set()
+        for part in spec.split(','):
+            if not part:
+                raise self.ParseException('empty part')
+            acc |= set(self._parse_part(part))
+        return acc
+    
+    def _parse_part(self, part):
+        for regex, handler in self.pats:
+            m = regex.match(part)
+            if m:
+                return handler(m.groups())
+        return self._expand_range((part, ))
+    
+    def _expand_range(self, toks):
+        fr = self._expand_number(toks[0])
+        if len(toks) > 1:
+            to = self._expand_number(toks[1])
+            if to < fr:  # Wrap around max_ if necessary
+                return (list(range(fr, self.min_ + self.max_)) +
+                        list(range(self.min_, to + 1)))
+            return list(range(fr, to + 1))
+        return [fr]
+    
+    def _range_steps(self, toks):
+        if len(toks) != 3 or not toks[2]:
+            raise self.ParseException('empty filter')
+        return self._expand_range(toks[:2])[::int(toks[2])]
+    
+    def _star_steps(self, toks):
+        if not toks or not toks[0]:
+            raise self.ParseException('empty filter')
+        return self._expand_star()[::int(toks[0])]
+    def _expand_star(self, *args):
+        return list(range(self.min_, self.max_ + self.min_))
+    
+    def _expand_number(self, s):
+        if isinstance(s, str) and s[0] == '-':
+            raise self.ParseException('negative numbers not supported')
+        try:
+            i = int(s)
+        except ValueError:
+            try:
+                i = weekday(s)
+            except KeyError:
+                raise ValueError('Invalid weekday literal {0!r}.'.format(s))
+        max_val = self.min_ + self.max_ - 1
+        if i > max_val:
+            raise ValueError(
+                'Invalid end range: {0} > {1}.'.format(i, max_val))
+        if i < self.min_:
+            raise ValueError(
+                'Invalid beginning range: {0} < {1}.'.format(i, self.min_))
+        return i
+
 class Setting(object):
     def __init__(self, manage):
         self.manage = manage
@@ -28,8 +103,12 @@ class Setting(object):
         """ get db settings from settings.py file without importing """
         settings = {'__file__': self.settings_file}
         with open(self.settings_file) as f:
-            __file__ = 'rata'
-            exec(f.read(), settings)
+            content = ''
+            for line in f.readlines():
+                # This is very costly, skip
+                if not line.startswith(('import djcelery', 'djcelery.setup_loader()')):
+                    content += line
+            exec(content, settings)
         return settings
     
     def get_settings_file(self, manage):
@@ -85,11 +164,11 @@ def fire_pending_tasks(manage, db):
     def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year):
         n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now
         return (
-            n_minute in CrontabParser(60).parse(minute) and
-            n_hour in CrontabParser(24).parse(hour) and
-            n_day_of_week in CrontabParser(7).parse(day_of_week) and
-            n_day_of_month in CrontabParser(31, 1).parse(day_of_month) and
-            n_month_of_year in CrontabParser(12, 1).parse(month_of_year)
+            n_minute in crontab_parser(60).parse(minute) and
+            n_hour in crontab_parser(24).parse(hour) and
+            n_day_of_week in crontab_parser(7).parse(day_of_week) and
+            n_day_of_month in crontab_parser(31, 1).parse(day_of_month) and
+            n_month_of_year in crontab_parser(12, 1).parse(month_of_year)
         )
     
     now = datetime.utcnow()

orchestra/contrib/accounts/models.py

@@ -7,7 +7,7 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.contrib.orchestration.middlewares import OperationsMiddleware
 from orchestra.contrib.orchestration import Operation
-from orchestra.utils import send_email_template
+from orchestra.utils.mail import send_email_template
 
 from . import settings
 

orchestra/contrib/domains/admin.py

@@ -2,6 +2,7 @@ import re
 
 from django import forms
 from django.contrib import admin
+from django.db.models.functions import Concat
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.admin import ExtendedModelAdmin
@@ -103,16 +104,8 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin):
         """ Order by structured name and imporve performance """
         qs = super(DomainAdmin, self).get_queryset(request)
         qs = qs.select_related('top', 'account')
-        # Order by structured name
         if request.method == 'GET':
-            # For some reason if we do this we know for sure that join table will be called T4
-            query = str(qs.query)
-            table = re.findall(r'(T\d+)\."account_id"', query)[0]
-            qs = qs.extra(
-                select={
-                    'structured_name': 'CONCAT({table}.name, domains_domain.name)'.format(table=table)
-                },
-            ).order_by('structured_name')
+            qs = qs.annotate(structured_name=Concat('top__name', 'name')).order_by('structured_name')
         if apps.isinstalled('orchestra.contrib.websites'):
             qs = qs.prefetch_related('websites')
         return qs

orchestra/contrib/issues/models.py

@@ -6,7 +6,7 @@ from orchestra.contrib.contacts import settings as contacts_settings
 from orchestra.contrib.contacts.models import Contact
 from orchestra.core.translations import ModelTranslation
 from orchestra.models.fields import MultiSelectField
-from orchestra.utils import send_email_template
+from orchestra.utils.mail import send_email_template
 
 from . import settings
 

orchestra/contrib/mailboxes/actions.py

@@ -10,4 +10,4 @@ class SendMailboxEmail(SendEmail):
 class SendAddressEmail(SendEmail):
     def get_email_addresses(self):
         for address in self.queryset.all():
-            yield address.emails
+            yield address.email

orchestra/contrib/orchestration/models.py

@@ -135,7 +135,7 @@ class Route(models.Model):
                     "<em>instance</em> referes to the current object."))
     async = models.BooleanField(default=False,
         help_text=_("Whether or not block the request/response cycle waitting this backend to "
-                    "finish its execution."))
+                    "finish its execution. Usually you want slave servers to run asynchronously."))
 #    method = models.CharField(_("method"), max_lenght=32, choices=method_choices,
 #            default=MethodBackend.get_default())
     is_active = models.BooleanField(_("active"), default=True)

orchestra/contrib/orders/apps.py

@@ -1,7 +1,7 @@
 from django.apps import AppConfig
 
 from orchestra.core import accounts
-from orchestra.utils import database_ready
+from orchestra.utils.db import database_ready
 
 
 class OrdersConfig(AppConfig):

orchestra/contrib/resources/admin.py

@@ -11,7 +11,7 @@ from orchestra.admin import ExtendedModelAdmin
 from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
 from orchestra.contrib.orchestration.models import Route
 from orchestra.core import services
-from orchestra.utils import database_ready
+from orchestra.utils.db import database_ready
 from orchestra.utils.functional import cached
 
 from .actions import run_monitor

orchestra/contrib/resources/apps.py

@@ -1,7 +1,7 @@
 from django import db
 from django.apps import AppConfig
 
-from orchestra.utils import database_ready
+from orchestra.utils.db import database_ready
 
 
 class ResourcesConfig(AppConfig):

orchestra/contrib/resources/serializers.py

@@ -1,7 +1,7 @@
 from rest_framework import serializers
 
 from orchestra.api import router
-from orchestra.utils import database_ready
+from orchestra.utils.db import database_ready
 
 from .models import Resource, ResourceData
 

orchestra/contrib/services/__init__.py

@@ -1 +1 @@
-
+default_app_config = 'orchestra.contrib.services.apps.ServicesConfig'

orchestra/contrib/services/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ServicesConfig(AppConfig):
+    name = 'orchestra.contrib.services'
+    verbose_name = 'Services'

orchestra/contrib/services/models.py

@@ -20,7 +20,6 @@ autodiscover_modules('handlers')
 rate_class = import_class(settings.SERVICES_RATE_CLASS)
 
 
-
 class Service(models.Model):
     NEVER = ''
 #    DAILY = 'DAILY'

orchestra/contrib/websites/apps.py

@@ -1,7 +1,7 @@
 from django.apps import AppConfig
 from django.contrib.contenttypes.fields import GenericRelation
 
-from orchestra.utils import database_ready
+from orchestra.utils.db import database_ready
 
 
 class WebsiteConfig(AppConfig):

orchestra/utils/__init__.py

@@ -1 +0,0 @@
-from .options import *

orchestra/utils/db.py

@@ -1,6 +1,24 @@
+import sys
+
 from django import db
 
 
+def running_syncdb():
+    return 'migrate' in sys.argv or 'syncdb' in sys.argv or 'makemigrations' in sys.argv
+
+
+def database_ready():
+    return (
+        not running_syncdb() and
+        'setuppostgres' not in sys.argv and
+        'test' not in sys.argv and
+        # Celerybeat has yet to stablish a connection at AppConf.ready()
+        'celerybeat' not in sys.argv and
+        # Allow to run python manage.py without a database
+        sys.argv != ['manage.py'] and '--help' not in sys.argv
+    )
+
+
 def close_connection(execute):
     """ Threads have their own connection pool, closing it when finishing """
     def wrapper(*args, **kwargs):

orchestra/utils/mail.py

@@ -1,4 +1,3 @@
-import sys
 from urllib.parse import urlparse
 
 from django.core.mail import EmailMultiAlternatives
@@ -38,30 +37,3 @@ def send_email_template(template, context, to, email_from=None, html=None, attac
         msg.attach_alternative(html_message, "text/html")
     msg.send()
 
-
-def running_syncdb():
-    return 'migrate' in sys.argv or 'syncdb' in sys.argv or 'makemigrations' in sys.argv
-
-
-def database_ready():
-    return (not running_syncdb() and
-            'setuppostgres' not in sys.argv and
-            'test' not in sys.argv and
-            # Celerybeat has yet to stablish a connection at AppConf.ready()
-            'celerybeat' not in sys.argv and
-            # Allow to run python manage.py without a database
-            sys.argv != ['manage.py'] and '--help' not in sys.argv)
-
-
-def dict_setting_to_choices(choices):
-    return sorted(
-        [ (name, opt.get('verbose_name', 'name')) for name, opt in choices.items() ],
-        key=lambda e: e[0]
-    )
-
-
-def tuple_setting_to_choices(choices):
-    return sorted(
-        tuple((name, opt[0]) for name, opt in choices.items()),
-        key=lambda e: e[0]
-    )