Added mailer

TODO.md

@@ -311,62 +311,36 @@ Replace celery by a custom solution?
             *priority: custom Thread backend
             *bulk: wrapper arround django-mailer to avoid loading django system
 
-# Create a new virtualenv
-python3 -mvenv env-django-orchestra
-source env-django-orchestra/bin/activate
-pip3 install django-orchestra==dev --allow-external django-orchestra --allow-unverified django-orchestra
 
-# Install dependencies
-sudo apt-get install python3.4-dev libxml2-dev libxslt1-dev libcrack2-dev
-pip3 install -r https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt
-
-# Create an orchestra instance
-orchestra-admin startproject panel
-python3 panel/manage.py migrate accounts
-python3 panel/manage.py migrate
-python3 panel/manage.py runserver
-
-http://localhost:8000/admin/
-
-setupcrontab
+pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
 
 
-Collecting lxml==3.3.5 (from -r re (line 22))
-  Downloading lxml-3.3.5.tar.gz (3.5MB)
-    100% |################################| 3.5MB 60kB/s 
-    Building lxml version 3.3.5.
-    Building without Cython.
-    ERROR: b'/bin/sh: 1: xslt-config: not found\n'
-    ** make sure the development packages of libxml2 and libxslt are installed **
-    Using build configuration of libxslt
-    /usr/lib/python3.4/distutils/dist.py:260: UserWarning: Unknown distribution option: 'bugtrack_url'
-      warnings.warn(msg)
+
+# TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy
+    if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT:
+        import uwsgi
+        def uwsgi_beat(signum):
+            print "It's 5 o'clock of the first day of the month."
+        uwsgi.register_signal(99, '', uwsgi_beat)
+        uwsgi.add_timer(99, 60)
+# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
+# SHip orchestra production-ready (no DEBUG etc)
 
 
-# Setupcron
-# uwsgi enable threads
-# register signals in app ready()
-# database_ready(): connect to the database or inspect django connection
 
-# move Setting to contrib app __init__
-# cracklib vs crack
-# remove system dependencies
-# deprecate install_dependnecies in favour of only requirements.txt
 # import module and sed
 # if setting.value == default. remove
-# TASKS_ENABLE_UWSGI_CRON
 # reload generic admin view ?redirect=http...
-# inspecting django db connection for asserting db readines?
+# inspecting django db connection for asserting db readines? or performing a query
 # wake up django mailer on send_mail
 
 # project settings modified copy of django's default project settings
 
-# migrate accounts break on superuser insert because of orders signals: ready() + db_ready()
+# all signals + accouns.register() services.register() on apps.py
 
 # if backend.async: don't join.
 # RELATED: domains.sync to ns3 make it async
 
-# ngnix setup certificate
         from orchestra.contrib.tasks import task
         import time, sys
         @task(name='rata')
@@ -378,10 +352,7 @@ Collecting lxml==3.3.5 (from -r re (line 22))
                 time.sleep(1)
         counter.apply_async(10, '/tmp/kakas')
 
-# setup main systemuser on post_migrate SystemUser
 # Provide some fixtures with mocked data
-don't make hard dependencies strict dependencies, fail when needed.
-# on project_settings add debug settings but commented
 
 
 TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
@@ -389,8 +360,4 @@ TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
 TODO mount the filesystem with "nosuid" option
 # execute Make after postfix update
 # wkhtmltopdf -> reportlab
-
-
-# MAKE DEPENDENCIES OPTIONAL, check on deploy and warn that functionallity will not be available
-
-
+# autoiscover modules on app.ready()

orchestra/admin/menu.py

@@ -95,15 +95,14 @@ def get_administration_items():
         task = reverse('admin:djcelery_taskstate_changelist')
         periodic = reverse('admin:djcelery_periodictask_changelist')
         worker = reverse('admin:djcelery_workerstate_changelist')
-        childrens.append(items.MenuItem(_("Celery"), task, children=[
-            items.MenuItem(_("Tasks"), task),
+        childrens.append(items.MenuItem(_("Tasks"), task, children=[
+            items.MenuItem(_("Logs"), task),
             items.MenuItem(_("Periodic tasks"), periodic),
             items.MenuItem(_("Workers"), worker),
         ]))
     return childrens
 
 
-
 class OrchestraMenu(Menu):
     template = 'admin/orchestra/menu.html'
     

orchestra/contrib/accounts/settings.py

@@ -1,7 +1,8 @@
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.contrib.settings import Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 
 ACCOUNTS_TYPES = Setting('ACCOUNTS_TYPES',

orchestra/contrib/bills/settings.py

@@ -1,7 +1,8 @@
 from django.conf import settings
 from django_countries import data
 
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.contrib.settings import Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 
 BILLS_NUMBER_LENGTH = Setting('BILLS_NUMBER_LENGTH',

orchestra/contrib/contacts/settings.py

@@ -1,6 +1,6 @@
 from django_countries import data
 
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 CONTACTS_DEFAULT_EMAIL_USAGES = Setting('CONTACTS_DEFAULT_EMAIL_USAGES',

orchestra/contrib/databases/settings.py

@@ -1,6 +1,6 @@
 from orchestra.core.validators import validate_hostname
 
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 DATABASES_TYPE_CHOICES = Setting('DATABASES_TYPE_CHOICES',

orchestra/contrib/domains/settings.py

@@ -1,5 +1,6 @@
+from orchestra.contrib.settings import Setting
 from orchestra.core.validators import validate_ipv4_address, validate_ipv6_address, validate_ip_address
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 from .validators import validate_zone_interval, validate_mx_record, validate_domain_name
 

orchestra/contrib/issues/settings.py

@@ -1,6 +1,7 @@
 from django.core.validators import validate_email
 
-from orchestra.settings import Setting, ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL
+from orchestra.contrib.settings import Setting
+from orchestra.settings import ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL
 
 
 ISSUES_SUPPORT_EMAILS = Setting('ISSUES_SUPPORT_EMAILS',

orchestra/contrib/lists/settings.py

@@ -1,4 +1,5 @@
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.contrib.settings import Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 
 LISTS_DOMAIN_MODEL = Setting('LISTS_DOMAIN_MODEL',

orchestra/contrib/mailboxes/settings.py

@@ -5,8 +5,9 @@ from django.utils.functional import lazy
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 
+from orchestra.contrib.settings import Setting
 from orchestra.core.validators import validate_name
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 
 _names = ('name', 'username',)

orchestra/contrib/mailer/admin.py

@@ -0,0 +1,23 @@
+from django.contrib import admin
+
+from orchestra.admin.utils import admin_link
+
+from .models import Message, SMTPLog
+
+
+class MessageAdmin(admin.ModelAdmin):
+    list_display = (
+        'id', 'state', 'priority', 'to_address', 'from_address', 'created_at', 'retries', 'last_retry'
+    )
+
+
+class SMTPLogAdmin(admin.ModelAdmin):
+    list_display = (
+        'id', 'message_link', 'result', 'date', 'log_message'
+    )
+    
+    message_link = admin_link('message')
+
+
+admin.site.register(Message, MessageAdmin)
+admin.site.register(SMTPLog, SMTPLogAdmin)

orchestra/contrib/mailer/backends.py

@@ -0,0 +1,30 @@
+from django.core.mail.backends.base import BaseEmailBackend
+
+from .models import Message
+from .tasks import send_message
+
+
+class EmailBackend(BaseEmailBackend):
+    '''
+    A wrapper that manages a queued SMTP system.
+    '''
+    def send_messages(self, email_messages):
+        if not email_messages:
+            return
+        num_sent = 0
+        for message in email_messages:
+            priority = message.extra_headers.get('X-Mail-Priority', Message.NORMAL)
+            if priority == Message.CRITICAL:
+                send_message(message).apply_async()
+            else:
+                content = message.message().as_string()
+                for to_email in message.recipients():
+                    message = Message.objects.create(
+                        priority=priority,
+                        to_address=to_email,
+                        from_address=message.from_email,
+                        subject=message.subject,
+                        content=content,
+                    )
+            num_sent += 1
+        return num_sent

orchestra/contrib/mailer/engine.py

@@ -0,0 +1,52 @@
+import smtplib
+from socket import error as SocketError
+
+from django.core.mail import get_connection
+from django.utils.encoding import smart_str
+
+from .models import Message
+
+
+def send_message(message, num, connection, bulk):
+    if num >= bulk:
+        connection.close()
+        connection = None
+    if connection is None:
+        # Reset connection
+        connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend')
+        connection.open()
+    error = None
+    try:
+        connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content))
+    except (SocketError, smtplib.SMTPSenderRefused,
+            smtplib.SMTPRecipientsRefused,
+            smtplib.SMTPAuthenticationError) as err:
+        message.defer()
+        error = err
+    else:
+        message.sent()
+    message.log(error)
+
+
+def send_pending(bulk=100):
+    # TODO aquire lock
+    connection = None
+    num = 0
+    for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'):
+        send_message(message, num, connection, bulk)
+    from django.utils import timezone
+    from . import settings
+    from datetime import timedelta
+    from django.db.models import Q
+    
+    now = timezone.now()
+    qs = Q()
+    for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
+        delta = timedelta(seconds=seconds)
+        qs = qs | Q(retries=retries, last_retry__lte=now-delta)
+
+    for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'):
+        send_message(message, num, connection, bulk)
+    if connection is not None:
+        connection.close()
+

orchestra/contrib/mailer/management/commands/send.py

@@ -0,0 +1,11 @@
+import json
+
+from django.core.management.base import BaseCommand, CommandError
+
+from ...engine import send_pending
+
+class Command(BaseCommand):
+    help = 'Runs Orchestra method.'
+    
+    def handle(self, *args, **options):
+        send_pending()

orchestra/contrib/mailer/models.py

@@ -0,0 +1,68 @@
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from . import settings
+
+
+class Message(models.Model):
+    QUEUED = 'QUEUED'
+    SENT = 'SENT'
+    DEFERRED = 'DEFERRED'
+    FAILED = 'FAILED'
+    STATES = (
+        (QUEUED, _("Queued")),
+        (SENT, _("Sent")),
+        (DEFERRED, _("Deferred")),
+        (FAILED, _("Failes")),
+    )
+    
+    CRITICAL = '0'
+    HIGH = '1'
+    NORMAL = '2'
+    LOW = '3'
+    PRIORITIES = (
+        (CRITICAL, _("Critical (not queued)")),
+        (HIGH, _("High")),
+        (NORMAL, _("Normal")),
+        (LOW, _("Low")),
+    )
+    
+    state = models.CharField(_("State"), max_length=16, choices=STATES, default=QUEUED)
+    priority = models.PositiveIntegerField(_("Priority"), choices=PRIORITIES, default=NORMAL)
+    to_address = models.CharField(max_length=256)
+    from_address = models.CharField(max_length=256)
+    subject = models.CharField(max_length=256)
+    content = models.TextField()
+    created_at = models.DateTimeField(auto_now_add=True)
+    retries = models.PositiveIntegerField(default=0)
+    last_retry = models.DateTimeField(auto_now=True)
+    
+    def defer(self):
+        self.state = self.DEFERRED
+        # Max tries
+        if self.retries >= len(settings.MAILER_DEFERE_SECONDS):
+            self.state = self.FAILED
+        self.save(update_fields=('state', 'retries'))
+    
+    def sent(self):
+        self.state = self.SENT
+        self.save(update_fields=('state',))
+    
+    def log(self, error):
+        result = SMTPLog.SUCCESS
+        if error:
+            result= SMTPLog.FAILURE
+        self.logs.create(log_message=str(error), result=result)
+
+
+class SMTPLog(models.Model):
+    SUCCESS = 'SUCCESS'
+    FAILURE = 'FAILURE'
+    RESULTS = (
+        (SUCCESS, _("Success")),
+        (FAILURE, _("Failure")),
+    )
+    message = models.ForeignKey(Message, editable=False, related_name='logs')
+    result = models.CharField(max_length=16, choices=RESULTS, default=SUCCESS)
+    date = models.DateTimeField(auto_now_add=True)
+    log_message = models.TextField()

orchestra/contrib/mailer/settings.py

@@ -0,0 +1,6 @@
+from orchestra.contrib.settings import Setting
+
+
+MAILER_DEFERE_SECONDS = Setting('MAILER_DEFERE_SECONDS',
+    (300, 600, 60*60, 60*60*24),
+)

orchestra/contrib/mailer/tasks.py

@@ -0,0 +1,6 @@
+def send_message():
+    pass
+
+
+def cleanup_messages():
+    pass

orchestra/contrib/miscellaneous/settings.py

@@ -1,4 +1,4 @@
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 MISCELLANEOUS_IDENTIFIER_VALIDATORS = Setting('MISCELLANEOUS_IDENTIFIER_VALIDATORS',

orchestra/contrib/orchestration/settings.py

@@ -1,6 +1,6 @@
 from os import path
 
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 ORCHESTRATION_OS_CHOICES = Setting('ORCHESTRATION_OS_CHOICES',

orchestra/contrib/orders/settings.py

@@ -1,4 +1,4 @@
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 ORDERS_BILLING_BACKEND = Setting('ORDERS_BILLING_BACKEND',

orchestra/contrib/payments/settings.py

@@ -1,4 +1,4 @@
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 from .. import payments
 

orchestra/contrib/resources/settings.py

@@ -1,4 +1,4 @@
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 RESOURCES_TASK_BACKEND = Setting('RESOURCES_TASK_BACKEND',

orchestra/contrib/saas/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'orchestra.contrib.saas.apps.SaaSConfig'

orchestra/contrib/saas/apps.py

@@ -0,0 +1,15 @@
+from django.apps import AppConfig
+
+from orchestra.core import services
+
+
+class SaaSConfig(AppConfig):
+    name = 'orchestra.contrib.saas'
+    verbose_name = 'Saas'
+    
+    def ready(self):
+        from . import signals
+        from .models import SaaS
+        services.register(SaaS)
+
+

orchestra/contrib/saas/models.py

@@ -1,11 +1,9 @@
 from django.db import models
-from django.db.models.signals import pre_save, pre_delete
-from django.dispatch import receiver
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from jsonfield import JSONField
 
-from orchestra.core import services, validators
+from orchestra.core import validators
 
 from .fields import VirtualDatabaseRelation
 from .services import SoftwareService
@@ -73,23 +71,3 @@ class SaaS(models.Model):
     
     def set_password(self, password):
         self.password = password
-
-
-services.register(SaaS)
-
-
-# Admin bulk deletion doesn't call model.delete()
-# So, signals are used instead of model method overriding
-
-@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save')
-def type_save(sender, *args, **kwargs):
-    instance = kwargs['instance']
-    instance.service_instance.save()
-
-@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete')
-def type_delete(sender, *args, **kwargs):
-    instance = kwargs['instance']
-    try:
-        instance.service_instance.delete()
-    except KeyError:
-        pass

orchestra/contrib/saas/settings.py

@@ -1,4 +1,5 @@
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.contrib.settings import Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 from .. import saas
 

orchestra/contrib/saas/signals.py

@@ -0,0 +1,21 @@
+from django.db.models.signals import pre_save, pre_delete
+from django.dispatch import receiver
+
+from .models import SaaS
+
+
+# Admin bulk deletion doesn't call model.delete()
+# So, signals are used instead of model method overriding
+
+@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save')
+def type_save(sender, *args, **kwargs):
+    instance = kwargs['instance']
+    instance.service_instance.save()
+
+@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete')
+def type_delete(sender, *args, **kwargs):
+    instance = kwargs['instance']
+    try:
+        instance.service_instance.delete()
+    except KeyError:
+        pass

orchestra/contrib/services/settings.py

@@ -1,6 +1,6 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 SERVICES_SERVICE_TAXES = Setting('SERVICES_SERVICE_TAXES',

orchestra/contrib/settings/__init__.py

@@ -0,0 +1,101 @@
+import re
+import sys
+from collections import OrderedDict
+
+from django.conf import settings
+from django.core.exceptions import ValidationError
+from django.utils.functional import Promise
+from django.utils.translation import ugettext_lazy as _
+
+from orchestra.core import validators
+from orchestra.utils.python import import_class, format_exception
+
+
+default_app_config = 'orchestra.contrib.settings.apps.SettingsConfig'
+
+
+class Setting(object):
+    """
+    Keeps track of the defined settings and provides extra batteries like value validation.
+    """
+    conf_settings = settings
+    settings = OrderedDict()
+    
+    def __str__(self):
+        return self.name
+    
+    def __repr__(self):
+        value = str(self.value)
+        value = ("'%s'" if isinstance(value, str) else '%s') % value
+        return '<%s: %s>' % (self.name, value)
+    
+    def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True,
+                multiple=False, validators=[], types=[], call_init=False):
+        if call_init:
+            return super(Setting, cls).__new__(cls)
+        cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable,
+            serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True)
+        return cls.get_value(name, default)
+    
+    def __init__(self, *args, **kwargs):
+        self.name, self.default = args
+        for name, value in kwargs.items():
+            setattr(self, name, value)
+        self.value = self.get_value(self.name, self.default)
+        self.settings[name] = self
+    
+    @classmethod
+    def validate_choices(cls, value):
+        if not isinstance(value, (list, tuple)):
+            raise ValidationError("%s is not a valid choices." % str(value))
+        for choice in value:
+            if not isinstance(choice, (list, tuple)) or len(choice) != 2:
+                raise ValidationError("%s is not a valid choice." % str(choice))
+            value, verbose = choice
+            if not isinstance(verbose, (str, Promise)):
+                raise ValidationError("%s is not a valid verbose name." % value)
+    
+    @classmethod
+    def validate_import_class(cls, value):
+        try:
+            import_class(value)
+        except Exception as exc:
+            raise ValidationError(format_exception(exc))
+    
+    @classmethod
+    def validate_model_label(cls, value):
+        from django.apps import apps
+        try:
+            apps.get_model(*value.split('.'))
+        except Exception as exc:
+            raise ValidationError(format_exception(exc))
+    
+    @classmethod
+    def string_format_validator(cls, names, modulo=True):
+        def validate_string_format(value, names=names, modulo=modulo):
+            errors = []
+            regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}'
+            for n in re.findall(regex, value):
+                if n not in names:
+                    errors.append(
+                        ValidationError('%s is not a valid format name.' % n)
+                    )
+            if errors:
+                raise ValidationError(errors)
+        return validate_string_format
+    
+    def validate_value(self, value):
+        if value:
+            validators.all_valid(value, self.validators)
+        valid_types = list(self.types)
+        if isinstance(self.default, (list, tuple)):
+            valid_types.extend([list, tuple])
+        valid_types.append(type(self.default))
+        if not isinstance(value, tuple(valid_types)):
+            raise ValidationError("%s is not a valid type (%s)." %
+                (type(value).__name__, ', '.join(t.__name__ for t in valid_types))
+            )
+    
+    @classmethod
+    def get_value(cls, name, default):
+        return getattr(cls.conf_settings, name, default)

orchestra/contrib/settings/admin.py

@@ -7,7 +7,7 @@ from django.views import generic
 from django.utils.translation import ngettext, ugettext_lazy as _
 
 from orchestra.admin.dashboard import OrchestraIndexDashboard
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 from orchestra.utils import sys, paths
 
 from . import parser

orchestra/contrib/settings/apps.py

@@ -0,0 +1,21 @@
+from django.apps import AppConfig
+from django.core.checks import register, Error
+from django.core.exceptions import ValidationError
+
+from . import Setting
+
+class SettingsConfig(AppConfig):
+    name = 'orchestra.contrib.settings'
+    verbose_name = 'Settings'
+    
+    @register()
+    def check_settings(app_configs, **kwargs):
+        """ perfroms all the validation """
+        messages = []
+        for name, setting in Setting.settings.items():
+            try:
+                setting.validate_value(setting.value)
+            except ValidationError as exc:
+                msg = "Error validating setting with value %s: %s" % (setting.value, str(exc))
+                messages.append(Error(msg, obj=name, id='settings.E001'))
+        return messages

orchestra/contrib/systemusers/settings.py

@@ -1,6 +1,6 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 _names = ('user', 'username')

orchestra/contrib/tasks/__init__.py

@@ -1,107 +1,4 @@
-import traceback
-from functools import partial, wraps, update_wrapper
-from multiprocessing import Process
-from uuid import uuid4
-from threading import Thread
+import sys
 
-from celery import shared_task as celery_shared_task
-from celery import states
-from celery.decorators import periodic_task as celery_periodic_task
-from django.utils import timezone
-
-from orchestra.utils.db import close_connection
-from orchestra.utils.python import AttrDict, OrderedSet
-
-
-def get_id():
-    return str(uuid4())
-
-
-def get_name(fn):
-    return '.'.join((fn.__module__, fn.__name__))
-
-
-def keep_state(fn):
-    """ logs task on djcelery's TaskState model """
-    @wraps(fn)
-    def wrapper(task_id, name, *args, **kwargs):
-        from djcelery.models import TaskState
-        now = timezone.now()
-        state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
-            kwargs=str(kwargs), tstamp=now)
-        try:
-            result = fn(*args, **kwargs)
-        except Exception as exc:
-            state.state = states.FAILURE
-            state.traceback = traceback.format_exc()
-            state.runtime = (timezone.now()-now).total_seconds()
-            state.save()
-            return
-            # TODO send email
-        else:
-            state.state = states.SUCCESS
-            state.result = str(result)
-            state.runtime = (timezone.now()-now).total_seconds()
-            state.save()
-        return result
-    return wrapper
-
-
-def apply_async(fn, name=None, method='thread'):
-    """ replaces celery apply_async """
-    def inner(fn, name, method, *args, **kwargs):
-        task_id = get_id()
-        args = (task_id, name) + args
-        thread = method(target=fn, args=args, kwargs=kwargs)
-        thread.start()
-        # Celery API compat
-        thread.request = AttrDict(id=task_id)
-        return thread
-    
-    if name is None:
-        name = get_name(fn)
-    if method == 'thread':
-        method = Thread
-    elif method == 'process':
-        method = Process
-    else:
-        raise NotImplementedError("%s concurrency method is not supported." % method)
-    fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method)
-    fn.delay = fn.apply_async
-    return fn
-
-
-def task(fn=None, **kwargs):
-    # TODO override this if 'celerybeat' in sys.argv ?
 from . import settings
-    # register task
-    if fn is None:
-        name = kwargs.get('name', None)
-        if settings.TASKS_BACKEND in ('thread', 'process'):
-            def decorator(fn):
-                return apply_async(celery_shared_task(**kwargs)(fn), name=name)
-            return decorator
-        else:
-            return celery_shared_task(**kwargs)
-    fn = update_wraper(partial(celery_shared_task, fn))
-    if settings.TASKS_BACKEND in ('thread', 'process'):
-        fn = update_wrapper(apply_async(fn), fn)
-    return fn
-
-
-def periodic_task(fn=None, **kwargs):
-    from . import settings
-    # register task
-    if fn is None:
-        name = kwargs.get('name', None)
-        if settings.TASKS_BACKEND in ('thread', 'process'):
-            def decorator(fn):
-                return apply_async(celery_periodic_task(**kwargs)(fn), name=name)
-            return decorator
-        else:
-            return celery_periodic_task(**kwargs)
-    fn = update_wraper(celery_periodic_task(fn), fn)
-    if settings.TASKS_BACKEND in ('thread', 'process'):
-        name = kwargs.pop('name', None)
-        fn = update_wrapper(apply_async(fn, name), fn)
-    return fn
+from .decorators import task, periodic_task, keep_state, apply_async

orchestra/contrib/tasks/beat.py

@@ -5,7 +5,7 @@ from celery.schedules import crontab_parser as CrontabParser
 from django.utils import timezone
 from djcelery.models import PeriodicTask
 
-from . import apply_async
+from .decorators import apply_async
 
 
 def is_due(task, time=None):

orchestra/contrib/tasks/decorators.py

@@ -0,0 +1,100 @@
+import traceback
+from functools import partial, wraps, update_wrapper
+from multiprocessing import Process
+from threading import Thread
+
+from celery import shared_task as celery_shared_task
+from celery import states
+from celery.decorators import periodic_task as celery_periodic_task
+from django.utils import timezone
+
+from orchestra.utils.db import close_connection
+from orchestra.utils.python import AttrDict, OrderedSet
+
+from .utils import get_name, get_id
+
+
+def keep_state(fn):
+    """ logs task on djcelery's TaskState model """
+    @wraps(fn)
+    def wrapper(task_id, name, *args, **kwargs):
+        from djcelery.models import TaskState
+        now = timezone.now()
+        state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
+            kwargs=str(kwargs), tstamp=now)
+        try:
+            result = fn(*args, **kwargs)
+        except Exception as exc:
+            state.state = states.FAILURE
+            state.traceback = traceback.format_exc()
+            state.runtime = (timezone.now()-now).total_seconds()
+            state.save()
+            return
+            # TODO send email
+        else:
+            state.state = states.SUCCESS
+            state.result = str(result)
+            state.runtime = (timezone.now()-now).total_seconds()
+            state.save()
+        return result
+    return wrapper
+
+
+def apply_async(fn, name=None, method='thread'):
+    """ replaces celery apply_async """
+    def inner(fn, name, method, *args, **kwargs):
+        task_id = get_id()
+        args = (task_id, name) + args
+        thread = method(target=fn, args=args, kwargs=kwargs)
+        thread.start()
+        # Celery API compat
+        thread.request = AttrDict(id=task_id)
+        return thread
+    
+    if name is None:
+        name = get_name(fn)
+    if method == 'thread':
+        method = Thread
+    elif method == 'process':
+        method = Process
+    else:
+        raise NotImplementedError("%s concurrency method is not supported." % method)
+    fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method)
+    fn.delay = fn.apply_async
+    return fn
+
+
+def task(fn=None, **kwargs):
+    # TODO override this if 'celerybeat' in sys.argv ?
+    from . import settings
+    # register task
+    if fn is None:
+        name = kwargs.get('name', None)
+        if settings.TASKS_BACKEND in ('thread', 'process'):
+            def decorator(fn):
+                return apply_async(celery_shared_task(**kwargs)(fn), name=name)
+            return decorator
+        else:
+            return celery_shared_task(**kwargs)
+    fn = update_wraper(partial(celery_shared_task, fn))
+    if settings.TASKS_BACKEND in ('thread', 'process'):
+        fn = update_wrapper(apply_async(fn), fn)
+    return fn
+
+
+def periodic_task(fn=None, **kwargs):
+    from . import settings
+    # register task
+    if fn is None:
+        name = kwargs.get('name', None)
+        if settings.TASKS_BACKEND in ('thread', 'process'):
+            def decorator(fn):
+                return apply_async(celery_periodic_task(**kwargs)(fn), name=name)
+            return decorator
+        else:
+            return celery_periodic_task(**kwargs)
+    fn = update_wraper(celery_periodic_task(fn), fn)
+    if settings.TASKS_BACKEND in ('thread', 'process'):
+        name = kwargs.pop('name', None)
+        fn = update_wrapper(apply_async(fn, name), fn)
+    return fn

orchestra/contrib/tasks/management/commands/runtask.py

@@ -5,7 +5,8 @@ from django.core.management.base import BaseCommand, CommandError
 from django.utils import timezone
 from djcelery.models import PeriodicTask
 
-from ... import keep_state, get_id, get_name
+from ...decorators import keep_state
+from ...utils import get_id, get_name
 
 
 class Command(BaseCommand):

orchestra/contrib/tasks/settings.py

@@ -1,4 +1,4 @@
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 TASKS_BACKEND = Setting('TASKS_BACKEND',
@@ -9,3 +9,9 @@ TASKS_BACKEND = Setting('TASKS_BACKEND',
         ('celery', "Celery (with queue)"),
     )
 )
+
+
+TASKS_ENABLE_UWSGI_CRON_BEAT = Setting('TASKS_ENABLE_UWSGI_CRON_BEAT',
+    False,
+    help_text="Not implemented.",
+)

orchestra/contrib/tasks/utils.py

@@ -1,9 +1,16 @@
 import threading
+from uuid import uuid4
 
 from orchestra.utils.db import close_connection
 
 
-# TODO import as_task
+def get_id():
+    return str(uuid4())
+
+
+def get_name(fn):
+    return '.'.join((fn.__module__, fn.__name__))
+
 
 def run(method, *args, **kwargs):
     async = kwargs.pop('async', True)

orchestra/contrib/vps/settings.py

@@ -1,4 +1,4 @@
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 
 VPS_TYPES = Setting('VPS_TYPES',

orchestra/contrib/webapps/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'orchestra.contrib.webapps.apps.WebAppsConfig'

orchestra/contrib/webapps/apps.py

@@ -0,0 +1,13 @@
+from django.apps import AppConfig
+
+from orchestra.core import services
+
+
+class WebAppsConfig(AppConfig):
+    name = 'orchestra.contrib.webapps'
+    verbose_name = 'Webapps'
+    
+    def ready(self):
+        from . import signals
+        from .models import WebApp
+        services.register(WebApp)

orchestra/contrib/webapps/models.py

@@ -2,13 +2,11 @@ import os
 from collections import OrderedDict
 
 from django.db import models
-from django.db.models.signals import pre_save, pre_delete
-from django.dispatch import receiver
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from jsonfield import JSONField
 
-from orchestra.core import validators, services
+from orchestra.core import validators
 from orchestra.utils.functional import cached
 
 from . import settings
@@ -121,23 +119,3 @@ class WebAppOption(models.Model):
     
     def clean(self):
         self.option_instance.validate()
-
-
-services.register(WebApp)
-
-
-# Admin bulk deletion doesn't call model.delete()
-# So, signals are used instead of model method overriding
-
-@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save')
-def type_save(sender, *args, **kwargs):
-    instance = kwargs['instance']
-    instance.type_instance.save()
-
-@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete')
-def type_delete(sender, *args, **kwargs):
-    instance = kwargs['instance']
-    try:
-        instance.type_instance.delete()
-    except KeyError:
-        pass

orchestra/contrib/webapps/settings.py

@@ -1,4 +1,5 @@
-from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
+from orchestra.contrib.settings import Setting
+from orchestra.settings import ORCHESTRA_BASE_DOMAIN
 
 from .. import webapps
 

orchestra/contrib/webapps/signals.py

@@ -0,0 +1,22 @@
+from django.db.models.signals import pre_save, pre_delete
+from django.dispatch import receiver
+
+from .models import WebApp
+
+
+# Admin bulk deletion doesn't call model.delete()
+# So, signals are used instead of model method overriding
+
+@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save')
+def type_save(sender, *args, **kwargs):
+    instance = kwargs['instance']
+    instance.type_instance.save()
+
+
+@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete')
+def type_delete(sender, *args, **kwargs):
+    instance = kwargs['instance']
+    try:
+        instance.type_instance.delete()
+    except KeyError:
+        pass

orchestra/contrib/websites/settings.py

@@ -1,6 +1,6 @@
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.settings import Setting
+from orchestra.contrib.settings import Setting
 
 from .. import websites
 

orchestra/settings.py

@@ -1,124 +1,7 @@
-import re
-import sys
-from collections import OrderedDict
-
-from django.conf import settings
-from django.core.checks import register, Error
-from django.core.exceptions import ValidationError, AppRegistryNotReady
 from django.core.validators import validate_email
-from django.db.models import get_model
-from django.utils.functional import Promise
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.utils.python import import_class, format_exception
-
-from .core import validators
-
-
-class Setting(object):
-    """
-    Keeps track of the defined settings and provides extra batteries like value validation.
-    """
-    conf_settings = settings
-    settings = OrderedDict()
-    
-    def __str__(self):
-        return self.name
-    
-    def __repr__(self):
-        value = str(self.value)
-        value = ("'%s'" if isinstance(value, str) else '%s') % value
-        return '<%s: %s>' % (self.name, value)
-    
-    def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True,
-                multiple=False, validators=[], types=[], call_init=False):
-        if call_init:
-            return super(Setting, cls).__new__(cls)
-        cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable,
-            serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True)
-        return cls.get_value(name, default)
-    
-    def __init__(self, *args, **kwargs):
-        self.name, self.default = args
-        for name, value in kwargs.items():
-            setattr(self, name, value)
-        self.value = self.get_value(self.name, self.default)
-        self.settings[name] = self
-    
-    @classmethod
-    def validate_choices(cls, value):
-        if not isinstance(value, (list, tuple)):
-            raise ValidationError("%s is not a valid choices." % str(value))
-        for choice in value:
-            if not isinstance(choice, (list, tuple)) or len(choice) != 2:
-                raise ValidationError("%s is not a valid choice." % str(choice))
-            value, verbose = choice
-            if not isinstance(verbose, (str, Promise)):
-                raise ValidationError("%s is not a valid verbose name." % value)
-    
-    @classmethod
-    def validate_import_class(cls, value):
-        try:
-            import_class(value)
-        except ImportError as exc:
-            if "cannot import name 'settings'" in str(exc):
-                # circular dependency on init time
-                pass
-        except Exception as exc:
-            raise ValidationError(format_exception(exc))
-    
-    @classmethod
-    def validate_model_label(cls, value):
-        try:
-            get_model(*value.split('.'))
-        except AppRegistryNotReady:
-            # circular dependency on init time
-            pass
-        except Exception as exc:
-            raise ValidationError(format_exception(exc))
-    
-    @classmethod
-    def string_format_validator(cls, names, modulo=True):
-        def validate_string_format(value, names=names, modulo=modulo):
-            errors = []
-            regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}'
-            for n in re.findall(regex, value):
-                if n not in names:
-                    errors.append(
-                        ValidationError('%s is not a valid format name.' % n)
-                    )
-            if errors:
-                raise ValidationError(errors)
-        return validate_string_format
-    
-    def validate_value(self, value):
-        if value:
-            validators.all_valid(value, self.validators)
-        valid_types = list(self.types)
-        if isinstance(self.default, (list, tuple)):
-            valid_types.extend([list, tuple])
-        valid_types.append(type(self.default))
-        if not isinstance(value, tuple(valid_types)):
-            raise ValidationError("%s is not a valid type (%s)." %
-                (type(value).__name__, ', '.join(t.__name__ for t in valid_types))
-            )
-    
-    @classmethod
-    def get_value(cls, name, default):
-        return getattr(cls.conf_settings, name, default)
-
-
-@register()
-def check_settings(app_configs, **kwargs):
-    """ perfroms all the validation """
-    messages = []
-    for name, setting in Setting.settings.items():
-        try:
-            setting.validate_value(setting.value)
-        except ValidationError as exc:
-            msg = "Error validating setting with value %s: %s" % (setting.value, str(exc))
-            messages.append(Error(msg, obj=name, id='settings.E001'))
-    return messages
+from orchestra.contrib.settings import Setting
 
 
 ORCHESTRA_BASE_DOMAIN = Setting('ORCHESTRA_BASE_DOMAIN',