Added mailer

This commit is contained in:
glic3 2015-05-04 21:52:53 +02:00
parent 5b662ba41a
commit f3ec1af691
47 changed files with 559 additions and 344 deletions

61
TODO.md
View File

@ -311,62 +311,36 @@ Replace celery by a custom solution?
*priority: custom Thread backend
*bulk: wrapper arround django-mailer to avoid loading django system
# Create a new virtualenv
python3 -mvenv env-django-orchestra
source env-django-orchestra/bin/activate
pip3 install django-orchestra==dev --allow-external django-orchestra --allow-unverified django-orchestra
# Install dependencies
sudo apt-get install python3.4-dev libxml2-dev libxslt1-dev libcrack2-dev
pip3 install -r https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt
# Create an orchestra instance
orchestra-admin startproject panel
python3 panel/manage.py migrate accounts
python3 panel/manage.py migrate
python3 panel/manage.py runserver
http://localhost:8000/admin/
setupcrontab
pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
Collecting lxml==3.3.5 (from -r re (line 22))
Downloading lxml-3.3.5.tar.gz (3.5MB)
100% |################################| 3.5MB 60kB/s
Building lxml version 3.3.5.
Building without Cython.
ERROR: b'/bin/sh: 1: xslt-config: not found\n'
** make sure the development packages of libxml2 and libxslt are installed **
Using build configuration of libxslt
/usr/lib/python3.4/distutils/dist.py:260: UserWarning: Unknown distribution option: 'bugtrack_url'
warnings.warn(msg)
# TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy
if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT:
import uwsgi
def uwsgi_beat(signum):
print "It's 5 o'clock of the first day of the month."
uwsgi.register_signal(99, '', uwsgi_beat)
uwsgi.add_timer(99, 60)
# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
# SHip orchestra production-ready (no DEBUG etc)
# Setupcron
# uwsgi enable threads
# register signals in app ready()
# database_ready(): connect to the database or inspect django connection
# move Setting to contrib app __init__
# cracklib vs crack
# remove system dependencies
# deprecate install_dependnecies in favour of only requirements.txt
# import module and sed
# if setting.value == default. remove
# TASKS_ENABLE_UWSGI_CRON
# reload generic admin view ?redirect=http...
# inspecting django db connection for asserting db readines?
# inspecting django db connection for asserting db readines? or performing a query
# wake up django mailer on send_mail
# project settings modified copy of django's default project settings
# migrate accounts break on superuser insert because of orders signals: ready() + db_ready()
# all signals + accouns.register() services.register() on apps.py
# if backend.async: don't join.
# RELATED: domains.sync to ns3 make it async
# ngnix setup certificate
from orchestra.contrib.tasks import task
import time, sys
@task(name='rata')
@ -378,10 +352,7 @@ Collecting lxml==3.3.5 (from -r re (line 22))
time.sleep(1)
counter.apply_async(10, '/tmp/kakas')
# setup main systemuser on post_migrate SystemUser
# Provide some fixtures with mocked data
don't make hard dependencies strict dependencies, fail when needed.
# on project_settings add debug settings but commented
TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
@ -389,8 +360,4 @@ TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
TODO mount the filesystem with "nosuid" option
# execute Make after postfix update
# wkhtmltopdf -> reportlab
# MAKE DEPENDENCIES OPTIONAL, check on deploy and warn that functionallity will not be available
# autoiscover modules on app.ready()

View File

@ -95,15 +95,14 @@ def get_administration_items():
task = reverse('admin:djcelery_taskstate_changelist')
periodic = reverse('admin:djcelery_periodictask_changelist')
worker = reverse('admin:djcelery_workerstate_changelist')
childrens.append(items.MenuItem(_("Celery"), task, children=[
items.MenuItem(_("Tasks"), task),
childrens.append(items.MenuItem(_("Tasks"), task, children=[
items.MenuItem(_("Logs"), task),
items.MenuItem(_("Periodic tasks"), periodic),
items.MenuItem(_("Workers"), worker),
]))
return childrens
class OrchestraMenu(Menu):
template = 'admin/orchestra/menu.html'

View File

@ -1,7 +1,8 @@
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.contrib.settings import Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
ACCOUNTS_TYPES = Setting('ACCOUNTS_TYPES',

View File

@ -1,7 +1,8 @@
from django.conf import settings
from django_countries import data
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.contrib.settings import Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
BILLS_NUMBER_LENGTH = Setting('BILLS_NUMBER_LENGTH',

View File

@ -1,6 +1,6 @@
from django_countries import data
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
CONTACTS_DEFAULT_EMAIL_USAGES = Setting('CONTACTS_DEFAULT_EMAIL_USAGES',

View File

@ -1,6 +1,6 @@
from orchestra.core.validators import validate_hostname
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
DATABASES_TYPE_CHOICES = Setting('DATABASES_TYPE_CHOICES',

View File

@ -1,5 +1,6 @@
from orchestra.contrib.settings import Setting
from orchestra.core.validators import validate_ipv4_address, validate_ipv6_address, validate_ip_address
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
from .validators import validate_zone_interval, validate_mx_record, validate_domain_name

View File

@ -1,6 +1,7 @@
from django.core.validators import validate_email
from orchestra.settings import Setting, ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL
from orchestra.contrib.settings import Setting
from orchestra.settings import ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL
ISSUES_SUPPORT_EMAILS = Setting('ISSUES_SUPPORT_EMAILS',

View File

@ -1,4 +1,5 @@
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.contrib.settings import Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
LISTS_DOMAIN_MODEL = Setting('LISTS_DOMAIN_MODEL',

View File

@ -5,8 +5,9 @@ from django.utils.functional import lazy
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.settings import Setting
from orchestra.core.validators import validate_name
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
_names = ('name', 'username',)

View File

View File

@ -0,0 +1,23 @@
from django.contrib import admin
from orchestra.admin.utils import admin_link
from .models import Message, SMTPLog
class MessageAdmin(admin.ModelAdmin):
list_display = (
'id', 'state', 'priority', 'to_address', 'from_address', 'created_at', 'retries', 'last_retry'
)
class SMTPLogAdmin(admin.ModelAdmin):
list_display = (
'id', 'message_link', 'result', 'date', 'log_message'
)
message_link = admin_link('message')
admin.site.register(Message, MessageAdmin)
admin.site.register(SMTPLog, SMTPLogAdmin)

View File

@ -0,0 +1,30 @@
from django.core.mail.backends.base import BaseEmailBackend
from .models import Message
from .tasks import send_message
class EmailBackend(BaseEmailBackend):
'''
A wrapper that manages a queued SMTP system.
'''
def send_messages(self, email_messages):
if not email_messages:
return
num_sent = 0
for message in email_messages:
priority = message.extra_headers.get('X-Mail-Priority', Message.NORMAL)
if priority == Message.CRITICAL:
send_message(message).apply_async()
else:
content = message.message().as_string()
for to_email in message.recipients():
message = Message.objects.create(
priority=priority,
to_address=to_email,
from_address=message.from_email,
subject=message.subject,
content=content,
)
num_sent += 1
return num_sent

View File

@ -0,0 +1,52 @@
import smtplib
from socket import error as SocketError
from django.core.mail import get_connection
from django.utils.encoding import smart_str
from .models import Message
def send_message(message, num, connection, bulk):
if num >= bulk:
connection.close()
connection = None
if connection is None:
# Reset connection
connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend')
connection.open()
error = None
try:
connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content))
except (SocketError, smtplib.SMTPSenderRefused,
smtplib.SMTPRecipientsRefused,
smtplib.SMTPAuthenticationError) as err:
message.defer()
error = err
else:
message.sent()
message.log(error)
def send_pending(bulk=100):
# TODO aquire lock
connection = None
num = 0
for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'):
send_message(message, num, connection, bulk)
from django.utils import timezone
from . import settings
from datetime import timedelta
from django.db.models import Q
now = timezone.now()
qs = Q()
for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
delta = timedelta(seconds=seconds)
qs = qs | Q(retries=retries, last_retry__lte=now-delta)
for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'):
send_message(message, num, connection, bulk)
if connection is not None:
connection.close()

View File

@ -0,0 +1,11 @@
import json
from django.core.management.base import BaseCommand, CommandError
from ...engine import send_pending
class Command(BaseCommand):
help = 'Runs Orchestra method.'
def handle(self, *args, **options):
send_pending()

View File

@ -0,0 +1,68 @@
from django.db import models
from django.utils.translation import ugettext_lazy as _
from . import settings
class Message(models.Model):
QUEUED = 'QUEUED'
SENT = 'SENT'
DEFERRED = 'DEFERRED'
FAILED = 'FAILED'
STATES = (
(QUEUED, _("Queued")),
(SENT, _("Sent")),
(DEFERRED, _("Deferred")),
(FAILED, _("Failes")),
)
CRITICAL = '0'
HIGH = '1'
NORMAL = '2'
LOW = '3'
PRIORITIES = (
(CRITICAL, _("Critical (not queued)")),
(HIGH, _("High")),
(NORMAL, _("Normal")),
(LOW, _("Low")),
)
state = models.CharField(_("State"), max_length=16, choices=STATES, default=QUEUED)
priority = models.PositiveIntegerField(_("Priority"), choices=PRIORITIES, default=NORMAL)
to_address = models.CharField(max_length=256)
from_address = models.CharField(max_length=256)
subject = models.CharField(max_length=256)
content = models.TextField()
created_at = models.DateTimeField(auto_now_add=True)
retries = models.PositiveIntegerField(default=0)
last_retry = models.DateTimeField(auto_now=True)
def defer(self):
self.state = self.DEFERRED
# Max tries
if self.retries >= len(settings.MAILER_DEFERE_SECONDS):
self.state = self.FAILED
self.save(update_fields=('state', 'retries'))
def sent(self):
self.state = self.SENT
self.save(update_fields=('state',))
def log(self, error):
result = SMTPLog.SUCCESS
if error:
result= SMTPLog.FAILURE
self.logs.create(log_message=str(error), result=result)
class SMTPLog(models.Model):
SUCCESS = 'SUCCESS'
FAILURE = 'FAILURE'
RESULTS = (
(SUCCESS, _("Success")),
(FAILURE, _("Failure")),
)
message = models.ForeignKey(Message, editable=False, related_name='logs')
result = models.CharField(max_length=16, choices=RESULTS, default=SUCCESS)
date = models.DateTimeField(auto_now_add=True)
log_message = models.TextField()

View File

@ -0,0 +1,6 @@
from orchestra.contrib.settings import Setting
MAILER_DEFERE_SECONDS = Setting('MAILER_DEFERE_SECONDS',
(300, 600, 60*60, 60*60*24),
)

View File

@ -0,0 +1,6 @@
def send_message():
pass
def cleanup_messages():
pass

View File

@ -1,4 +1,4 @@
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
MISCELLANEOUS_IDENTIFIER_VALIDATORS = Setting('MISCELLANEOUS_IDENTIFIER_VALIDATORS',

View File

@ -1,6 +1,6 @@
from os import path
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
ORCHESTRATION_OS_CHOICES = Setting('ORCHESTRATION_OS_CHOICES',

View File

@ -1,4 +1,4 @@
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
ORDERS_BILLING_BACKEND = Setting('ORDERS_BILLING_BACKEND',

View File

@ -1,4 +1,4 @@
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
from .. import payments

View File

@ -1,4 +1,4 @@
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
RESOURCES_TASK_BACKEND = Setting('RESOURCES_TASK_BACKEND',

View File

@ -0,0 +1 @@
default_app_config = 'orchestra.contrib.saas.apps.SaaSConfig'

View File

@ -0,0 +1,15 @@
from django.apps import AppConfig
from orchestra.core import services
class SaaSConfig(AppConfig):
name = 'orchestra.contrib.saas'
verbose_name = 'Saas'
def ready(self):
from . import signals
from .models import SaaS
services.register(SaaS)

View File

@ -1,11 +1,9 @@
from django.db import models
from django.db.models.signals import pre_save, pre_delete
from django.dispatch import receiver
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from jsonfield import JSONField
from orchestra.core import services, validators
from orchestra.core import validators
from .fields import VirtualDatabaseRelation
from .services import SoftwareService
@ -73,23 +71,3 @@ class SaaS(models.Model):
def set_password(self, password):
self.password = password
services.register(SaaS)
# Admin bulk deletion doesn't call model.delete()
# So, signals are used instead of model method overriding
@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save')
def type_save(sender, *args, **kwargs):
instance = kwargs['instance']
instance.service_instance.save()
@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete')
def type_delete(sender, *args, **kwargs):
instance = kwargs['instance']
try:
instance.service_instance.delete()
except KeyError:
pass

View File

@ -1,4 +1,5 @@
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.contrib.settings import Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
from .. import saas

View File

@ -0,0 +1,21 @@
from django.db.models.signals import pre_save, pre_delete
from django.dispatch import receiver
from .models import SaaS
# Admin bulk deletion doesn't call model.delete()
# So, signals are used instead of model method overriding
@receiver(pre_save, sender=SaaS, dispatch_uid='saas.service.save')
def type_save(sender, *args, **kwargs):
instance = kwargs['instance']
instance.service_instance.save()
@receiver(pre_delete, sender=SaaS, dispatch_uid='saas.service.delete')
def type_delete(sender, *args, **kwargs):
instance = kwargs['instance']
try:
instance.service_instance.delete()
except KeyError:
pass

View File

@ -1,6 +1,6 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
SERVICES_SERVICE_TAXES = Setting('SERVICES_SERVICE_TAXES',

View File

@ -0,0 +1,101 @@
import re
import sys
from collections import OrderedDict
from django.conf import settings
from django.core.exceptions import ValidationError
from django.utils.functional import Promise
from django.utils.translation import ugettext_lazy as _
from orchestra.core import validators
from orchestra.utils.python import import_class, format_exception
default_app_config = 'orchestra.contrib.settings.apps.SettingsConfig'
class Setting(object):
"""
Keeps track of the defined settings and provides extra batteries like value validation.
"""
conf_settings = settings
settings = OrderedDict()
def __str__(self):
return self.name
def __repr__(self):
value = str(self.value)
value = ("'%s'" if isinstance(value, str) else '%s') % value
return '<%s: %s>' % (self.name, value)
def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True,
multiple=False, validators=[], types=[], call_init=False):
if call_init:
return super(Setting, cls).__new__(cls)
cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable,
serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True)
return cls.get_value(name, default)
def __init__(self, *args, **kwargs):
self.name, self.default = args
for name, value in kwargs.items():
setattr(self, name, value)
self.value = self.get_value(self.name, self.default)
self.settings[name] = self
@classmethod
def validate_choices(cls, value):
if not isinstance(value, (list, tuple)):
raise ValidationError("%s is not a valid choices." % str(value))
for choice in value:
if not isinstance(choice, (list, tuple)) or len(choice) != 2:
raise ValidationError("%s is not a valid choice." % str(choice))
value, verbose = choice
if not isinstance(verbose, (str, Promise)):
raise ValidationError("%s is not a valid verbose name." % value)
@classmethod
def validate_import_class(cls, value):
try:
import_class(value)
except Exception as exc:
raise ValidationError(format_exception(exc))
@classmethod
def validate_model_label(cls, value):
from django.apps import apps
try:
apps.get_model(*value.split('.'))
except Exception as exc:
raise ValidationError(format_exception(exc))
@classmethod
def string_format_validator(cls, names, modulo=True):
def validate_string_format(value, names=names, modulo=modulo):
errors = []
regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}'
for n in re.findall(regex, value):
if n not in names:
errors.append(
ValidationError('%s is not a valid format name.' % n)
)
if errors:
raise ValidationError(errors)
return validate_string_format
def validate_value(self, value):
if value:
validators.all_valid(value, self.validators)
valid_types = list(self.types)
if isinstance(self.default, (list, tuple)):
valid_types.extend([list, tuple])
valid_types.append(type(self.default))
if not isinstance(value, tuple(valid_types)):
raise ValidationError("%s is not a valid type (%s)." %
(type(value).__name__, ', '.join(t.__name__ for t in valid_types))
)
@classmethod
def get_value(cls, name, default):
return getattr(cls.conf_settings, name, default)

View File

@ -7,7 +7,7 @@ from django.views import generic
from django.utils.translation import ngettext, ugettext_lazy as _
from orchestra.admin.dashboard import OrchestraIndexDashboard
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
from orchestra.utils import sys, paths
from . import parser

View File

@ -0,0 +1,21 @@
from django.apps import AppConfig
from django.core.checks import register, Error
from django.core.exceptions import ValidationError
from . import Setting
class SettingsConfig(AppConfig):
name = 'orchestra.contrib.settings'
verbose_name = 'Settings'
@register()
def check_settings(app_configs, **kwargs):
""" perfroms all the validation """
messages = []
for name, setting in Setting.settings.items():
try:
setting.validate_value(setting.value)
except ValidationError as exc:
msg = "Error validating setting with value %s: %s" % (setting.value, str(exc))
messages.append(Error(msg, obj=name, id='settings.E001'))
return messages

View File

@ -1,6 +1,6 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
_names = ('user', 'username')

View File

@ -1,107 +1,4 @@
import traceback
from functools import partial, wraps, update_wrapper
from multiprocessing import Process
from uuid import uuid4
from threading import Thread
import sys
from celery import shared_task as celery_shared_task
from celery import states
from celery.decorators import periodic_task as celery_periodic_task
from django.utils import timezone
from orchestra.utils.db import close_connection
from orchestra.utils.python import AttrDict, OrderedSet
def get_id():
return str(uuid4())
def get_name(fn):
return '.'.join((fn.__module__, fn.__name__))
def keep_state(fn):
""" logs task on djcelery's TaskState model """
@wraps(fn)
def wrapper(task_id, name, *args, **kwargs):
from djcelery.models import TaskState
now = timezone.now()
state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
kwargs=str(kwargs), tstamp=now)
try:
result = fn(*args, **kwargs)
except Exception as exc:
state.state = states.FAILURE
state.traceback = traceback.format_exc()
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return
# TODO send email
else:
state.state = states.SUCCESS
state.result = str(result)
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return result
return wrapper
def apply_async(fn, name=None, method='thread'):
""" replaces celery apply_async """
def inner(fn, name, method, *args, **kwargs):
task_id = get_id()
args = (task_id, name) + args
thread = method(target=fn, args=args, kwargs=kwargs)
thread.start()
# Celery API compat
thread.request = AttrDict(id=task_id)
return thread
if name is None:
name = get_name(fn)
if method == 'thread':
method = Thread
elif method == 'process':
method = Process
else:
raise NotImplementedError("%s concurrency method is not supported." % method)
fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method)
fn.delay = fn.apply_async
return fn
def task(fn=None, **kwargs):
# TODO override this if 'celerybeat' in sys.argv ?
from . import settings
# register task
if fn is None:
name = kwargs.get('name', None)
if settings.TASKS_BACKEND in ('thread', 'process'):
def decorator(fn):
return apply_async(celery_shared_task(**kwargs)(fn), name=name)
return decorator
else:
return celery_shared_task(**kwargs)
fn = update_wraper(partial(celery_shared_task, fn))
if settings.TASKS_BACKEND in ('thread', 'process'):
fn = update_wrapper(apply_async(fn), fn)
return fn
def periodic_task(fn=None, **kwargs):
from . import settings
# register task
if fn is None:
name = kwargs.get('name', None)
if settings.TASKS_BACKEND in ('thread', 'process'):
def decorator(fn):
return apply_async(celery_periodic_task(**kwargs)(fn), name=name)
return decorator
else:
return celery_periodic_task(**kwargs)
fn = update_wraper(celery_periodic_task(fn), fn)
if settings.TASKS_BACKEND in ('thread', 'process'):
name = kwargs.pop('name', None)
fn = update_wrapper(apply_async(fn, name), fn)
return fn
from .decorators import task, periodic_task, keep_state, apply_async

View File

@ -5,7 +5,7 @@ from celery.schedules import crontab_parser as CrontabParser
from django.utils import timezone
from djcelery.models import PeriodicTask
from . import apply_async
from .decorators import apply_async
def is_due(task, time=None):

View File

@ -0,0 +1,100 @@
import traceback
from functools import partial, wraps, update_wrapper
from multiprocessing import Process
from threading import Thread
from celery import shared_task as celery_shared_task
from celery import states
from celery.decorators import periodic_task as celery_periodic_task
from django.utils import timezone
from orchestra.utils.db import close_connection
from orchestra.utils.python import AttrDict, OrderedSet
from .utils import get_name, get_id
def keep_state(fn):
""" logs task on djcelery's TaskState model """
@wraps(fn)
def wrapper(task_id, name, *args, **kwargs):
from djcelery.models import TaskState
now = timezone.now()
state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
kwargs=str(kwargs), tstamp=now)
try:
result = fn(*args, **kwargs)
except Exception as exc:
state.state = states.FAILURE
state.traceback = traceback.format_exc()
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return
# TODO send email
else:
state.state = states.SUCCESS
state.result = str(result)
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return result
return wrapper
def apply_async(fn, name=None, method='thread'):
""" replaces celery apply_async """
def inner(fn, name, method, *args, **kwargs):
task_id = get_id()
args = (task_id, name) + args
thread = method(target=fn, args=args, kwargs=kwargs)
thread.start()
# Celery API compat
thread.request = AttrDict(id=task_id)
return thread
if name is None:
name = get_name(fn)
if method == 'thread':
method = Thread
elif method == 'process':
method = Process
else:
raise NotImplementedError("%s concurrency method is not supported." % method)
fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method)
fn.delay = fn.apply_async
return fn
def task(fn=None, **kwargs):
# TODO override this if 'celerybeat' in sys.argv ?
from . import settings
# register task
if fn is None:
name = kwargs.get('name', None)
if settings.TASKS_BACKEND in ('thread', 'process'):
def decorator(fn):
return apply_async(celery_shared_task(**kwargs)(fn), name=name)
return decorator
else:
return celery_shared_task(**kwargs)
fn = update_wraper(partial(celery_shared_task, fn))
if settings.TASKS_BACKEND in ('thread', 'process'):
fn = update_wrapper(apply_async(fn), fn)
return fn
def periodic_task(fn=None, **kwargs):
from . import settings
# register task
if fn is None:
name = kwargs.get('name', None)
if settings.TASKS_BACKEND in ('thread', 'process'):
def decorator(fn):
return apply_async(celery_periodic_task(**kwargs)(fn), name=name)
return decorator
else:
return celery_periodic_task(**kwargs)
fn = update_wraper(celery_periodic_task(fn), fn)
if settings.TASKS_BACKEND in ('thread', 'process'):
name = kwargs.pop('name', None)
fn = update_wrapper(apply_async(fn, name), fn)
return fn

View File

@ -5,7 +5,8 @@ from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from djcelery.models import PeriodicTask
from ... import keep_state, get_id, get_name
from ...decorators import keep_state
from ...utils import get_id, get_name
class Command(BaseCommand):

View File

@ -1,4 +1,4 @@
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
TASKS_BACKEND = Setting('TASKS_BACKEND',
@ -9,3 +9,9 @@ TASKS_BACKEND = Setting('TASKS_BACKEND',
('celery', "Celery (with queue)"),
)
)
TASKS_ENABLE_UWSGI_CRON_BEAT = Setting('TASKS_ENABLE_UWSGI_CRON_BEAT',
False,
help_text="Not implemented.",
)

View File

@ -1,9 +1,16 @@
import threading
from uuid import uuid4
from orchestra.utils.db import close_connection
# TODO import as_task
def get_id():
return str(uuid4())
def get_name(fn):
return '.'.join((fn.__module__, fn.__name__))
def run(method, *args, **kwargs):
async = kwargs.pop('async', True)

View File

@ -1,4 +1,4 @@
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
VPS_TYPES = Setting('VPS_TYPES',

View File

@ -0,0 +1 @@
default_app_config = 'orchestra.contrib.webapps.apps.WebAppsConfig'

View File

@ -0,0 +1,13 @@
from django.apps import AppConfig
from orchestra.core import services
class WebAppsConfig(AppConfig):
name = 'orchestra.contrib.webapps'
verbose_name = 'Webapps'
def ready(self):
from . import signals
from .models import WebApp
services.register(WebApp)

View File

@ -2,13 +2,11 @@ import os
from collections import OrderedDict
from django.db import models
from django.db.models.signals import pre_save, pre_delete
from django.dispatch import receiver
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from jsonfield import JSONField
from orchestra.core import validators, services
from orchestra.core import validators
from orchestra.utils.functional import cached
from . import settings
@ -121,23 +119,3 @@ class WebAppOption(models.Model):
def clean(self):
self.option_instance.validate()
services.register(WebApp)
# Admin bulk deletion doesn't call model.delete()
# So, signals are used instead of model method overriding
@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save')
def type_save(sender, *args, **kwargs):
instance = kwargs['instance']
instance.type_instance.save()
@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete')
def type_delete(sender, *args, **kwargs):
instance = kwargs['instance']
try:
instance.type_instance.delete()
except KeyError:
pass

View File

@ -1,4 +1,5 @@
from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
from orchestra.contrib.settings import Setting
from orchestra.settings import ORCHESTRA_BASE_DOMAIN
from .. import webapps

View File

@ -0,0 +1,22 @@
from django.db.models.signals import pre_save, pre_delete
from django.dispatch import receiver
from .models import WebApp
# Admin bulk deletion doesn't call model.delete()
# So, signals are used instead of model method overriding
@receiver(pre_save, sender=WebApp, dispatch_uid='webapps.type.save')
def type_save(sender, *args, **kwargs):
instance = kwargs['instance']
instance.type_instance.save()
@receiver(pre_delete, sender=WebApp, dispatch_uid='webapps.type.delete')
def type_delete(sender, *args, **kwargs):
instance = kwargs['instance']
try:
instance.type_instance.delete()
except KeyError:
pass

View File

@ -1,6 +1,6 @@
from django.utils.translation import ugettext_lazy as _
from orchestra.settings import Setting
from orchestra.contrib.settings import Setting
from .. import websites

View File

@ -1,124 +1,7 @@
import re
import sys
from collections import OrderedDict
from django.conf import settings
from django.core.checks import register, Error
from django.core.exceptions import ValidationError, AppRegistryNotReady
from django.core.validators import validate_email
from django.db.models import get_model
from django.utils.functional import Promise
from django.utils.translation import ugettext_lazy as _
from orchestra.utils.python import import_class, format_exception
from .core import validators
class Setting(object):
"""
Keeps track of the defined settings and provides extra batteries like value validation.
"""
conf_settings = settings
settings = OrderedDict()
def __str__(self):
return self.name
def __repr__(self):
value = str(self.value)
value = ("'%s'" if isinstance(value, str) else '%s') % value
return '<%s: %s>' % (self.name, value)
def __new__(cls, name, default, help_text="", choices=None, editable=True, serializable=True,
multiple=False, validators=[], types=[], call_init=False):
if call_init:
return super(Setting, cls).__new__(cls)
cls.settings[name] = cls(name, default, help_text=help_text, choices=choices, editable=editable,
serializable=serializable, multiple=multiple, validators=validators, types=types, call_init=True)
return cls.get_value(name, default)
def __init__(self, *args, **kwargs):
self.name, self.default = args
for name, value in kwargs.items():
setattr(self, name, value)
self.value = self.get_value(self.name, self.default)
self.settings[name] = self
@classmethod
def validate_choices(cls, value):
if not isinstance(value, (list, tuple)):
raise ValidationError("%s is not a valid choices." % str(value))
for choice in value:
if not isinstance(choice, (list, tuple)) or len(choice) != 2:
raise ValidationError("%s is not a valid choice." % str(choice))
value, verbose = choice
if not isinstance(verbose, (str, Promise)):
raise ValidationError("%s is not a valid verbose name." % value)
@classmethod
def validate_import_class(cls, value):
try:
import_class(value)
except ImportError as exc:
if "cannot import name 'settings'" in str(exc):
# circular dependency on init time
pass
except Exception as exc:
raise ValidationError(format_exception(exc))
@classmethod
def validate_model_label(cls, value):
try:
get_model(*value.split('.'))
except AppRegistryNotReady:
# circular dependency on init time
pass
except Exception as exc:
raise ValidationError(format_exception(exc))
@classmethod
def string_format_validator(cls, names, modulo=True):
def validate_string_format(value, names=names, modulo=modulo):
errors = []
regex = r'%\(([^\)]+)\)' if modulo else r'{([^}]+)}'
for n in re.findall(regex, value):
if n not in names:
errors.append(
ValidationError('%s is not a valid format name.' % n)
)
if errors:
raise ValidationError(errors)
return validate_string_format
def validate_value(self, value):
if value:
validators.all_valid(value, self.validators)
valid_types = list(self.types)
if isinstance(self.default, (list, tuple)):
valid_types.extend([list, tuple])
valid_types.append(type(self.default))
if not isinstance(value, tuple(valid_types)):
raise ValidationError("%s is not a valid type (%s)." %
(type(value).__name__, ', '.join(t.__name__ for t in valid_types))
)
@classmethod
def get_value(cls, name, default):
return getattr(cls.conf_settings, name, default)
@register()
def check_settings(app_configs, **kwargs):
""" perfroms all the validation """
messages = []
for name, setting in Setting.settings.items():
try:
setting.validate_value(setting.value)
except ValidationError as exc:
msg = "Error validating setting with value %s: %s" % (setting.value, str(exc))
messages.append(Error(msg, obj=name, id='settings.E001'))
return messages
from orchestra.contrib.settings import Setting
ORCHESTRA_BASE_DOMAIN = Setting('ORCHESTRA_BASE_DOMAIN',