Reandom fixes

INSTALL.md

@@ -53,7 +53,8 @@ Django-orchestra can be installed on any Linux system, however it is **strongly
 6. Configure periodic execution of tasks (choose one)
     1. Use cron
         ```bash
-        sudo python3 manage.py setupcronbeat
+        python3 manage.py setupcronbeat
+        python3 panel/manage.py syncperiodictasks
         ```
 
     2. Use celeryd

README.md

@@ -55,6 +55,7 @@ python3 panel/manage.py runserver
 
 # Enable periodic tasks with cron (optional)
 python3 panel/manage.py setupcronbeat
+python3 panel/manage.py syncperiodictasks
 ```
 
 Now you can see the web interface on http://localhost:8000/admin

TODO.md

@@ -288,9 +288,9 @@ https://code.djangoproject.com/ticket/24576
 # insert settings on dashboard dynamically
 
 # convert all complex settings to string
-# @ something database names
+# size monitor of @002 @003 database names
 # password validation cracklib on change password form=?????
-# reset setting buton 
+# reset setting button 
 
 # periodic cleaning of spam mailboxes
 
@@ -298,21 +298,6 @@ https://code.djangoproject.com/ticket/24576
 # django SITE_NAME vs ORCHESTRA_SITE_NAME ?
 
 
-Replace celery by a custom solution?
-    # TODO create decorator wrapper that abstract the task away from the backen (cron/celery)
-    # TODO crontab model localhost/autoadded attribute
-    * No more jumbo dependencies and wierd bugs
-    1) Periodic Monitoring:
-        * runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system)
-    2) Single time shot:
-        sys.run("python3 manage.py runtas 'task' args")
-    3) Emails:
-        Custom backend that distinguishes between priority and bulk mail
-            *priority: custom Thread backend
-            *bulk: wrapper arround django-mailer to avoid loading django system
-
-pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
-
 # TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy
     if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT:
         import uwsgi
@@ -321,7 +306,7 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
         uwsgi.register_signal(99, '', uwsgi_beat)
         uwsgi.add_timer(99, 60)
 # TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
-# SHip orchestra production-ready (no DEBUG etc)
+# Ship orchestra production-ready (no DEBUG etc)
 
 # import module and sed
 # if setting.value == default. remove
@@ -329,8 +314,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
 # inspecting django db connection for asserting db readines? or performing a query
 # wake up django mailer on send_mail
 
-# project settings modified copy of django's default project settings
-
 # all signals + accouns.register() services.register() on apps.py
 
         from orchestra.contrib.tasks import task
@@ -350,44 +333,23 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
 TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
 TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
 TODO mount the filesystem with "nosuid" option
-# wkhtmltopdf -> reportlab
+
-# autoiscover modules on app.ready() ? lazy choices on models for plugins
-# ModelTranslation.register on app.ready()
 # uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
 
 # mailboxes.address settings multiple local domains, not only one?
-# backend.context = self.get_context() or save(obj, context=None)
+# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data
 
 # smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
 
-# create registered periodic_task  on beat execution: and management command: syncperiodictasks
-
-# MERGE beats and inspect INSTALLED_APPS and get IS_ENABLED
-
 # rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
 # virtdomains file is not ideal, prevent fake/error on domains there! and make sure this file is required!
 
-# Message last_retry auto_now doesn't work!
-
-# don't block on beat, and --report periodic tasks
-
 # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
-
 # orchestrate async stdout stderr (inspired on pangea managemengt commands)
-
 # orchestra-beat support for uwsgi cron
 
-# message.log if 1: return changeform
+# message.log if len() == 1: return changeform
-
-# generate nginx certs on project dir rather than nginx
-
-# Register icons
-
-# send_message doesn't log task
 
 make django admin taskstate uncollapse fucking traceback, ( if exists ?)
 
-# receive tass stuck at RECEIVED	
+# form for custom message on admin save "comment & save"?
-# monitor tasks in started and backend already in success?
-
-# custom message on admin save

orchestra/admin/dashboard.py

@@ -1,12 +1,13 @@
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext_lazy as _
-from fluent_dashboard import dashboard
+from fluent_dashboard import dashboard, appsettings
 from fluent_dashboard.modules import CmsAppIconList
 
 from orchestra.core import services, accounts, administration
 
 
 class AppDefaultIconList(CmsAppIconList):
+    """ Provides support for custom default icons """
     def __init__(self, *args, **kwargs):
         self.icons = kwargs.pop('icons')
         super(AppDefaultIconList, self).__init__(*args, **kwargs)
@@ -17,25 +18,28 @@ class AppDefaultIconList(CmsAppIconList):
 
 
 class OrchestraIndexDashboard(dashboard.FluentIndexDashboard):
+    """ Gets application modules from services, accounts and administration registries """
     def process_registered_view(self, module, view_name, options):
         app_name, name = view_name.split('_')[:-1]
         module.icons['.'.join((app_name, name))] = options.get('icon')
         url = reverse('admin:' + view_name)
         add_url = '/'.join(url.split('/')[:-2])
         module.children.append({
-            'models': [{
+            'models': [
+                {
                     'add_url': add_url,
                     'app_name': app_name,
                     'change_url': url,
                     'name': name,
-                'title': options.get('verbose_name_plural')}],
+                    'title': options.get('verbose_name_plural')
+                }
+            ],
             'name': app_name,
             'title': options.get('verbose_name_plural'),
             'url': add_url,
         })
     
     def get_application_modules(self):
-        from fluent_dashboard import appsettings
         modules = []
         # Honor settings override, hacky. I Know
         if appsettings.FLUENT_DASHBOARD_APP_GROUPS[0][0] != _('CMS'):

orchestra/contrib/issues/apps.py

@@ -1,6 +1,7 @@
 from django.apps import AppConfig
 
 from orchestra.core import accounts, administration
+from orchestra.core.translations import ModelTranslation
 
 
 class IssuesConfig(AppConfig):
@@ -11,3 +12,4 @@ class IssuesConfig(AppConfig):
         from .models import Queue, Ticket
         accounts.register(Ticket, icon='Ticket_star.png')
         administration.register(Queue, dashboard=False)
+        ModelTranslation.register(Queue, ('verbose_name',))

orchestra/contrib/issues/models.py

@@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
 
 from orchestra.contrib.contacts import settings as contacts_settings
 from orchestra.contrib.contacts.models import Contact
-from orchestra.core.translations import ModelTranslation
 from orchestra.models.fields import MultiSelectField
 from orchestra.utils.mail import send_email_template
 
@@ -191,6 +190,3 @@ class TicketTracker(models.Model):
         unique_together = (
             ('ticket', 'user'),
         )
-
-
-ModelTranslation.register(Queue, ('verbose_name',))

orchestra/contrib/mailer/admin.py

@@ -1,10 +1,13 @@
 from django.contrib import admin
 from django.core.urlresolvers import reverse
-from django.db.models import Count
+from django.db.models import Count, Prefetch
+from django.shortcuts import redirect
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.admin.utils import admin_link, admin_colored, admin_date
+from orchestra.admin.utils import admin_link, admin_colored, admin_date, wrap_admin_view
+from orchestra.contrib.tasks import task
 
+from .engine import send_pending
 from .models import Message, SMTPLog
 
 
@@ -20,27 +23,51 @@ COLORS = {
 
 class MessageAdmin(admin.ModelAdmin):
     list_display = (
-        'id', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta',
+        'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta',
         'retries', 'last_retry_delta', 'num_logs',
     )
     list_filter = ('state', 'priority', 'retries')
+    list_prefetch_related = ('logs__id')
     
     colored_state = admin_colored('state', colors=COLORS)
     created_at_delta = admin_date('created_at')
     last_retry_delta = admin_date('last_retry')
     
+    def display_subject(self, instance):
+        return instance.subject[:32]
+    display_subject.short_description = _("Subject")
+    display_subject.admin_order_field = 'subject'
+    
     def num_logs(self, instance):
         num = instance.logs__count
+        if num == 1:
+            pk = instance.logs.all()[0].id
+            url = reverse('admin:mailer_smtplog_change', args=(pk,))
+        else:
             url = reverse('admin:mailer_smtplog_changelist')
             url += '?&message=%i' % instance.pk
-        return '<a href="%s">%d</a>' % (url, num)
+        return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, num)
     num_logs.short_description = _("Logs")
     num_logs.admin_order_field = 'logs__count'
     num_logs.allow_tags = True
     
+    def get_urls(self):
+        from django.conf.urls import url
+        urls = super(MessageAdmin, self).get_urls()
+        info = self.model._meta.app_label, self.model._meta.model_name
+        urls.insert(0,
+            url(r'^send-pending/$', wrap_admin_view(self, self.send_pending_view), name='%s_%s_send_pending' % info)
+        )
+        return urls
+    
     def get_queryset(self, request):
         qs = super(MessageAdmin, self).get_queryset(request)
-        return qs.annotate(Count('logs'))
+        return qs.annotate(Count('logs')).prefetch_related('logs')
+    
+    def send_pending_view(self, request):
+        task(send_pending).apply_async()
+        self.message_user(request, _("Pending messages are being sent on the background."))
+        return redirect('..')
 
 
 class SMTPLogAdmin(admin.ModelAdmin):
@@ -48,6 +75,8 @@ class SMTPLogAdmin(admin.ModelAdmin):
         'id', 'message_link', 'colored_result', 'date_delta', 'log_message'
     )
     list_filter = ('result',)
+    fields = ('message_link', 'colored_result', 'date_delta', 'log_message')
+    readonly_fields = fields
     
     message_link = admin_link('message')
     colored_result = admin_colored('result', colors=COLORS, bold=False)

orchestra/contrib/mailer/engine.py

@@ -1,11 +1,15 @@
 import smtplib
+from datetime import timedelta
 from socket import error as SocketError
 
 from django.core.mail import get_connection
+from django.db.models import Q
+from django.utils import timezone
 from django.utils.encoding import smart_str
 
 from orchestra.utils.sys import LockFile
 
+from . import settings
 from .models import Message
 
 
@@ -37,11 +41,6 @@ def send_pending(bulk=100):
         for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'):
             send_message(message, num, connection, bulk)
             num += 1
-        from django.utils import timezone
-        from . import settings
-        from datetime import timedelta
-        from django.db.models import Q
-        
         now = timezone.now()
         qs = Q()
         for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):

orchestra/contrib/mailer/management/commands/sendpendingmessages.py

@@ -2,10 +2,13 @@ import json
 
 from django.core.management.base import BaseCommand, CommandError
 
+from orchestra.contrib.tasks.decorators import keep_state
+
 from ...engine import send_pending
 
+
 class Command(BaseCommand):
     help = 'Runs Orchestra method.'
     
     def handle(self, *args, **options):
-        send_pending()
+        keep_state(send_pending)()

orchestra/contrib/mailer/models.py

@@ -38,6 +38,9 @@ class Message(models.Model):
     # TODO rename to last_try
     last_retry = models.DateTimeField(_("last try"), auto_now=True)
     
+    def __str__(self):
+        return '%s to %s' % (self.subject, self.to_address)
+    
     def defer(self):
         self.state = self.DEFERRED
         # Max tries

orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html

@@ -0,0 +1,14 @@
+{% extends "admin/change_list.html" %}
+{% load i18n admin_urls admin_static admin_list %}
+
+
+{% block object-tools-items %}
+    <li>
+      {% url cl.opts|admin_urlname:'send_pending' as send_pending_url %}
+      <a href="{% add_preserved_filters send_pending_url is_popup to_field %}" class="historylink">
+        {% blocktrans with cl.opts.verbose_name as name %}Send pending{% endblocktrans %}
+      </a>
+    </li>
+    {{ block.super }}
+{% endblock %}
+

orchestra/contrib/orchestration/__init__.py

@@ -33,7 +33,7 @@ class Operation():
         self.routes = routes
     
     @classmethod
-    def execute(cls, operations, serialize=False, async=False):
+    def execute(cls, operations, serialize=False, async=None):
         from . import manager
         scripts, oserialize = manager.generate(operations)
         return manager.execute(scripts, serialize=(serialize or oserialize), async=async)

orchestra/contrib/orchestration/manager.py

@@ -97,8 +97,13 @@ def generate(operations):
     return scripts, serialize
 
 
-def execute(scripts, serialize=False, async=False):
+def execute(scripts, serialize=False, async=None):
-    """ executes the operations on the servers """
+    """
+    executes the operations on the servers
+    
+    serialize: execute one backend at a time
+    async: do not join threads (overrides route.async)
+    """
     if settings.ORCHESTRATION_DISABLE_EXECUTION:
         logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION settings.')
         return []
@@ -110,7 +115,10 @@ def execute(scripts, serialize=False, async=False):
         route, __ = key
         backend, operations = value
         args = (route.host,)
-        async = not serialize and (async or route.async)
+        if async is None:
+            async = not serialize and route.async
+        else:
+            async = not serialize and async
         kwargs = {
             'async': async,
         }

orchestra/contrib/plans/models.py

@@ -4,7 +4,6 @@ from django.db.models import Q
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.core import services, accounts
 from orchestra.core.validators import validate_name
 from orchestra.models import queryset
 

orchestra/contrib/resources/admin.py

@@ -11,7 +11,7 @@ from orchestra.admin import ExtendedModelAdmin
 from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
 from orchestra.contrib.orchestration.models import Route
 from orchestra.core import services
-from orchestra.utils.db import database_ready
+from orchestra.utils import db, sys
 from orchestra.utils.functional import cached
 
 from .actions import run_monitor
@@ -45,7 +45,9 @@ class ResourceAdmin(ExtendedModelAdmin):
     actions = (run_monitor,)
     change_view_actions = actions
     change_readonly_fields = ('name', 'content_type')
-    prepopulated_fields = {'name': ('verbose_name',)}
+    prepopulated_fields = {
+        'name': ('verbose_name',)
+    }
     list_select_related = ('content_type', 'crontab',)
     
     def change_view(self, request, object_id, form_url='', extra_context=None):
@@ -70,6 +72,7 @@ class ResourceAdmin(ExtendedModelAdmin):
     
     def save_model(self, request, obj, form, change):
         super(ResourceAdmin, self).save_model(request, obj, form, change)
+        # best-effort
         model = obj.content_type.model_class()
         modeladmin = type(get_modeladmin(model))
         resources = obj.content_type.resource_set.filter(is_active=True)
@@ -79,6 +82,8 @@ class ResourceAdmin(ExtendedModelAdmin):
                 inline = resource_inline_factory(resources)
             inlines.append(inline)
         modeladmin.inlines = inlines
+        # reload Not always work
+        sys.touch_wsgi()
     
     def formfield_for_dbfield(self, db_field, **kwargs):
         """ filter service content_types """
@@ -271,5 +276,5 @@ def insert_resource_inlines():
         insertattr(model, 'inlines', inline)
 
 
-if database_ready():
+if db.database_ready():
     insert_resource_inlines()

orchestra/contrib/resources/models.py

@@ -9,8 +9,6 @@ from django.utils.translation import ugettext_lazy as _
 from orchestra.core import validators
 from orchestra.models import queryset, fields
 from orchestra.models.utils import get_model_field_path
-from orchestra.utils.paths import get_project_dir
-from orchestra.utils.sys import run
 
 from . import tasks, settings
 from .backends import ServiceMonitor
@@ -122,7 +120,6 @@ class Resource(models.Model):
         self.sync_periodic_task()
         # This only work on tests (multiprocessing used on real deployments)
         apps.get_app_config('resources').reload_relations()
-        run('{ sleep 2 && touch %s/wsgi.py; } &' % get_project_dir(), async=True)
     
     def delete(self, *args, **kwargs):
         super(Resource, self).delete(*args, **kwargs)

orchestra/contrib/settings/admin.py

@@ -73,7 +73,7 @@ class SettingView(generic.edit.FormView):
             
             # Save changes
             parser.save(changes)
-            sys.run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True)
+            sys.touch_wsgi()
             n = len(changes)
             context = {
                 'message': ngettext(

orchestra/contrib/systemusers/models.py

@@ -6,7 +6,7 @@ from django.db import models
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.core import services, validators
+from orchestra.core import validators
 
 from . import settings
 

orchestra/contrib/tasks/decorators.py

@@ -18,14 +18,18 @@ from .utils import get_name, get_id
 def keep_state(fn):
     """ logs task on djcelery's TaskState model """
     @wraps(fn)
-    def wrapper(task_id, name, *args, **kwargs):
+    def wrapper(*args, _task_id=None, _name=None, **kwargs):
         from djcelery.models import TaskState
         now = timezone.now()
-        state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
+        if _task_id is None:
+            _task_id = get_id()
+        if _name is None:
+            _name = get_name(fn)
+        state = TaskState.objects.create(state=states.STARTED, task_id=_task_id, name=_name, args=str(args),
             kwargs=str(kwargs), tstamp=now)
         try:
             result = fn(*args, **kwargs)
-        except Exception as exc:
+        except:
             state.state = states.FAILURE
             state.traceback = trace
             state.runtime = (timezone.now()-now).total_seconds()
@@ -35,7 +39,7 @@ def keep_state(fn):
             logger.error(subject)
             logger.error(trace)
             mail_admins(subject, trace)
-            return
+            raise
         else:
             state.state = states.SUCCESS
             state.result = str(result)
@@ -49,7 +53,10 @@ def apply_async(fn, name=None, method='thread'):
     """ replaces celery apply_async """
     def inner(fn, name, method, *args, **kwargs):
         task_id = get_id()
-        args = (task_id, name) + args
+        kwargs.update({
+            '_name': name, 
+            '_task_id': task_id,
+        })
         thread = method(target=fn, args=args, kwargs=kwargs)
         thread.start()
         # Celery API compat

orchestra/contrib/tasks/management/commands/runtask.py

@@ -6,7 +6,6 @@ from django.utils import timezone
 from djcelery.models import PeriodicTask
 
 from ...decorators import keep_state
-from ...utils import get_id, get_name
 
 
 class Command(BaseCommand):
@@ -46,4 +45,4 @@ class Command(BaseCommand):
                     arguments.append(arg)
             args = arguments
         # Run task synchronously, but logging TaskState
-        keep_state(task)(get_id(), get_name(task), *args, **kwargs)
+        keep_state(task)(*args, **kwargs)

orchestra/management/commands/setupnginx.py

@@ -19,9 +19,11 @@ class Command(BaseCommand):
             make_option('--cert-key', dest='cert_key', default='',
                 help='Nginx SSL certificate key.'),
             
-            make_option('--cert-path', dest='cert_path', default='/etc/nginx/ssl/orchestra.crt',
+            make_option('--cert-path', dest='cert_path',
+                default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.crt'),
                 help='Nginx SSL certificate, one will be created by default.'),
-            make_option('--cert-key-path', dest='cert_key_path', default='/etc/nginx/ssl/orchestra.key',
+            make_option('--cert-key-path', dest='cert_key_path',
+                default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.key'),
                 help='Nginx SSL certificate key.'),
             # Cert options
             make_option('--cert-override', dest='cert_override', action='store_true',

orchestra/utils/sys.py

@@ -195,3 +195,8 @@ class LockFile(object):
     def __exit__(self, type, value, traceback):
         if not self.unlocked:
             self.release()
+
+
+def touch_wsgi():
+    from . import paths
+    run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True)

scripts/container/deploy.sh

@@ -106,6 +106,7 @@ if [[ $CELERY == true ]]; then
     sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2
 else
     surun "$PYTHON_BIN $MANAGE setupcronbeat"
+    surun "$PYTHON_BIN $MANAGE syncperiodictasks"
 fi