diff --git a/INSTALL.md b/INSTALL.md index fddfc3ff..ed52fa49 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -53,7 +53,8 @@ Django-orchestra can be installed on any Linux system, however it is **strongly 6. Configure periodic execution of tasks (choose one) 1. Use cron ```bash - sudo python3 manage.py setupcronbeat + python3 manage.py setupcronbeat + python3 panel/manage.py syncperiodictasks ``` 2. Use celeryd diff --git a/README.md b/README.md index d47a9226..a8be5feb 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ python3 panel/manage.py runserver # Enable periodic tasks with cron (optional) python3 panel/manage.py setupcronbeat +python3 panel/manage.py syncperiodictasks ``` Now you can see the web interface on http://localhost:8000/admin diff --git a/TODO.md b/TODO.md index ecbd5b3b..7b1e1e6b 100644 --- a/TODO.md +++ b/TODO.md @@ -288,9 +288,9 @@ https://code.djangoproject.com/ticket/24576 # insert settings on dashboard dynamically # convert all complex settings to string -# @ something database names +# size monitor of @002 @003 database names # password validation cracklib on change password form=????? -# reset setting buton +# reset setting button # periodic cleaning of spam mailboxes @@ -298,21 +298,6 @@ https://code.djangoproject.com/ticket/24576 # django SITE_NAME vs ORCHESTRA_SITE_NAME ? -Replace celery by a custom solution? - # TODO create decorator wrapper that abstract the task away from the backen (cron/celery) - # TODO crontab model localhost/autoadded attribute - * No more jumbo dependencies and wierd bugs - 1) Periodic Monitoring: - * runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system) - 2) Single time shot: - sys.run("python3 manage.py runtas 'task' args") - 3) Emails: - Custom backend that distinguishes between priority and bulk mail - *priority: custom Thread backend - *bulk: wrapper arround django-mailer to avoid loading django system - -pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip - # TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT: import uwsgi @@ -321,7 +306,7 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip uwsgi.register_signal(99, '', uwsgi_beat) uwsgi.add_timer(99, 60) # TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi') -# SHip orchestra production-ready (no DEBUG etc) +# Ship orchestra production-ready (no DEBUG etc) # import module and sed # if setting.value == default. remove @@ -329,8 +314,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip # inspecting django db connection for asserting db readines? or performing a query # wake up django mailer on send_mail -# project settings modified copy of django's default project settings - # all signals + accouns.register() services.register() on apps.py from orchestra.contrib.tasks import task @@ -350,44 +333,23 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix TODO mount the filesystem with "nosuid" option -# wkhtmltopdf -> reportlab -# autoiscover modules on app.ready() ? lazy choices on models for plugins -# ModelTranslation.register on app.ready() + # uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask # mailboxes.address settings multiple local domains, not only one? -# backend.context = self.get_context() or save(obj, context=None) +# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data # smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209') -# create registered periodic_task on beat execution: and management command: syncperiodictasks - -# MERGE beats and inspect INSTALLED_APPS and get IS_ENABLED - # rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ? # virtdomains file is not ideal, prevent fake/error on domains there! and make sure this file is required! -# Message last_retry auto_now doesn't work! - -# don't block on beat, and --report periodic tasks - # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery) - # orchestrate async stdout stderr (inspired on pangea managemengt commands) - # orchestra-beat support for uwsgi cron -# message.log if 1: return changeform - -# generate nginx certs on project dir rather than nginx - -# Register icons - -# send_message doesn't log task +# message.log if len() == 1: return changeform make django admin taskstate uncollapse fucking traceback, ( if exists ?) -# receive tass stuck at RECEIVED -# monitor tasks in started and backend already in success? - -# custom message on admin save +# form for custom message on admin save "comment & save"? diff --git a/orchestra/admin/dashboard.py b/orchestra/admin/dashboard.py index 62df66f9..1e09822d 100644 --- a/orchestra/admin/dashboard.py +++ b/orchestra/admin/dashboard.py @@ -1,12 +1,13 @@ from django.core.urlresolvers import reverse from django.utils.translation import ugettext_lazy as _ -from fluent_dashboard import dashboard +from fluent_dashboard import dashboard, appsettings from fluent_dashboard.modules import CmsAppIconList from orchestra.core import services, accounts, administration class AppDefaultIconList(CmsAppIconList): + """ Provides support for custom default icons """ def __init__(self, *args, **kwargs): self.icons = kwargs.pop('icons') super(AppDefaultIconList, self).__init__(*args, **kwargs) @@ -17,25 +18,28 @@ class AppDefaultIconList(CmsAppIconList): class OrchestraIndexDashboard(dashboard.FluentIndexDashboard): + """ Gets application modules from services, accounts and administration registries """ def process_registered_view(self, module, view_name, options): app_name, name = view_name.split('_')[:-1] module.icons['.'.join((app_name, name))] = options.get('icon') url = reverse('admin:' + view_name) add_url = '/'.join(url.split('/')[:-2]) module.children.append({ - 'models': [{ - 'add_url': add_url, - 'app_name': app_name, - 'change_url': url, - 'name': name, - 'title': options.get('verbose_name_plural')}], - 'name': app_name, - 'title': options.get('verbose_name_plural'), - 'url': add_url, + 'models': [ + { + 'add_url': add_url, + 'app_name': app_name, + 'change_url': url, + 'name': name, + 'title': options.get('verbose_name_plural') + } + ], + 'name': app_name, + 'title': options.get('verbose_name_plural'), + 'url': add_url, }) def get_application_modules(self): - from fluent_dashboard import appsettings modules = [] # Honor settings override, hacky. I Know if appsettings.FLUENT_DASHBOARD_APP_GROUPS[0][0] != _('CMS'): diff --git a/orchestra/contrib/issues/apps.py b/orchestra/contrib/issues/apps.py index fdc35f24..c2c32def 100644 --- a/orchestra/contrib/issues/apps.py +++ b/orchestra/contrib/issues/apps.py @@ -1,6 +1,7 @@ from django.apps import AppConfig from orchestra.core import accounts, administration +from orchestra.core.translations import ModelTranslation class IssuesConfig(AppConfig): @@ -11,3 +12,4 @@ class IssuesConfig(AppConfig): from .models import Queue, Ticket accounts.register(Ticket, icon='Ticket_star.png') administration.register(Queue, dashboard=False) + ModelTranslation.register(Queue, ('verbose_name',)) diff --git a/orchestra/contrib/issues/models.py b/orchestra/contrib/issues/models.py index 62bf22dd..1e2aab0d 100644 --- a/orchestra/contrib/issues/models.py +++ b/orchestra/contrib/issues/models.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.contacts import settings as contacts_settings from orchestra.contrib.contacts.models import Contact -from orchestra.core.translations import ModelTranslation from orchestra.models.fields import MultiSelectField from orchestra.utils.mail import send_email_template @@ -191,6 +190,3 @@ class TicketTracker(models.Model): unique_together = ( ('ticket', 'user'), ) - - -ModelTranslation.register(Queue, ('verbose_name',)) diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py index a884360d..fc786de5 100644 --- a/orchestra/contrib/mailer/admin.py +++ b/orchestra/contrib/mailer/admin.py @@ -1,10 +1,13 @@ from django.contrib import admin from django.core.urlresolvers import reverse -from django.db.models import Count +from django.db.models import Count, Prefetch +from django.shortcuts import redirect from django.utils.translation import ugettext_lazy as _ -from orchestra.admin.utils import admin_link, admin_colored, admin_date +from orchestra.admin.utils import admin_link, admin_colored, admin_date, wrap_admin_view +from orchestra.contrib.tasks import task +from .engine import send_pending from .models import Message, SMTPLog @@ -20,27 +23,51 @@ COLORS = { class MessageAdmin(admin.ModelAdmin): list_display = ( - 'id', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta', + 'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta', 'retries', 'last_retry_delta', 'num_logs', ) list_filter = ('state', 'priority', 'retries') + list_prefetch_related = ('logs__id') colored_state = admin_colored('state', colors=COLORS) created_at_delta = admin_date('created_at') last_retry_delta = admin_date('last_retry') + def display_subject(self, instance): + return instance.subject[:32] + display_subject.short_description = _("Subject") + display_subject.admin_order_field = 'subject' + def num_logs(self, instance): num = instance.logs__count - url = reverse('admin:mailer_smtplog_changelist') - url += '?&message=%i' % instance.pk - return '%d' % (url, num) + if num == 1: + pk = instance.logs.all()[0].id + url = reverse('admin:mailer_smtplog_change', args=(pk,)) + else: + url = reverse('admin:mailer_smtplog_changelist') + url += '?&message=%i' % instance.pk + return '%d' % (url, num) num_logs.short_description = _("Logs") num_logs.admin_order_field = 'logs__count' num_logs.allow_tags = True + def get_urls(self): + from django.conf.urls import url + urls = super(MessageAdmin, self).get_urls() + info = self.model._meta.app_label, self.model._meta.model_name + urls.insert(0, + url(r'^send-pending/$', wrap_admin_view(self, self.send_pending_view), name='%s_%s_send_pending' % info) + ) + return urls + def get_queryset(self, request): qs = super(MessageAdmin, self).get_queryset(request) - return qs.annotate(Count('logs')) + return qs.annotate(Count('logs')).prefetch_related('logs') + + def send_pending_view(self, request): + task(send_pending).apply_async() + self.message_user(request, _("Pending messages are being sent on the background.")) + return redirect('..') class SMTPLogAdmin(admin.ModelAdmin): @@ -48,6 +75,8 @@ class SMTPLogAdmin(admin.ModelAdmin): 'id', 'message_link', 'colored_result', 'date_delta', 'log_message' ) list_filter = ('result',) + fields = ('message_link', 'colored_result', 'date_delta', 'log_message') + readonly_fields = fields message_link = admin_link('message') colored_result = admin_colored('result', colors=COLORS, bold=False) diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py index a9c0a2ed..da6f4dbc 100644 --- a/orchestra/contrib/mailer/engine.py +++ b/orchestra/contrib/mailer/engine.py @@ -1,11 +1,15 @@ import smtplib +from datetime import timedelta from socket import error as SocketError from django.core.mail import get_connection +from django.db.models import Q +from django.utils import timezone from django.utils.encoding import smart_str from orchestra.utils.sys import LockFile +from . import settings from .models import Message @@ -37,11 +41,6 @@ def send_pending(bulk=100): for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'): send_message(message, num, connection, bulk) num += 1 - from django.utils import timezone - from . import settings - from datetime import timedelta - from django.db.models import Q - now = timezone.now() qs = Q() for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS): diff --git a/orchestra/contrib/mailer/management/commands/sendpendingmessages.py b/orchestra/contrib/mailer/management/commands/sendpendingmessages.py index 24d28c7a..5bb413cf 100644 --- a/orchestra/contrib/mailer/management/commands/sendpendingmessages.py +++ b/orchestra/contrib/mailer/management/commands/sendpendingmessages.py @@ -2,10 +2,13 @@ import json from django.core.management.base import BaseCommand, CommandError +from orchestra.contrib.tasks.decorators import keep_state + from ...engine import send_pending + class Command(BaseCommand): help = 'Runs Orchestra method.' def handle(self, *args, **options): - send_pending() + keep_state(send_pending)() diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py index 21786c1e..e0e3c3b8 100644 --- a/orchestra/contrib/mailer/models.py +++ b/orchestra/contrib/mailer/models.py @@ -38,6 +38,9 @@ class Message(models.Model): # TODO rename to last_try last_retry = models.DateTimeField(_("last try"), auto_now=True) + def __str__(self): + return '%s to %s' % (self.subject, self.to_address) + def defer(self): self.state = self.DEFERRED # Max tries diff --git a/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html b/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html new file mode 100644 index 00000000..93f17d12 --- /dev/null +++ b/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html @@ -0,0 +1,14 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls admin_static admin_list %} + + +{% block object-tools-items %} +