diff --git a/INSTALL.md b/INSTALL.md index fddfc3ff..ed52fa49 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -53,7 +53,8 @@ Django-orchestra can be installed on any Linux system, however it is **strongly 6. Configure periodic execution of tasks (choose one) 1. Use cron ```bash - sudo python3 manage.py setupcronbeat + python3 manage.py setupcronbeat + python3 panel/manage.py syncperiodictasks ``` 2. Use celeryd diff --git a/README.md b/README.md index d47a9226..a8be5feb 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ python3 panel/manage.py runserver # Enable periodic tasks with cron (optional) python3 panel/manage.py setupcronbeat +python3 panel/manage.py syncperiodictasks ``` Now you can see the web interface on http://localhost:8000/admin diff --git a/TODO.md b/TODO.md index ecbd5b3b..7b1e1e6b 100644 --- a/TODO.md +++ b/TODO.md @@ -288,9 +288,9 @@ https://code.djangoproject.com/ticket/24576 # insert settings on dashboard dynamically # convert all complex settings to string -# @ something database names +# size monitor of @002 @003 database names # password validation cracklib on change password form=????? -# reset setting buton +# reset setting button # periodic cleaning of spam mailboxes @@ -298,21 +298,6 @@ https://code.djangoproject.com/ticket/24576 # django SITE_NAME vs ORCHESTRA_SITE_NAME ? -Replace celery by a custom solution? - # TODO create decorator wrapper that abstract the task away from the backen (cron/celery) - # TODO crontab model localhost/autoadded attribute - * No more jumbo dependencies and wierd bugs - 1) Periodic Monitoring: - * runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system) - 2) Single time shot: - sys.run("python3 manage.py runtas 'task' args") - 3) Emails: - Custom backend that distinguishes between priority and bulk mail - *priority: custom Thread backend - *bulk: wrapper arround django-mailer to avoid loading django system - -pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip - # TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT: import uwsgi @@ -321,7 +306,7 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip uwsgi.register_signal(99, '', uwsgi_beat) uwsgi.add_timer(99, 60) # TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi') -# SHip orchestra production-ready (no DEBUG etc) +# Ship orchestra production-ready (no DEBUG etc) # import module and sed # if setting.value == default. remove @@ -329,8 +314,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip # inspecting django db connection for asserting db readines? or performing a query # wake up django mailer on send_mail -# project settings modified copy of django's default project settings - # all signals + accouns.register() services.register() on apps.py from orchestra.contrib.tasks import task @@ -350,44 +333,23 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix TODO mount the filesystem with "nosuid" option -# wkhtmltopdf -> reportlab -# autoiscover modules on app.ready() ? lazy choices on models for plugins -# ModelTranslation.register on app.ready() + # uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask # mailboxes.address settings multiple local domains, not only one? -# backend.context = self.get_context() or save(obj, context=None) +# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data # smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209') -# create registered periodic_task on beat execution: and management command: syncperiodictasks - -# MERGE beats and inspect INSTALLED_APPS and get IS_ENABLED - # rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ? # virtdomains file is not ideal, prevent fake/error on domains there! and make sure this file is required! -# Message last_retry auto_now doesn't work! - -# don't block on beat, and --report periodic tasks - # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery) - # orchestrate async stdout stderr (inspired on pangea managemengt commands) - # orchestra-beat support for uwsgi cron -# message.log if 1: return changeform - -# generate nginx certs on project dir rather than nginx - -# Register icons - -# send_message doesn't log task +# message.log if len() == 1: return changeform make django admin taskstate uncollapse fucking traceback, ( if exists ?) -# receive tass stuck at RECEIVED -# monitor tasks in started and backend already in success? - -# custom message on admin save +# form for custom message on admin save "comment & save"? diff --git a/orchestra/admin/dashboard.py b/orchestra/admin/dashboard.py index 62df66f9..1e09822d 100644 --- a/orchestra/admin/dashboard.py +++ b/orchestra/admin/dashboard.py @@ -1,12 +1,13 @@ from django.core.urlresolvers import reverse from django.utils.translation import ugettext_lazy as _ -from fluent_dashboard import dashboard +from fluent_dashboard import dashboard, appsettings from fluent_dashboard.modules import CmsAppIconList from orchestra.core import services, accounts, administration class AppDefaultIconList(CmsAppIconList): + """ Provides support for custom default icons """ def __init__(self, *args, **kwargs): self.icons = kwargs.pop('icons') super(AppDefaultIconList, self).__init__(*args, **kwargs) @@ -17,25 +18,28 @@ class AppDefaultIconList(CmsAppIconList): class OrchestraIndexDashboard(dashboard.FluentIndexDashboard): + """ Gets application modules from services, accounts and administration registries """ def process_registered_view(self, module, view_name, options): app_name, name = view_name.split('_')[:-1] module.icons['.'.join((app_name, name))] = options.get('icon') url = reverse('admin:' + view_name) add_url = '/'.join(url.split('/')[:-2]) module.children.append({ - 'models': [{ - 'add_url': add_url, - 'app_name': app_name, - 'change_url': url, - 'name': name, - 'title': options.get('verbose_name_plural')}], - 'name': app_name, - 'title': options.get('verbose_name_plural'), - 'url': add_url, + 'models': [ + { + 'add_url': add_url, + 'app_name': app_name, + 'change_url': url, + 'name': name, + 'title': options.get('verbose_name_plural') + } + ], + 'name': app_name, + 'title': options.get('verbose_name_plural'), + 'url': add_url, }) def get_application_modules(self): - from fluent_dashboard import appsettings modules = [] # Honor settings override, hacky. I Know if appsettings.FLUENT_DASHBOARD_APP_GROUPS[0][0] != _('CMS'): diff --git a/orchestra/contrib/issues/apps.py b/orchestra/contrib/issues/apps.py index fdc35f24..c2c32def 100644 --- a/orchestra/contrib/issues/apps.py +++ b/orchestra/contrib/issues/apps.py @@ -1,6 +1,7 @@ from django.apps import AppConfig from orchestra.core import accounts, administration +from orchestra.core.translations import ModelTranslation class IssuesConfig(AppConfig): @@ -11,3 +12,4 @@ class IssuesConfig(AppConfig): from .models import Queue, Ticket accounts.register(Ticket, icon='Ticket_star.png') administration.register(Queue, dashboard=False) + ModelTranslation.register(Queue, ('verbose_name',)) diff --git a/orchestra/contrib/issues/models.py b/orchestra/contrib/issues/models.py index 62bf22dd..1e2aab0d 100644 --- a/orchestra/contrib/issues/models.py +++ b/orchestra/contrib/issues/models.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.contacts import settings as contacts_settings from orchestra.contrib.contacts.models import Contact -from orchestra.core.translations import ModelTranslation from orchestra.models.fields import MultiSelectField from orchestra.utils.mail import send_email_template @@ -191,6 +190,3 @@ class TicketTracker(models.Model): unique_together = ( ('ticket', 'user'), ) - - -ModelTranslation.register(Queue, ('verbose_name',)) diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py index a884360d..fc786de5 100644 --- a/orchestra/contrib/mailer/admin.py +++ b/orchestra/contrib/mailer/admin.py @@ -1,10 +1,13 @@ from django.contrib import admin from django.core.urlresolvers import reverse -from django.db.models import Count +from django.db.models import Count, Prefetch +from django.shortcuts import redirect from django.utils.translation import ugettext_lazy as _ -from orchestra.admin.utils import admin_link, admin_colored, admin_date +from orchestra.admin.utils import admin_link, admin_colored, admin_date, wrap_admin_view +from orchestra.contrib.tasks import task +from .engine import send_pending from .models import Message, SMTPLog @@ -20,27 +23,51 @@ COLORS = { class MessageAdmin(admin.ModelAdmin): list_display = ( - 'id', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta', + 'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta', 'retries', 'last_retry_delta', 'num_logs', ) list_filter = ('state', 'priority', 'retries') + list_prefetch_related = ('logs__id') colored_state = admin_colored('state', colors=COLORS) created_at_delta = admin_date('created_at') last_retry_delta = admin_date('last_retry') + def display_subject(self, instance): + return instance.subject[:32] + display_subject.short_description = _("Subject") + display_subject.admin_order_field = 'subject' + def num_logs(self, instance): num = instance.logs__count - url = reverse('admin:mailer_smtplog_changelist') - url += '?&message=%i' % instance.pk - return '%d' % (url, num) + if num == 1: + pk = instance.logs.all()[0].id + url = reverse('admin:mailer_smtplog_change', args=(pk,)) + else: + url = reverse('admin:mailer_smtplog_changelist') + url += '?&message=%i' % instance.pk + return '%d' % (url, num) num_logs.short_description = _("Logs") num_logs.admin_order_field = 'logs__count' num_logs.allow_tags = True + def get_urls(self): + from django.conf.urls import url + urls = super(MessageAdmin, self).get_urls() + info = self.model._meta.app_label, self.model._meta.model_name + urls.insert(0, + url(r'^send-pending/$', wrap_admin_view(self, self.send_pending_view), name='%s_%s_send_pending' % info) + ) + return urls + def get_queryset(self, request): qs = super(MessageAdmin, self).get_queryset(request) - return qs.annotate(Count('logs')) + return qs.annotate(Count('logs')).prefetch_related('logs') + + def send_pending_view(self, request): + task(send_pending).apply_async() + self.message_user(request, _("Pending messages are being sent on the background.")) + return redirect('..') class SMTPLogAdmin(admin.ModelAdmin): @@ -48,6 +75,8 @@ class SMTPLogAdmin(admin.ModelAdmin): 'id', 'message_link', 'colored_result', 'date_delta', 'log_message' ) list_filter = ('result',) + fields = ('message_link', 'colored_result', 'date_delta', 'log_message') + readonly_fields = fields message_link = admin_link('message') colored_result = admin_colored('result', colors=COLORS, bold=False) diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py index a9c0a2ed..da6f4dbc 100644 --- a/orchestra/contrib/mailer/engine.py +++ b/orchestra/contrib/mailer/engine.py @@ -1,11 +1,15 @@ import smtplib +from datetime import timedelta from socket import error as SocketError from django.core.mail import get_connection +from django.db.models import Q +from django.utils import timezone from django.utils.encoding import smart_str from orchestra.utils.sys import LockFile +from . import settings from .models import Message @@ -37,11 +41,6 @@ def send_pending(bulk=100): for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'): send_message(message, num, connection, bulk) num += 1 - from django.utils import timezone - from . import settings - from datetime import timedelta - from django.db.models import Q - now = timezone.now() qs = Q() for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS): diff --git a/orchestra/contrib/mailer/management/commands/sendpendingmessages.py b/orchestra/contrib/mailer/management/commands/sendpendingmessages.py index 24d28c7a..5bb413cf 100644 --- a/orchestra/contrib/mailer/management/commands/sendpendingmessages.py +++ b/orchestra/contrib/mailer/management/commands/sendpendingmessages.py @@ -2,10 +2,13 @@ import json from django.core.management.base import BaseCommand, CommandError +from orchestra.contrib.tasks.decorators import keep_state + from ...engine import send_pending + class Command(BaseCommand): help = 'Runs Orchestra method.' def handle(self, *args, **options): - send_pending() + keep_state(send_pending)() diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py index 21786c1e..e0e3c3b8 100644 --- a/orchestra/contrib/mailer/models.py +++ b/orchestra/contrib/mailer/models.py @@ -38,6 +38,9 @@ class Message(models.Model): # TODO rename to last_try last_retry = models.DateTimeField(_("last try"), auto_now=True) + def __str__(self): + return '%s to %s' % (self.subject, self.to_address) + def defer(self): self.state = self.DEFERRED # Max tries diff --git a/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html b/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html new file mode 100644 index 00000000..93f17d12 --- /dev/null +++ b/orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html @@ -0,0 +1,14 @@ +{% extends "admin/change_list.html" %} +{% load i18n admin_urls admin_static admin_list %} + + +{% block object-tools-items %} +
  • + {% url cl.opts|admin_urlname:'send_pending' as send_pending_url %} + + {% blocktrans with cl.opts.verbose_name as name %}Send pending{% endblocktrans %} + +
  • + {{ block.super }} +{% endblock %} + diff --git a/orchestra/contrib/orchestration/__init__.py b/orchestra/contrib/orchestration/__init__.py index 262c9d81..86219de0 100644 --- a/orchestra/contrib/orchestration/__init__.py +++ b/orchestra/contrib/orchestration/__init__.py @@ -33,7 +33,7 @@ class Operation(): self.routes = routes @classmethod - def execute(cls, operations, serialize=False, async=False): + def execute(cls, operations, serialize=False, async=None): from . import manager scripts, oserialize = manager.generate(operations) return manager.execute(scripts, serialize=(serialize or oserialize), async=async) diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index 0ee544f4..9bcafb26 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -97,8 +97,13 @@ def generate(operations): return scripts, serialize -def execute(scripts, serialize=False, async=False): - """ executes the operations on the servers """ +def execute(scripts, serialize=False, async=None): + """ + executes the operations on the servers + + serialize: execute one backend at a time + async: do not join threads (overrides route.async) + """ if settings.ORCHESTRATION_DISABLE_EXECUTION: logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION settings.') return [] @@ -110,7 +115,10 @@ def execute(scripts, serialize=False, async=False): route, __ = key backend, operations = value args = (route.host,) - async = not serialize and (async or route.async) + if async is None: + async = not serialize and route.async + else: + async = not serialize and async kwargs = { 'async': async, } diff --git a/orchestra/contrib/plans/models.py b/orchestra/contrib/plans/models.py index 7caf1a57..4710f252 100644 --- a/orchestra/contrib/plans/models.py +++ b/orchestra/contrib/plans/models.py @@ -4,7 +4,6 @@ from django.db.models import Q from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ -from orchestra.core import services, accounts from orchestra.core.validators import validate_name from orchestra.models import queryset diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py index 060c21d6..afefa72d 100644 --- a/orchestra/contrib/resources/admin.py +++ b/orchestra/contrib/resources/admin.py @@ -11,7 +11,7 @@ from orchestra.admin import ExtendedModelAdmin from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date from orchestra.contrib.orchestration.models import Route from orchestra.core import services -from orchestra.utils.db import database_ready +from orchestra.utils import db, sys from orchestra.utils.functional import cached from .actions import run_monitor @@ -45,7 +45,9 @@ class ResourceAdmin(ExtendedModelAdmin): actions = (run_monitor,) change_view_actions = actions change_readonly_fields = ('name', 'content_type') - prepopulated_fields = {'name': ('verbose_name',)} + prepopulated_fields = { + 'name': ('verbose_name',) + } list_select_related = ('content_type', 'crontab',) def change_view(self, request, object_id, form_url='', extra_context=None): @@ -70,6 +72,7 @@ class ResourceAdmin(ExtendedModelAdmin): def save_model(self, request, obj, form, change): super(ResourceAdmin, self).save_model(request, obj, form, change) + # best-effort model = obj.content_type.model_class() modeladmin = type(get_modeladmin(model)) resources = obj.content_type.resource_set.filter(is_active=True) @@ -79,6 +82,8 @@ class ResourceAdmin(ExtendedModelAdmin): inline = resource_inline_factory(resources) inlines.append(inline) modeladmin.inlines = inlines + # reload Not always work + sys.touch_wsgi() def formfield_for_dbfield(self, db_field, **kwargs): """ filter service content_types """ @@ -271,5 +276,5 @@ def insert_resource_inlines(): insertattr(model, 'inlines', inline) -if database_ready(): +if db.database_ready(): insert_resource_inlines() diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index 53a1c7ac..72e9697c 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -9,8 +9,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.core import validators from orchestra.models import queryset, fields from orchestra.models.utils import get_model_field_path -from orchestra.utils.paths import get_project_dir -from orchestra.utils.sys import run from . import tasks, settings from .backends import ServiceMonitor @@ -122,7 +120,6 @@ class Resource(models.Model): self.sync_periodic_task() # This only work on tests (multiprocessing used on real deployments) apps.get_app_config('resources').reload_relations() - run('{ sleep 2 && touch %s/wsgi.py; } &' % get_project_dir(), async=True) def delete(self, *args, **kwargs): super(Resource, self).delete(*args, **kwargs) diff --git a/orchestra/contrib/settings/admin.py b/orchestra/contrib/settings/admin.py index 93e44325..c3d6d688 100644 --- a/orchestra/contrib/settings/admin.py +++ b/orchestra/contrib/settings/admin.py @@ -73,7 +73,7 @@ class SettingView(generic.edit.FormView): # Save changes parser.save(changes) - sys.run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True) + sys.touch_wsgi() n = len(changes) context = { 'message': ngettext( diff --git a/orchestra/contrib/systemusers/models.py b/orchestra/contrib/systemusers/models.py index 11190f93..1a15b8dc 100644 --- a/orchestra/contrib/systemusers/models.py +++ b/orchestra/contrib/systemusers/models.py @@ -6,7 +6,7 @@ from django.db import models from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ -from orchestra.core import services, validators +from orchestra.core import validators from . import settings diff --git a/orchestra/contrib/tasks/decorators.py b/orchestra/contrib/tasks/decorators.py index 725d4a8e..c42ad824 100644 --- a/orchestra/contrib/tasks/decorators.py +++ b/orchestra/contrib/tasks/decorators.py @@ -18,14 +18,18 @@ from .utils import get_name, get_id def keep_state(fn): """ logs task on djcelery's TaskState model """ @wraps(fn) - def wrapper(task_id, name, *args, **kwargs): + def wrapper(*args, _task_id=None, _name=None, **kwargs): from djcelery.models import TaskState now = timezone.now() - state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args), + if _task_id is None: + _task_id = get_id() + if _name is None: + _name = get_name(fn) + state = TaskState.objects.create(state=states.STARTED, task_id=_task_id, name=_name, args=str(args), kwargs=str(kwargs), tstamp=now) try: result = fn(*args, **kwargs) - except Exception as exc: + except: state.state = states.FAILURE state.traceback = trace state.runtime = (timezone.now()-now).total_seconds() @@ -35,7 +39,7 @@ def keep_state(fn): logger.error(subject) logger.error(trace) mail_admins(subject, trace) - return + raise else: state.state = states.SUCCESS state.result = str(result) @@ -49,7 +53,10 @@ def apply_async(fn, name=None, method='thread'): """ replaces celery apply_async """ def inner(fn, name, method, *args, **kwargs): task_id = get_id() - args = (task_id, name) + args + kwargs.update({ + '_name': name, + '_task_id': task_id, + }) thread = method(target=fn, args=args, kwargs=kwargs) thread.start() # Celery API compat diff --git a/orchestra/contrib/tasks/management/commands/runtask.py b/orchestra/contrib/tasks/management/commands/runtask.py index a6a33f82..362fe5b9 100644 --- a/orchestra/contrib/tasks/management/commands/runtask.py +++ b/orchestra/contrib/tasks/management/commands/runtask.py @@ -6,7 +6,6 @@ from django.utils import timezone from djcelery.models import PeriodicTask from ...decorators import keep_state -from ...utils import get_id, get_name class Command(BaseCommand): @@ -46,4 +45,4 @@ class Command(BaseCommand): arguments.append(arg) args = arguments # Run task synchronously, but logging TaskState - keep_state(task)(get_id(), get_name(task), *args, **kwargs) + keep_state(task)(*args, **kwargs) diff --git a/orchestra/management/commands/setupnginx.py b/orchestra/management/commands/setupnginx.py index 8ab1ffa1..30744944 100644 --- a/orchestra/management/commands/setupnginx.py +++ b/orchestra/management/commands/setupnginx.py @@ -19,9 +19,11 @@ class Command(BaseCommand): make_option('--cert-key', dest='cert_key', default='', help='Nginx SSL certificate key.'), - make_option('--cert-path', dest='cert_path', default='/etc/nginx/ssl/orchestra.crt', + make_option('--cert-path', dest='cert_path', + default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.crt'), help='Nginx SSL certificate, one will be created by default.'), - make_option('--cert-key-path', dest='cert_key_path', default='/etc/nginx/ssl/orchestra.key', + make_option('--cert-key-path', dest='cert_key_path', + default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.key'), help='Nginx SSL certificate key.'), # Cert options make_option('--cert-override', dest='cert_override', action='store_true', diff --git a/orchestra/utils/sys.py b/orchestra/utils/sys.py index aad7d6f7..d17ea275 100644 --- a/orchestra/utils/sys.py +++ b/orchestra/utils/sys.py @@ -195,3 +195,8 @@ class LockFile(object): def __exit__(self, type, value, traceback): if not self.unlocked: self.release() + + +def touch_wsgi(): + from . import paths + run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True) diff --git a/scripts/container/deploy.sh b/scripts/container/deploy.sh index 3ea1479b..94a08e59 100755 --- a/scripts/container/deploy.sh +++ b/scripts/container/deploy.sh @@ -106,6 +106,7 @@ if [[ $CELERY == true ]]; then sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2 else surun "$PYTHON_BIN $MANAGE setupcronbeat" + surun "$PYTHON_BIN $MANAGE syncperiodictasks" fi