Reandom fixes
This commit is contained in:
parent
34596c3485
commit
b5a3f64aa9
|
@ -53,7 +53,8 @@ Django-orchestra can be installed on any Linux system, however it is **strongly
|
||||||
6. Configure periodic execution of tasks (choose one)
|
6. Configure periodic execution of tasks (choose one)
|
||||||
1. Use cron
|
1. Use cron
|
||||||
```bash
|
```bash
|
||||||
sudo python3 manage.py setupcronbeat
|
python3 manage.py setupcronbeat
|
||||||
|
python3 panel/manage.py syncperiodictasks
|
||||||
```
|
```
|
||||||
|
|
||||||
2. Use celeryd
|
2. Use celeryd
|
||||||
|
|
|
@ -55,6 +55,7 @@ python3 panel/manage.py runserver
|
||||||
|
|
||||||
# Enable periodic tasks with cron (optional)
|
# Enable periodic tasks with cron (optional)
|
||||||
python3 panel/manage.py setupcronbeat
|
python3 panel/manage.py setupcronbeat
|
||||||
|
python3 panel/manage.py syncperiodictasks
|
||||||
```
|
```
|
||||||
|
|
||||||
Now you can see the web interface on http://localhost:8000/admin
|
Now you can see the web interface on http://localhost:8000/admin
|
||||||
|
|
52
TODO.md
52
TODO.md
|
@ -288,9 +288,9 @@ https://code.djangoproject.com/ticket/24576
|
||||||
# insert settings on dashboard dynamically
|
# insert settings on dashboard dynamically
|
||||||
|
|
||||||
# convert all complex settings to string
|
# convert all complex settings to string
|
||||||
# @ something database names
|
# size monitor of @002 @003 database names
|
||||||
# password validation cracklib on change password form=?????
|
# password validation cracklib on change password form=?????
|
||||||
# reset setting buton
|
# reset setting button
|
||||||
|
|
||||||
# periodic cleaning of spam mailboxes
|
# periodic cleaning of spam mailboxes
|
||||||
|
|
||||||
|
@ -298,21 +298,6 @@ https://code.djangoproject.com/ticket/24576
|
||||||
# django SITE_NAME vs ORCHESTRA_SITE_NAME ?
|
# django SITE_NAME vs ORCHESTRA_SITE_NAME ?
|
||||||
|
|
||||||
|
|
||||||
Replace celery by a custom solution?
|
|
||||||
# TODO create decorator wrapper that abstract the task away from the backen (cron/celery)
|
|
||||||
# TODO crontab model localhost/autoadded attribute
|
|
||||||
* No more jumbo dependencies and wierd bugs
|
|
||||||
1) Periodic Monitoring:
|
|
||||||
* runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system)
|
|
||||||
2) Single time shot:
|
|
||||||
sys.run("python3 manage.py runtas 'task' args")
|
|
||||||
3) Emails:
|
|
||||||
Custom backend that distinguishes between priority and bulk mail
|
|
||||||
*priority: custom Thread backend
|
|
||||||
*bulk: wrapper arround django-mailer to avoid loading django system
|
|
||||||
|
|
||||||
pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
|
|
||||||
|
|
||||||
# TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy
|
# TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy
|
||||||
if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT:
|
if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT:
|
||||||
import uwsgi
|
import uwsgi
|
||||||
|
@ -321,7 +306,7 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
|
||||||
uwsgi.register_signal(99, '', uwsgi_beat)
|
uwsgi.register_signal(99, '', uwsgi_beat)
|
||||||
uwsgi.add_timer(99, 60)
|
uwsgi.add_timer(99, 60)
|
||||||
# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
|
# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
|
||||||
# SHip orchestra production-ready (no DEBUG etc)
|
# Ship orchestra production-ready (no DEBUG etc)
|
||||||
|
|
||||||
# import module and sed
|
# import module and sed
|
||||||
# if setting.value == default. remove
|
# if setting.value == default. remove
|
||||||
|
@ -329,8 +314,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
|
||||||
# inspecting django db connection for asserting db readines? or performing a query
|
# inspecting django db connection for asserting db readines? or performing a query
|
||||||
# wake up django mailer on send_mail
|
# wake up django mailer on send_mail
|
||||||
|
|
||||||
# project settings modified copy of django's default project settings
|
|
||||||
|
|
||||||
# all signals + accouns.register() services.register() on apps.py
|
# all signals + accouns.register() services.register() on apps.py
|
||||||
|
|
||||||
from orchestra.contrib.tasks import task
|
from orchestra.contrib.tasks import task
|
||||||
|
@ -350,44 +333,23 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
|
||||||
TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
|
TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
|
||||||
TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
|
TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
|
||||||
TODO mount the filesystem with "nosuid" option
|
TODO mount the filesystem with "nosuid" option
|
||||||
# wkhtmltopdf -> reportlab
|
|
||||||
# autoiscover modules on app.ready() ? lazy choices on models for plugins
|
|
||||||
# ModelTranslation.register on app.ready()
|
|
||||||
# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
|
# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
|
||||||
|
|
||||||
# mailboxes.address settings multiple local domains, not only one?
|
# mailboxes.address settings multiple local domains, not only one?
|
||||||
# backend.context = self.get_context() or save(obj, context=None)
|
# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data
|
||||||
|
|
||||||
# smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
|
# smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
|
||||||
|
|
||||||
# create registered periodic_task on beat execution: and management command: syncperiodictasks
|
|
||||||
|
|
||||||
# MERGE beats and inspect INSTALLED_APPS and get IS_ENABLED
|
|
||||||
|
|
||||||
# rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
|
# rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
|
||||||
# virtdomains file is not ideal, prevent fake/error on domains there! and make sure this file is required!
|
# virtdomains file is not ideal, prevent fake/error on domains there! and make sure this file is required!
|
||||||
|
|
||||||
# Message last_retry auto_now doesn't work!
|
|
||||||
|
|
||||||
# don't block on beat, and --report periodic tasks
|
|
||||||
|
|
||||||
# Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
|
# Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
|
||||||
|
|
||||||
# orchestrate async stdout stderr (inspired on pangea managemengt commands)
|
# orchestrate async stdout stderr (inspired on pangea managemengt commands)
|
||||||
|
|
||||||
# orchestra-beat support for uwsgi cron
|
# orchestra-beat support for uwsgi cron
|
||||||
|
|
||||||
# message.log if 1: return changeform
|
# message.log if len() == 1: return changeform
|
||||||
|
|
||||||
# generate nginx certs on project dir rather than nginx
|
|
||||||
|
|
||||||
# Register icons
|
|
||||||
|
|
||||||
# send_message doesn't log task
|
|
||||||
|
|
||||||
make django admin taskstate uncollapse fucking traceback, ( if exists ?)
|
make django admin taskstate uncollapse fucking traceback, ( if exists ?)
|
||||||
|
|
||||||
# receive tass stuck at RECEIVED
|
# form for custom message on admin save "comment & save"?
|
||||||
# monitor tasks in started and backend already in success?
|
|
||||||
|
|
||||||
# custom message on admin save
|
|
||||||
|
|
|
@ -1,12 +1,13 @@
|
||||||
from django.core.urlresolvers import reverse
|
from django.core.urlresolvers import reverse
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
from fluent_dashboard import dashboard
|
from fluent_dashboard import dashboard, appsettings
|
||||||
from fluent_dashboard.modules import CmsAppIconList
|
from fluent_dashboard.modules import CmsAppIconList
|
||||||
|
|
||||||
from orchestra.core import services, accounts, administration
|
from orchestra.core import services, accounts, administration
|
||||||
|
|
||||||
|
|
||||||
class AppDefaultIconList(CmsAppIconList):
|
class AppDefaultIconList(CmsAppIconList):
|
||||||
|
""" Provides support for custom default icons """
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.icons = kwargs.pop('icons')
|
self.icons = kwargs.pop('icons')
|
||||||
super(AppDefaultIconList, self).__init__(*args, **kwargs)
|
super(AppDefaultIconList, self).__init__(*args, **kwargs)
|
||||||
|
@ -17,25 +18,28 @@ class AppDefaultIconList(CmsAppIconList):
|
||||||
|
|
||||||
|
|
||||||
class OrchestraIndexDashboard(dashboard.FluentIndexDashboard):
|
class OrchestraIndexDashboard(dashboard.FluentIndexDashboard):
|
||||||
|
""" Gets application modules from services, accounts and administration registries """
|
||||||
def process_registered_view(self, module, view_name, options):
|
def process_registered_view(self, module, view_name, options):
|
||||||
app_name, name = view_name.split('_')[:-1]
|
app_name, name = view_name.split('_')[:-1]
|
||||||
module.icons['.'.join((app_name, name))] = options.get('icon')
|
module.icons['.'.join((app_name, name))] = options.get('icon')
|
||||||
url = reverse('admin:' + view_name)
|
url = reverse('admin:' + view_name)
|
||||||
add_url = '/'.join(url.split('/')[:-2])
|
add_url = '/'.join(url.split('/')[:-2])
|
||||||
module.children.append({
|
module.children.append({
|
||||||
'models': [{
|
'models': [
|
||||||
|
{
|
||||||
'add_url': add_url,
|
'add_url': add_url,
|
||||||
'app_name': app_name,
|
'app_name': app_name,
|
||||||
'change_url': url,
|
'change_url': url,
|
||||||
'name': name,
|
'name': name,
|
||||||
'title': options.get('verbose_name_plural')}],
|
'title': options.get('verbose_name_plural')
|
||||||
|
}
|
||||||
|
],
|
||||||
'name': app_name,
|
'name': app_name,
|
||||||
'title': options.get('verbose_name_plural'),
|
'title': options.get('verbose_name_plural'),
|
||||||
'url': add_url,
|
'url': add_url,
|
||||||
})
|
})
|
||||||
|
|
||||||
def get_application_modules(self):
|
def get_application_modules(self):
|
||||||
from fluent_dashboard import appsettings
|
|
||||||
modules = []
|
modules = []
|
||||||
# Honor settings override, hacky. I Know
|
# Honor settings override, hacky. I Know
|
||||||
if appsettings.FLUENT_DASHBOARD_APP_GROUPS[0][0] != _('CMS'):
|
if appsettings.FLUENT_DASHBOARD_APP_GROUPS[0][0] != _('CMS'):
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
from django.apps import AppConfig
|
from django.apps import AppConfig
|
||||||
|
|
||||||
from orchestra.core import accounts, administration
|
from orchestra.core import accounts, administration
|
||||||
|
from orchestra.core.translations import ModelTranslation
|
||||||
|
|
||||||
|
|
||||||
class IssuesConfig(AppConfig):
|
class IssuesConfig(AppConfig):
|
||||||
|
@ -11,3 +12,4 @@ class IssuesConfig(AppConfig):
|
||||||
from .models import Queue, Ticket
|
from .models import Queue, Ticket
|
||||||
accounts.register(Ticket, icon='Ticket_star.png')
|
accounts.register(Ticket, icon='Ticket_star.png')
|
||||||
administration.register(Queue, dashboard=False)
|
administration.register(Queue, dashboard=False)
|
||||||
|
ModelTranslation.register(Queue, ('verbose_name',))
|
||||||
|
|
|
@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from orchestra.contrib.contacts import settings as contacts_settings
|
from orchestra.contrib.contacts import settings as contacts_settings
|
||||||
from orchestra.contrib.contacts.models import Contact
|
from orchestra.contrib.contacts.models import Contact
|
||||||
from orchestra.core.translations import ModelTranslation
|
|
||||||
from orchestra.models.fields import MultiSelectField
|
from orchestra.models.fields import MultiSelectField
|
||||||
from orchestra.utils.mail import send_email_template
|
from orchestra.utils.mail import send_email_template
|
||||||
|
|
||||||
|
@ -191,6 +190,3 @@ class TicketTracker(models.Model):
|
||||||
unique_together = (
|
unique_together = (
|
||||||
('ticket', 'user'),
|
('ticket', 'user'),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
ModelTranslation.register(Queue, ('verbose_name',))
|
|
||||||
|
|
|
@ -1,10 +1,13 @@
|
||||||
from django.contrib import admin
|
from django.contrib import admin
|
||||||
from django.core.urlresolvers import reverse
|
from django.core.urlresolvers import reverse
|
||||||
from django.db.models import Count
|
from django.db.models import Count, Prefetch
|
||||||
|
from django.shortcuts import redirect
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from orchestra.admin.utils import admin_link, admin_colored, admin_date
|
from orchestra.admin.utils import admin_link, admin_colored, admin_date, wrap_admin_view
|
||||||
|
from orchestra.contrib.tasks import task
|
||||||
|
|
||||||
|
from .engine import send_pending
|
||||||
from .models import Message, SMTPLog
|
from .models import Message, SMTPLog
|
||||||
|
|
||||||
|
|
||||||
|
@ -20,27 +23,51 @@ COLORS = {
|
||||||
|
|
||||||
class MessageAdmin(admin.ModelAdmin):
|
class MessageAdmin(admin.ModelAdmin):
|
||||||
list_display = (
|
list_display = (
|
||||||
'id', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta',
|
'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta',
|
||||||
'retries', 'last_retry_delta', 'num_logs',
|
'retries', 'last_retry_delta', 'num_logs',
|
||||||
)
|
)
|
||||||
list_filter = ('state', 'priority', 'retries')
|
list_filter = ('state', 'priority', 'retries')
|
||||||
|
list_prefetch_related = ('logs__id')
|
||||||
|
|
||||||
colored_state = admin_colored('state', colors=COLORS)
|
colored_state = admin_colored('state', colors=COLORS)
|
||||||
created_at_delta = admin_date('created_at')
|
created_at_delta = admin_date('created_at')
|
||||||
last_retry_delta = admin_date('last_retry')
|
last_retry_delta = admin_date('last_retry')
|
||||||
|
|
||||||
|
def display_subject(self, instance):
|
||||||
|
return instance.subject[:32]
|
||||||
|
display_subject.short_description = _("Subject")
|
||||||
|
display_subject.admin_order_field = 'subject'
|
||||||
|
|
||||||
def num_logs(self, instance):
|
def num_logs(self, instance):
|
||||||
num = instance.logs__count
|
num = instance.logs__count
|
||||||
|
if num == 1:
|
||||||
|
pk = instance.logs.all()[0].id
|
||||||
|
url = reverse('admin:mailer_smtplog_change', args=(pk,))
|
||||||
|
else:
|
||||||
url = reverse('admin:mailer_smtplog_changelist')
|
url = reverse('admin:mailer_smtplog_changelist')
|
||||||
url += '?&message=%i' % instance.pk
|
url += '?&message=%i' % instance.pk
|
||||||
return '<a href="%s">%d</a>' % (url, num)
|
return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, num)
|
||||||
num_logs.short_description = _("Logs")
|
num_logs.short_description = _("Logs")
|
||||||
num_logs.admin_order_field = 'logs__count'
|
num_logs.admin_order_field = 'logs__count'
|
||||||
num_logs.allow_tags = True
|
num_logs.allow_tags = True
|
||||||
|
|
||||||
|
def get_urls(self):
|
||||||
|
from django.conf.urls import url
|
||||||
|
urls = super(MessageAdmin, self).get_urls()
|
||||||
|
info = self.model._meta.app_label, self.model._meta.model_name
|
||||||
|
urls.insert(0,
|
||||||
|
url(r'^send-pending/$', wrap_admin_view(self, self.send_pending_view), name='%s_%s_send_pending' % info)
|
||||||
|
)
|
||||||
|
return urls
|
||||||
|
|
||||||
def get_queryset(self, request):
|
def get_queryset(self, request):
|
||||||
qs = super(MessageAdmin, self).get_queryset(request)
|
qs = super(MessageAdmin, self).get_queryset(request)
|
||||||
return qs.annotate(Count('logs'))
|
return qs.annotate(Count('logs')).prefetch_related('logs')
|
||||||
|
|
||||||
|
def send_pending_view(self, request):
|
||||||
|
task(send_pending).apply_async()
|
||||||
|
self.message_user(request, _("Pending messages are being sent on the background."))
|
||||||
|
return redirect('..')
|
||||||
|
|
||||||
|
|
||||||
class SMTPLogAdmin(admin.ModelAdmin):
|
class SMTPLogAdmin(admin.ModelAdmin):
|
||||||
|
@ -48,6 +75,8 @@ class SMTPLogAdmin(admin.ModelAdmin):
|
||||||
'id', 'message_link', 'colored_result', 'date_delta', 'log_message'
|
'id', 'message_link', 'colored_result', 'date_delta', 'log_message'
|
||||||
)
|
)
|
||||||
list_filter = ('result',)
|
list_filter = ('result',)
|
||||||
|
fields = ('message_link', 'colored_result', 'date_delta', 'log_message')
|
||||||
|
readonly_fields = fields
|
||||||
|
|
||||||
message_link = admin_link('message')
|
message_link = admin_link('message')
|
||||||
colored_result = admin_colored('result', colors=COLORS, bold=False)
|
colored_result = admin_colored('result', colors=COLORS, bold=False)
|
||||||
|
|
|
@ -1,11 +1,15 @@
|
||||||
import smtplib
|
import smtplib
|
||||||
|
from datetime import timedelta
|
||||||
from socket import error as SocketError
|
from socket import error as SocketError
|
||||||
|
|
||||||
from django.core.mail import get_connection
|
from django.core.mail import get_connection
|
||||||
|
from django.db.models import Q
|
||||||
|
from django.utils import timezone
|
||||||
from django.utils.encoding import smart_str
|
from django.utils.encoding import smart_str
|
||||||
|
|
||||||
from orchestra.utils.sys import LockFile
|
from orchestra.utils.sys import LockFile
|
||||||
|
|
||||||
|
from . import settings
|
||||||
from .models import Message
|
from .models import Message
|
||||||
|
|
||||||
|
|
||||||
|
@ -37,11 +41,6 @@ def send_pending(bulk=100):
|
||||||
for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'):
|
for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'):
|
||||||
send_message(message, num, connection, bulk)
|
send_message(message, num, connection, bulk)
|
||||||
num += 1
|
num += 1
|
||||||
from django.utils import timezone
|
|
||||||
from . import settings
|
|
||||||
from datetime import timedelta
|
|
||||||
from django.db.models import Q
|
|
||||||
|
|
||||||
now = timezone.now()
|
now = timezone.now()
|
||||||
qs = Q()
|
qs = Q()
|
||||||
for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
|
for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
|
||||||
|
|
|
@ -2,10 +2,13 @@ import json
|
||||||
|
|
||||||
from django.core.management.base import BaseCommand, CommandError
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
|
||||||
|
from orchestra.contrib.tasks.decorators import keep_state
|
||||||
|
|
||||||
from ...engine import send_pending
|
from ...engine import send_pending
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
help = 'Runs Orchestra method.'
|
help = 'Runs Orchestra method.'
|
||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
send_pending()
|
keep_state(send_pending)()
|
||||||
|
|
|
@ -38,6 +38,9 @@ class Message(models.Model):
|
||||||
# TODO rename to last_try
|
# TODO rename to last_try
|
||||||
last_retry = models.DateTimeField(_("last try"), auto_now=True)
|
last_retry = models.DateTimeField(_("last try"), auto_now=True)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return '%s to %s' % (self.subject, self.to_address)
|
||||||
|
|
||||||
def defer(self):
|
def defer(self):
|
||||||
self.state = self.DEFERRED
|
self.state = self.DEFERRED
|
||||||
# Max tries
|
# Max tries
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
{% extends "admin/change_list.html" %}
|
||||||
|
{% load i18n admin_urls admin_static admin_list %}
|
||||||
|
|
||||||
|
|
||||||
|
{% block object-tools-items %}
|
||||||
|
<li>
|
||||||
|
{% url cl.opts|admin_urlname:'send_pending' as send_pending_url %}
|
||||||
|
<a href="{% add_preserved_filters send_pending_url is_popup to_field %}" class="historylink">
|
||||||
|
{% blocktrans with cl.opts.verbose_name as name %}Send pending{% endblocktrans %}
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
{{ block.super }}
|
||||||
|
{% endblock %}
|
||||||
|
|
|
@ -33,7 +33,7 @@ class Operation():
|
||||||
self.routes = routes
|
self.routes = routes
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def execute(cls, operations, serialize=False, async=False):
|
def execute(cls, operations, serialize=False, async=None):
|
||||||
from . import manager
|
from . import manager
|
||||||
scripts, oserialize = manager.generate(operations)
|
scripts, oserialize = manager.generate(operations)
|
||||||
return manager.execute(scripts, serialize=(serialize or oserialize), async=async)
|
return manager.execute(scripts, serialize=(serialize or oserialize), async=async)
|
||||||
|
|
|
@ -97,8 +97,13 @@ def generate(operations):
|
||||||
return scripts, serialize
|
return scripts, serialize
|
||||||
|
|
||||||
|
|
||||||
def execute(scripts, serialize=False, async=False):
|
def execute(scripts, serialize=False, async=None):
|
||||||
""" executes the operations on the servers """
|
"""
|
||||||
|
executes the operations on the servers
|
||||||
|
|
||||||
|
serialize: execute one backend at a time
|
||||||
|
async: do not join threads (overrides route.async)
|
||||||
|
"""
|
||||||
if settings.ORCHESTRATION_DISABLE_EXECUTION:
|
if settings.ORCHESTRATION_DISABLE_EXECUTION:
|
||||||
logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION settings.')
|
logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION settings.')
|
||||||
return []
|
return []
|
||||||
|
@ -110,7 +115,10 @@ def execute(scripts, serialize=False, async=False):
|
||||||
route, __ = key
|
route, __ = key
|
||||||
backend, operations = value
|
backend, operations = value
|
||||||
args = (route.host,)
|
args = (route.host,)
|
||||||
async = not serialize and (async or route.async)
|
if async is None:
|
||||||
|
async = not serialize and route.async
|
||||||
|
else:
|
||||||
|
async = not serialize and async
|
||||||
kwargs = {
|
kwargs = {
|
||||||
'async': async,
|
'async': async,
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ from django.db.models import Q
|
||||||
from django.utils.functional import cached_property
|
from django.utils.functional import cached_property
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from orchestra.core import services, accounts
|
|
||||||
from orchestra.core.validators import validate_name
|
from orchestra.core.validators import validate_name
|
||||||
from orchestra.models import queryset
|
from orchestra.models import queryset
|
||||||
|
|
||||||
|
|
|
@ -11,7 +11,7 @@ from orchestra.admin import ExtendedModelAdmin
|
||||||
from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
|
from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
|
||||||
from orchestra.contrib.orchestration.models import Route
|
from orchestra.contrib.orchestration.models import Route
|
||||||
from orchestra.core import services
|
from orchestra.core import services
|
||||||
from orchestra.utils.db import database_ready
|
from orchestra.utils import db, sys
|
||||||
from orchestra.utils.functional import cached
|
from orchestra.utils.functional import cached
|
||||||
|
|
||||||
from .actions import run_monitor
|
from .actions import run_monitor
|
||||||
|
@ -45,7 +45,9 @@ class ResourceAdmin(ExtendedModelAdmin):
|
||||||
actions = (run_monitor,)
|
actions = (run_monitor,)
|
||||||
change_view_actions = actions
|
change_view_actions = actions
|
||||||
change_readonly_fields = ('name', 'content_type')
|
change_readonly_fields = ('name', 'content_type')
|
||||||
prepopulated_fields = {'name': ('verbose_name',)}
|
prepopulated_fields = {
|
||||||
|
'name': ('verbose_name',)
|
||||||
|
}
|
||||||
list_select_related = ('content_type', 'crontab',)
|
list_select_related = ('content_type', 'crontab',)
|
||||||
|
|
||||||
def change_view(self, request, object_id, form_url='', extra_context=None):
|
def change_view(self, request, object_id, form_url='', extra_context=None):
|
||||||
|
@ -70,6 +72,7 @@ class ResourceAdmin(ExtendedModelAdmin):
|
||||||
|
|
||||||
def save_model(self, request, obj, form, change):
|
def save_model(self, request, obj, form, change):
|
||||||
super(ResourceAdmin, self).save_model(request, obj, form, change)
|
super(ResourceAdmin, self).save_model(request, obj, form, change)
|
||||||
|
# best-effort
|
||||||
model = obj.content_type.model_class()
|
model = obj.content_type.model_class()
|
||||||
modeladmin = type(get_modeladmin(model))
|
modeladmin = type(get_modeladmin(model))
|
||||||
resources = obj.content_type.resource_set.filter(is_active=True)
|
resources = obj.content_type.resource_set.filter(is_active=True)
|
||||||
|
@ -79,6 +82,8 @@ class ResourceAdmin(ExtendedModelAdmin):
|
||||||
inline = resource_inline_factory(resources)
|
inline = resource_inline_factory(resources)
|
||||||
inlines.append(inline)
|
inlines.append(inline)
|
||||||
modeladmin.inlines = inlines
|
modeladmin.inlines = inlines
|
||||||
|
# reload Not always work
|
||||||
|
sys.touch_wsgi()
|
||||||
|
|
||||||
def formfield_for_dbfield(self, db_field, **kwargs):
|
def formfield_for_dbfield(self, db_field, **kwargs):
|
||||||
""" filter service content_types """
|
""" filter service content_types """
|
||||||
|
@ -271,5 +276,5 @@ def insert_resource_inlines():
|
||||||
insertattr(model, 'inlines', inline)
|
insertattr(model, 'inlines', inline)
|
||||||
|
|
||||||
|
|
||||||
if database_ready():
|
if db.database_ready():
|
||||||
insert_resource_inlines()
|
insert_resource_inlines()
|
||||||
|
|
|
@ -9,8 +9,6 @@ from django.utils.translation import ugettext_lazy as _
|
||||||
from orchestra.core import validators
|
from orchestra.core import validators
|
||||||
from orchestra.models import queryset, fields
|
from orchestra.models import queryset, fields
|
||||||
from orchestra.models.utils import get_model_field_path
|
from orchestra.models.utils import get_model_field_path
|
||||||
from orchestra.utils.paths import get_project_dir
|
|
||||||
from orchestra.utils.sys import run
|
|
||||||
|
|
||||||
from . import tasks, settings
|
from . import tasks, settings
|
||||||
from .backends import ServiceMonitor
|
from .backends import ServiceMonitor
|
||||||
|
@ -122,7 +120,6 @@ class Resource(models.Model):
|
||||||
self.sync_periodic_task()
|
self.sync_periodic_task()
|
||||||
# This only work on tests (multiprocessing used on real deployments)
|
# This only work on tests (multiprocessing used on real deployments)
|
||||||
apps.get_app_config('resources').reload_relations()
|
apps.get_app_config('resources').reload_relations()
|
||||||
run('{ sleep 2 && touch %s/wsgi.py; } &' % get_project_dir(), async=True)
|
|
||||||
|
|
||||||
def delete(self, *args, **kwargs):
|
def delete(self, *args, **kwargs):
|
||||||
super(Resource, self).delete(*args, **kwargs)
|
super(Resource, self).delete(*args, **kwargs)
|
||||||
|
|
|
@ -73,7 +73,7 @@ class SettingView(generic.edit.FormView):
|
||||||
|
|
||||||
# Save changes
|
# Save changes
|
||||||
parser.save(changes)
|
parser.save(changes)
|
||||||
sys.run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True)
|
sys.touch_wsgi()
|
||||||
n = len(changes)
|
n = len(changes)
|
||||||
context = {
|
context = {
|
||||||
'message': ngettext(
|
'message': ngettext(
|
||||||
|
|
|
@ -6,7 +6,7 @@ from django.db import models
|
||||||
from django.utils.functional import cached_property
|
from django.utils.functional import cached_property
|
||||||
from django.utils.translation import ugettext_lazy as _
|
from django.utils.translation import ugettext_lazy as _
|
||||||
|
|
||||||
from orchestra.core import services, validators
|
from orchestra.core import validators
|
||||||
|
|
||||||
from . import settings
|
from . import settings
|
||||||
|
|
||||||
|
|
|
@ -18,14 +18,18 @@ from .utils import get_name, get_id
|
||||||
def keep_state(fn):
|
def keep_state(fn):
|
||||||
""" logs task on djcelery's TaskState model """
|
""" logs task on djcelery's TaskState model """
|
||||||
@wraps(fn)
|
@wraps(fn)
|
||||||
def wrapper(task_id, name, *args, **kwargs):
|
def wrapper(*args, _task_id=None, _name=None, **kwargs):
|
||||||
from djcelery.models import TaskState
|
from djcelery.models import TaskState
|
||||||
now = timezone.now()
|
now = timezone.now()
|
||||||
state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
|
if _task_id is None:
|
||||||
|
_task_id = get_id()
|
||||||
|
if _name is None:
|
||||||
|
_name = get_name(fn)
|
||||||
|
state = TaskState.objects.create(state=states.STARTED, task_id=_task_id, name=_name, args=str(args),
|
||||||
kwargs=str(kwargs), tstamp=now)
|
kwargs=str(kwargs), tstamp=now)
|
||||||
try:
|
try:
|
||||||
result = fn(*args, **kwargs)
|
result = fn(*args, **kwargs)
|
||||||
except Exception as exc:
|
except:
|
||||||
state.state = states.FAILURE
|
state.state = states.FAILURE
|
||||||
state.traceback = trace
|
state.traceback = trace
|
||||||
state.runtime = (timezone.now()-now).total_seconds()
|
state.runtime = (timezone.now()-now).total_seconds()
|
||||||
|
@ -35,7 +39,7 @@ def keep_state(fn):
|
||||||
logger.error(subject)
|
logger.error(subject)
|
||||||
logger.error(trace)
|
logger.error(trace)
|
||||||
mail_admins(subject, trace)
|
mail_admins(subject, trace)
|
||||||
return
|
raise
|
||||||
else:
|
else:
|
||||||
state.state = states.SUCCESS
|
state.state = states.SUCCESS
|
||||||
state.result = str(result)
|
state.result = str(result)
|
||||||
|
@ -49,7 +53,10 @@ def apply_async(fn, name=None, method='thread'):
|
||||||
""" replaces celery apply_async """
|
""" replaces celery apply_async """
|
||||||
def inner(fn, name, method, *args, **kwargs):
|
def inner(fn, name, method, *args, **kwargs):
|
||||||
task_id = get_id()
|
task_id = get_id()
|
||||||
args = (task_id, name) + args
|
kwargs.update({
|
||||||
|
'_name': name,
|
||||||
|
'_task_id': task_id,
|
||||||
|
})
|
||||||
thread = method(target=fn, args=args, kwargs=kwargs)
|
thread = method(target=fn, args=args, kwargs=kwargs)
|
||||||
thread.start()
|
thread.start()
|
||||||
# Celery API compat
|
# Celery API compat
|
||||||
|
|
|
@ -6,7 +6,6 @@ from django.utils import timezone
|
||||||
from djcelery.models import PeriodicTask
|
from djcelery.models import PeriodicTask
|
||||||
|
|
||||||
from ...decorators import keep_state
|
from ...decorators import keep_state
|
||||||
from ...utils import get_id, get_name
|
|
||||||
|
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
|
@ -46,4 +45,4 @@ class Command(BaseCommand):
|
||||||
arguments.append(arg)
|
arguments.append(arg)
|
||||||
args = arguments
|
args = arguments
|
||||||
# Run task synchronously, but logging TaskState
|
# Run task synchronously, but logging TaskState
|
||||||
keep_state(task)(get_id(), get_name(task), *args, **kwargs)
|
keep_state(task)(*args, **kwargs)
|
||||||
|
|
|
@ -19,9 +19,11 @@ class Command(BaseCommand):
|
||||||
make_option('--cert-key', dest='cert_key', default='',
|
make_option('--cert-key', dest='cert_key', default='',
|
||||||
help='Nginx SSL certificate key.'),
|
help='Nginx SSL certificate key.'),
|
||||||
|
|
||||||
make_option('--cert-path', dest='cert_path', default='/etc/nginx/ssl/orchestra.crt',
|
make_option('--cert-path', dest='cert_path',
|
||||||
|
default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.crt'),
|
||||||
help='Nginx SSL certificate, one will be created by default.'),
|
help='Nginx SSL certificate, one will be created by default.'),
|
||||||
make_option('--cert-key-path', dest='cert_key_path', default='/etc/nginx/ssl/orchestra.key',
|
make_option('--cert-key-path', dest='cert_key_path',
|
||||||
|
default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.key'),
|
||||||
help='Nginx SSL certificate key.'),
|
help='Nginx SSL certificate key.'),
|
||||||
# Cert options
|
# Cert options
|
||||||
make_option('--cert-override', dest='cert_override', action='store_true',
|
make_option('--cert-override', dest='cert_override', action='store_true',
|
||||||
|
|
|
@ -195,3 +195,8 @@ class LockFile(object):
|
||||||
def __exit__(self, type, value, traceback):
|
def __exit__(self, type, value, traceback):
|
||||||
if not self.unlocked:
|
if not self.unlocked:
|
||||||
self.release()
|
self.release()
|
||||||
|
|
||||||
|
|
||||||
|
def touch_wsgi():
|
||||||
|
from . import paths
|
||||||
|
run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True)
|
||||||
|
|
|
@ -106,6 +106,7 @@ if [[ $CELERY == true ]]; then
|
||||||
sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2
|
sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2
|
||||||
else
|
else
|
||||||
surun "$PYTHON_BIN $MANAGE setupcronbeat"
|
surun "$PYTHON_BIN $MANAGE setupcronbeat"
|
||||||
|
surun "$PYTHON_BIN $MANAGE syncperiodictasks"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue