pedro/django-orchestra-test
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Reandom fixes

Browse source
This commit is contained in:
Marc Aymerich 2015-05-07 19:00:02 +00:00
parent 34596c3485
commit b5a3f64aa9
23 changed files with 133 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
INSTALL.md
View file

@ -53,7 +53,8 @@ Django-orchestra can be installed on any Linux system, however it is **strongly
6. Configure periodic execution of tasks (choose one)
1. Use cron
```bash
sudo python3 manage.py setupcronbeat
python3 manage.py setupcronbeat
python3 panel/manage.py syncperiodictasks
```
2. Use celeryd

1
README.md
View file

@ -55,6 +55,7 @@ python3 panel/manage.py runserver
# Enable periodic tasks with cron (optional)
python3 panel/manage.py setupcronbeat
python3 panel/manage.py syncperiodictasks
```
Now you can see the web interface on http://localhost:8000/admin

52
TODO.md
View file

@ -288,9 +288,9 @@ https://code.djangoproject.com/ticket/24576
# insert settings on dashboard dynamically
# convert all complex settings to string
# @ something database names
# size monitor of @002 @003 database names
# password validation cracklib on change password form=?????
# reset setting buton
# reset setting button
# periodic cleaning of spam mailboxes
@ -298,21 +298,6 @@ https://code.djangoproject.com/ticket/24576
# django SITE_NAME vs ORCHESTRA_SITE_NAME ?
Replace celery by a custom solution?
# TODO create decorator wrapper that abstract the task away from the backen (cron/celery)
# TODO crontab model localhost/autoadded attribute
* No more jumbo dependencies and wierd bugs
1) Periodic Monitoring:
* runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system)
2) Single time shot:
sys.run("python3 manage.py runtas 'task' args")
3) Emails:
Custom backend that distinguishes between priority and bulk mail
*priority: custom Thread backend
*bulk: wrapper arround django-mailer to avoid loading django system
pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
# TASKS_ENABLE_UWSGI_CRON_BEAT (default) for production + system check --deploy
if 'wsgi' in sys.argv and settings.TASKS_ENABLE_UWSGI_CRON_BEAT:
import uwsgi
@ -321,7 +306,7 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
uwsgi.register_signal(99, '', uwsgi_beat)
uwsgi.add_timer(99, 60)
# TASK_BEAT_BACKEND = ('cron', 'celerybeat', 'uwsgi')
# SHip orchestra production-ready (no DEBUG etc)
# Ship orchestra production-ready (no DEBUG etc)
# import module and sed
# if setting.value == default. remove
@ -329,8 +314,6 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
# inspecting django db connection for asserting db readines? or performing a query
# wake up django mailer on send_mail
# project settings modified copy of django's default project settings
# all signals + accouns.register() services.register() on apps.py
from orchestra.contrib.tasks import task
@ -350,44 +333,23 @@ pip3 install https://github.com/APSL/django-mailer-2/archive/master.zip
TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
TODO mount the filesystem with "nosuid" option
# wkhtmltopdf -> reportlab
# autoiscover modules on app.ready() ? lazy choices on models for plugins
# ModelTranslation.register on app.ready()
# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
# mailboxes.address settings multiple local domains, not only one?
# backend.context = self.get_context() or save(obj, context=None)
# backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data
# smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
# create registered periodic_task on beat execution: and management command: syncperiodictasks
# MERGE beats and inspect INSTALLED_APPS and get IS_ENABLED
# rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
# virtdomains file is not ideal, prevent fake/error on domains there! and make sure this file is required!
# Message last_retry auto_now doesn't work!
# don't block on beat, and --report periodic tasks
# Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
# orchestrate async stdout stderr (inspired on pangea managemengt commands)
# orchestra-beat support for uwsgi cron
# message.log if 1: return changeform
# generate nginx certs on project dir rather than nginx
# Register icons
# send_message doesn't log task
# message.log if len() == 1: return changeform
make django admin taskstate uncollapse fucking traceback, ( if exists ?)
# receive tass stuck at RECEIVED
# monitor tasks in started and backend already in success?
# custom message on admin save
# form for custom message on admin save "comment & save"?

26
orchestra/admin/dashboard.py
View file

@ -1,12 +1,13 @@
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from fluent_dashboard import dashboard
from fluent_dashboard import dashboard, appsettings
from fluent_dashboard.modules import CmsAppIconList
from orchestra.core import services, accounts, administration
class AppDefaultIconList(CmsAppIconList):
""" Provides support for custom default icons """
def __init__(self, *args, **kwargs):
self.icons = kwargs.pop('icons')
super(AppDefaultIconList, self).__init__(*args, **kwargs)
@ -17,25 +18,28 @@ class AppDefaultIconList(CmsAppIconList):
class OrchestraIndexDashboard(dashboard.FluentIndexDashboard):
""" Gets application modules from services, accounts and administration registries """
def process_registered_view(self, module, view_name, options):
app_name, name = view_name.split('_')[:-1]
module.icons['.'.join((app_name, name))] = options.get('icon')
url = reverse('admin:' + view_name)
add_url = '/'.join(url.split('/')[:-2])
module.children.append({
'models': [{
'add_url': add_url,
'app_name': app_name,
'change_url': url,
'name': name,
'title': options.get('verbose_name_plural')}],
'name': app_name,
'title': options.get('verbose_name_plural'),
'url': add_url,
'models': [
{
'add_url': add_url,
'app_name': app_name,
'change_url': url,
'name': name,
'title': options.get('verbose_name_plural')
}
],
'name': app_name,
'title': options.get('verbose_name_plural'),
'url': add_url,
})
def get_application_modules(self):
from fluent_dashboard import appsettings
modules = []
# Honor settings override, hacky. I Know
if appsettings.FLUENT_DASHBOARD_APP_GROUPS[0][0] != _('CMS'):

2
orchestra/contrib/issues/apps.py
View file

@ -1,6 +1,7 @@
from django.apps import AppConfig
from orchestra.core import accounts, administration
from orchestra.core.translations import ModelTranslation
class IssuesConfig(AppConfig):
@ -11,3 +12,4 @@ class IssuesConfig(AppConfig):
from .models import Queue, Ticket
accounts.register(Ticket, icon='Ticket_star.png')
administration.register(Queue, dashboard=False)
ModelTranslation.register(Queue, ('verbose_name',))

4
orchestra/contrib/issues/models.py
View file

@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.contacts import settings as contacts_settings
from orchestra.contrib.contacts.models import Contact
from orchestra.core.translations import ModelTranslation
from orchestra.models.fields import MultiSelectField
from orchestra.utils.mail import send_email_template
@ -191,6 +190,3 @@ class TicketTracker(models.Model):
unique_together = (
('ticket', 'user'),
)
ModelTranslation.register(Queue, ('verbose_name',))

43
orchestra/contrib/mailer/admin.py
View file

@ -1,10 +1,13 @@
from django.contrib import admin
from django.core.urlresolvers import reverse
from django.db.models import Count
from django.db.models import Count, Prefetch
from django.shortcuts import redirect
from django.utils.translation import ugettext_lazy as _
from orchestra.admin.utils import admin_link, admin_colored, admin_date
from orchestra.admin.utils import admin_link, admin_colored, admin_date, wrap_admin_view
from orchestra.contrib.tasks import task
from .engine import send_pending
from .models import Message, SMTPLog
@ -20,27 +23,51 @@ COLORS = {
class MessageAdmin(admin.ModelAdmin):
list_display = (
'id', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta',
'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', 'created_at_delta',
'retries', 'last_retry_delta', 'num_logs',
)
list_filter = ('state', 'priority', 'retries')
list_prefetch_related = ('logs__id')
colored_state = admin_colored('state', colors=COLORS)
created_at_delta = admin_date('created_at')
last_retry_delta = admin_date('last_retry')
def display_subject(self, instance):
return instance.subject[:32]
display_subject.short_description = _("Subject")
display_subject.admin_order_field = 'subject'
def num_logs(self, instance):
num = instance.logs__count
url = reverse('admin:mailer_smtplog_changelist')
url += '?&message=%i' % instance.pk
return '<a href="%s">%d</a>' % (url, num)
if num == 1:
pk = instance.logs.all()[0].id
url = reverse('admin:mailer_smtplog_change', args=(pk,))
else:
url = reverse('admin:mailer_smtplog_changelist')
url += '?&message=%i' % instance.pk
return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, num)
num_logs.short_description = _("Logs")
num_logs.admin_order_field = 'logs__count'
num_logs.allow_tags = True
def get_urls(self):
from django.conf.urls import url
urls = super(MessageAdmin, self).get_urls()
info = self.model._meta.app_label, self.model._meta.model_name
urls.insert(0,
url(r'^send-pending/$', wrap_admin_view(self, self.send_pending_view), name='%s_%s_send_pending' % info)
)
return urls
def get_queryset(self, request):
qs = super(MessageAdmin, self).get_queryset(request)
return qs.annotate(Count('logs'))
return qs.annotate(Count('logs')).prefetch_related('logs')
def send_pending_view(self, request):
task(send_pending).apply_async()
self.message_user(request, _("Pending messages are being sent on the background."))
return redirect('..')
class SMTPLogAdmin(admin.ModelAdmin):
@ -48,6 +75,8 @@ class SMTPLogAdmin(admin.ModelAdmin):
'id', 'message_link', 'colored_result', 'date_delta', 'log_message'
)
list_filter = ('result',)
fields = ('message_link', 'colored_result', 'date_delta', 'log_message')
readonly_fields = fields
message_link = admin_link('message')
colored_result = admin_colored('result', colors=COLORS, bold=False)

9
orchestra/contrib/mailer/engine.py
View file

@ -1,11 +1,15 @@
import smtplib
from datetime import timedelta
from socket import error as SocketError
from django.core.mail import get_connection
from django.db.models import Q
from django.utils import timezone
from django.utils.encoding import smart_str
from orchestra.utils.sys import LockFile
from . import settings
from .models import Message
@ -37,11 +41,6 @@ def send_pending(bulk=100):
for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'):
send_message(message, num, connection, bulk)
num += 1
from django.utils import timezone
from . import settings
from datetime import timedelta
from django.db.models import Q
now = timezone.now()
qs = Q()
for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):

5
orchestra/contrib/mailer/management/commands/sendpendingmessages.py
View file

@ -2,10 +2,13 @@ import json
from django.core.management.base import BaseCommand, CommandError
from orchestra.contrib.tasks.decorators import keep_state
from ...engine import send_pending
class Command(BaseCommand):
help = 'Runs Orchestra method.'
def handle(self, *args, **options):
send_pending()
keep_state(send_pending)()

3
orchestra/contrib/mailer/models.py
View file

@ -38,6 +38,9 @@ class Message(models.Model):
# TODO rename to last_try
last_retry = models.DateTimeField(_("last try"), auto_now=True)
def __str__(self):
return '%s to %s' % (self.subject, self.to_address)
def defer(self):
self.state = self.DEFERRED
# Max tries

14
orchestra/contrib/mailer/templates/admin/mailer/message/change_list.html Normal file
View file

@ -0,0 +1,14 @@
{% extends "admin/change_list.html" %}
{% load i18n admin_urls admin_static admin_list %}
{% block object-tools-items %}
<li>
{% url cl.opts|admin_urlname:'send_pending' as send_pending_url %}
<a href="{% add_preserved_filters send_pending_url is_popup to_field %}" class="historylink">
{% blocktrans with cl.opts.verbose_name as name %}Send pending{% endblocktrans %}
</a>
</li>
{{ block.super }}
{% endblock %}

2
orchestra/contrib/orchestration/__init__.py
View file

@ -33,7 +33,7 @@ class Operation():
self.routes = routes
@classmethod
def execute(cls, operations, serialize=False, async=False):
def execute(cls, operations, serialize=False, async=None):
from . import manager
scripts, oserialize = manager.generate(operations)
return manager.execute(scripts, serialize=(serialize or oserialize), async=async)

14
orchestra/contrib/orchestration/manager.py
View file

@ -97,8 +97,13 @@ def generate(operations):
return scripts, serialize
def execute(scripts, serialize=False, async=False):
""" executes the operations on the servers """
def execute(scripts, serialize=False, async=None):
"""
executes the operations on the servers
serialize: execute one backend at a time
async: do not join threads (overrides route.async)
"""
if settings.ORCHESTRATION_DISABLE_EXECUTION:
logger.info('Orchestration execution is dissabled by ORCHESTRATION_DISABLE_EXECUTION settings.')
return []
@ -110,7 +115,10 @@ def execute(scripts, serialize=False, async=False):
route, __ = key
backend, operations = value
args = (route.host,)
async = not serialize and (async or route.async)
if async is None:
async = not serialize and route.async
else:
async = not serialize and async
kwargs = {
'async': async,
}

1
orchestra/contrib/plans/models.py
View file

@ -4,7 +4,6 @@ from django.db.models import Q
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from orchestra.core import services, accounts
from orchestra.core.validators import validate_name
from orchestra.models import queryset

11
orchestra/contrib/resources/admin.py
View file

@ -11,7 +11,7 @@ from orchestra.admin import ExtendedModelAdmin
from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
from orchestra.contrib.orchestration.models import Route
from orchestra.core import services
from orchestra.utils.db import database_ready
from orchestra.utils import db, sys
from orchestra.utils.functional import cached
from .actions import run_monitor
@ -45,7 +45,9 @@ class ResourceAdmin(ExtendedModelAdmin):
actions = (run_monitor,)
change_view_actions = actions
change_readonly_fields = ('name', 'content_type')
prepopulated_fields = {'name': ('verbose_name',)}
prepopulated_fields = {
'name': ('verbose_name',)
}
list_select_related = ('content_type', 'crontab',)
def change_view(self, request, object_id, form_url='', extra_context=None):
@ -70,6 +72,7 @@ class ResourceAdmin(ExtendedModelAdmin):
def save_model(self, request, obj, form, change):
super(ResourceAdmin, self).save_model(request, obj, form, change)
# best-effort
model = obj.content_type.model_class()
modeladmin = type(get_modeladmin(model))
resources = obj.content_type.resource_set.filter(is_active=True)
@ -79,6 +82,8 @@ class ResourceAdmin(ExtendedModelAdmin):
inline = resource_inline_factory(resources)
inlines.append(inline)
modeladmin.inlines = inlines
# reload Not always work
sys.touch_wsgi()
def formfield_for_dbfield(self, db_field, **kwargs):
""" filter service content_types """
@ -271,5 +276,5 @@ def insert_resource_inlines():
insertattr(model, 'inlines', inline)
if database_ready():
if db.database_ready():
insert_resource_inlines()

3
orchestra/contrib/resources/models.py
View file

@ -9,8 +9,6 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.core import validators
from orchestra.models import queryset, fields
from orchestra.models.utils import get_model_field_path
from orchestra.utils.paths import get_project_dir
from orchestra.utils.sys import run
from . import tasks, settings
from .backends import ServiceMonitor
@ -122,7 +120,6 @@ class Resource(models.Model):
self.sync_periodic_task()
# This only work on tests (multiprocessing used on real deployments)
apps.get_app_config('resources').reload_relations()
run('{ sleep 2 && touch %s/wsgi.py; } &' % get_project_dir(), async=True)
def delete(self, *args, **kwargs):
super(Resource, self).delete(*args, **kwargs)

2
orchestra/contrib/settings/admin.py
View file

@ -73,7 +73,7 @@ class SettingView(generic.edit.FormView):
# Save changes
parser.save(changes)
sys.run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True)
sys.touch_wsgi()
n = len(changes)
context = {
'message': ngettext(

2
orchestra/contrib/systemusers/models.py
View file

@ -6,7 +6,7 @@ from django.db import models
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from orchestra.core import services, validators
from orchestra.core import validators
from . import settings

17
orchestra/contrib/tasks/decorators.py
View file

@ -18,14 +18,18 @@ from .utils import get_name, get_id
def keep_state(fn):
""" logs task on djcelery's TaskState model """
@wraps(fn)
def wrapper(task_id, name, *args, **kwargs):
def wrapper(*args, _task_id=None, _name=None, **kwargs):
from djcelery.models import TaskState
now = timezone.now()
state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
if _task_id is None:
_task_id = get_id()
if _name is None:
_name = get_name(fn)
state = TaskState.objects.create(state=states.STARTED, task_id=_task_id, name=_name, args=str(args),
kwargs=str(kwargs), tstamp=now)
try:
result = fn(*args, **kwargs)
except Exception as exc:
except:
state.state = states.FAILURE
state.traceback = trace
state.runtime = (timezone.now()-now).total_seconds()
@ -35,7 +39,7 @@ def keep_state(fn):
logger.error(subject)
logger.error(trace)
mail_admins(subject, trace)
return
raise
else:
state.state = states.SUCCESS
state.result = str(result)
@ -49,7 +53,10 @@ def apply_async(fn, name=None, method='thread'):
""" replaces celery apply_async """
def inner(fn, name, method, *args, **kwargs):
task_id = get_id()
args = (task_id, name) + args
kwargs.update({
'_name': name,
'_task_id': task_id,
})
thread = method(target=fn, args=args, kwargs=kwargs)
thread.start()
# Celery API compat

3
orchestra/contrib/tasks/management/commands/runtask.py
View file

@ -6,7 +6,6 @@ from django.utils import timezone
from djcelery.models import PeriodicTask
from ...decorators import keep_state
from ...utils import get_id, get_name
class Command(BaseCommand):
@ -46,4 +45,4 @@ class Command(BaseCommand):
arguments.append(arg)
args = arguments
# Run task synchronously, but logging TaskState
keep_state(task)(get_id(), get_name(task), *args, **kwargs)
keep_state(task)(*args, **kwargs)

6
orchestra/management/commands/setupnginx.py
View file

@ -19,9 +19,11 @@ class Command(BaseCommand):
make_option('--cert-key', dest='cert_key', default='',
help='Nginx SSL certificate key.'),
make_option('--cert-path', dest='cert_path', default='/etc/nginx/ssl/orchestra.crt',
make_option('--cert-path', dest='cert_path',
default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.crt'),
help='Nginx SSL certificate, one will be created by default.'),
make_option('--cert-key-path', dest='cert_key_path', default='/etc/nginx/ssl/orchestra.key',
make_option('--cert-key-path', dest='cert_key_path',
default=os.path.join(paths.get_site_dir(), 'ssl', 'orchestra.key'),
help='Nginx SSL certificate key.'),
# Cert options
make_option('--cert-override', dest='cert_override', action='store_true',

5
orchestra/utils/sys.py
View file

@ -195,3 +195,8 @@ class LockFile(object):
def __exit__(self, type, value, traceback):
if not self.unlocked:
self.release()
def touch_wsgi():
from . import paths
run('{ sleep 2 && touch %s/wsgi.py; } &' % paths.get_project_dir(), async=True)

1
scripts/container/deploy.sh
View file

@ -106,6 +106,7 @@ if [[ $CELERY == true ]]; then
sudo $PYTHON_BIN $MANAGE setupcelery --username $USER --processes 2
else
surun "$PYTHON_BIN $MANAGE setupcronbeat"
surun "$PYTHON_BIN $MANAGE syncperiodictasks"
fi