pedro
/
django-orchestra-test
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added tasks app

This commit is contained in:
Marc Aymerich 2015-05-03 17:44:46 +00:00
parent 1bb37b1175
commit 184436dbe4
28 changed files with 695 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
TODO.md
View File

@ -297,13 +297,8 @@ https://code.djangoproject.com/ticket/24576
# admin edit relevant djanog settings
# django SITE_NAME vs ORCHESTRA_SITE_NAME ?
# accounts.migrations link to last auth migration instead of first
# DNS allow transfer other NS servers instead of masters and slaves!
Replace celery by a custom solution?
# TODO create periodic task like settings, but parsing cronfiles!
# TODO create decorator wrapper that abstract the task away from the backen (cron/celery)
# TODO crontab model localhost/autoadded attribute
* No more jumbo dependencies and wierd bugs
@ -316,15 +311,49 @@ Replace celery by a custom solution?
*priority: custom Thread backend
*bulk: wrapper arround django-mailer to avoid loading django system
python3 -mvenv env-django-orchestra
source env-django-orchestra/bin/activate
pip3 install django-orchestra==dev --allow-external django-orchestra --allow-unverified django-orchestra
pip3 install -r https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt
# TODO make them optional
sudo apt-get install python3.4-dev libxml2-dev libxslt1-dev libcrack2-dev
wget -O - https://raw.githubusercontent.com/glic3rinu/django-orchestra/master/requirements.txt | xargs pip3 install
django-admin.py startproject panel --template="$HOME/django-orchestra/orchestra/conf/project_template"
python3 panel/manage.py migrate accounts
python3 panel/manage.py migrate
python3 panel/manage.py runserver
Collecting lxml==3.3.5 (from -r re (line 22))
Downloading lxml-3.3.5.tar.gz (3.5MB)
100% |################################| 3.5MB 60kB/s
Building lxml version 3.3.5.
Building without Cython.
ERROR: b'/bin/sh: 1: xslt-config: not found\n'
** make sure the development packages of libxml2 and libxslt are installed **
Using build configuration of libxslt
/usr/lib/python3.4/distutils/dist.py:260: UserWarning: Unknown distribution option: 'bugtrack_url'
warnings.warn(msg)
# Setupcron
# uwsgi enable threads
# Create superuser on migrate
# register signals in app ready()
def ready(self):
if self.has_attr('ready_run'): return
self.ready_run = True
# database_ready(): connect to the database or inspect django connection
# beat.sh
# do settings validation on orchestra.apps.ready(), not during startime
# move Setting to contrib app __init__
# cracklib vs crack
# remove system dependencies
# deprecate install_dependnecies in favour of only requirements.txt
# import module and sed
# if setting.value == default. remove
# cron backend: os.cron or uwsgi.cron
# reload generic admin view ?redirect=http...
# inspecting django db connection for asserting db readines?
# wake up django mailer on send_mail
# project settings modified copy of django's default project settings

2
orchestra/__init__.py
View File

@ -1,3 +1,5 @@
default_app_config = 'orchestra.apps.OrchestraConfig'
VERSION = (0, 0, 1, 'alpha', 1)

6
orchestra/apps.py Normal file
View File

@ -0,0 +1,6 @@
from django.apps import AppConfig
class OrchestraConfig(AppConfig):
name = 'orchestra'
verbose_name = 'Orchestra'

8
orchestra/bin/orchestra-admin
View File

@ -122,6 +122,11 @@ function install_requirements () {
check_root || true
ORCHESTRA_PATH=$(get_orchestra_dir) || true
# TODO reduce this list to 0
# include /usr/sbin/named-checkzone
# wkhtmltopdf -> reportlab
# remove rabbit, postgres
# uwsgi –py-autoreload for devel
APT="python3 \
python3-pip \
python3-psycopg2 \
@ -137,6 +142,7 @@ function install_requirements () {
ca-certificates \
gettext"
# TODO remove celery deps, django 1.8.1, glic3rinu fork, celery email
PIP="django==1.8 \
django-celery-email==1.0.4 \
https://github.com/glic3rinu/django-fluent-dashboard/archive/master.zip \
@ -202,12 +208,14 @@ function install_requirements () {
run pip3 install $PIP
# TODO remove
# Some versions of rabbitmq-server will not start automatically by default unless ...
sed -i "s/# Default-Start:.*/# Default-Start: 2 3 4 5/" /etc/init.d/rabbitmq-server
sed -i "s/# Default-Stop:.*/# Default-Stop: 0 1 6/" /etc/init.d/rabbitmq-server
run update-rc.d rabbitmq-server defaults
# Patch passlib
# TODO discover locaion by importing it
IMPORT="from django.contrib.auth.hashers import mask_hash, _"
COLLECTIONS="from collections import OrderedDict"
ls /usr/local/lib/python*/dist-packages/passlib/ext/django/utils.py \

6
orchestra/contrib/domains/settings.py
View File

@ -89,7 +89,7 @@ DOMAINS_DEFAULT_MX = Setting('DOMAINS_DEFAULT_MX',
'10 mail.{}.'.format(ORCHESTRA_BASE_DOMAIN),
'10 mail2.{}.'.format(ORCHESTRA_BASE_DOMAIN),
),
validators=[lambda mxs: map(validate_mx_record, mxs)],
validators=[lambda mxs: list(map(validate_mx_record, mxs))],
help_text="Uses <tt>ORCHESTRA_BASE_DOMAIN</tt> by default."
)
@ -99,7 +99,7 @@ DOMAINS_DEFAULT_NS = Setting('DOMAINS_DEFAULT_NS',
'ns1.{}.'.format(ORCHESTRA_BASE_DOMAIN),
'ns2.{}.'.format(ORCHESTRA_BASE_DOMAIN),
),
validators=[lambda nss: map(validate_domain_name, nss)],
validators=[lambda nss: list(map(validate_domain_name, nss))],
help_text="Uses <tt>ORCHESTRA_BASE_DOMAIN</tt> by default."
)
@ -118,6 +118,6 @@ DOMAINS_FORBIDDEN = Setting('DOMAINS_FORBIDDEN',
DOMAINS_MASTERS = Setting('DOMAINS_MASTERS',
(),
validators=[lambda masters: map(validate_ip_address, masters)],
validators=[lambda masters: list(map(validate_ip_address, masters))],
help_text="Additional master server ip addresses other than autodiscovered by router.get_servers()."
)

3
orchestra/contrib/orchestration/tasks.py
View File

@ -1,9 +1,10 @@
from datetime import timedelta
from celery.task.schedules import crontab
from celery.decorators import periodic_task
from django.utils import timezone
from orchestra.contrib.tasks import periodic_task
from .models import BackendLog

3
orchestra/contrib/resources/models.py
View File

@ -5,7 +5,6 @@ from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from djcelery.models import CrontabSchedule
from orchestra.core import validators
from orchestra.models import queryset, fields
@ -64,7 +63,7 @@ class Resource(models.Model):
"be prorcessed to match with unit. e.g. <tt>10**9</tt>"))
disable_trigger = models.BooleanField(_("disable trigger"), default=False,
help_text=_("Disables monitors exeeded and recovery triggers"))
crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"),
crontab = models.ForeignKey('djcelery.CrontabSchedule', verbose_name=_("crontab"),
null=True, blank=True,
help_text=_("Crontab for periodic execution. "
"Leave it empty to disable periodic monitoring"))

5
orchestra/contrib/resources/tasks.py
View File

@ -1,12 +1,11 @@
from celery import shared_task
from orchestra.contrib.orchestration import Operation
from orchestra.contrib.tasks import task
from orchestra.models.utils import get_model_field_path
from .backends import ServiceMonitor
@shared_task(name='resources.Monitor')
@task(name='resources.Monitor')
def monitor(resource_id, ids=None, async=True):
from .models import ResourceData, Resource

11
orchestra/contrib/settings/admin.py
View File

@ -86,6 +86,17 @@ class SettingView(generic.edit.FormView):
messages.success(self.request, _("No changes have been detected."))
return super(SettingView, self).form_valid(form)
from orchestra.contrib.tasks import task
import time, sys
@task(name='rata')
def counter(num, log):
for i in range(1, num):
with open(log, 'a') as handler:
handler.write(str(i))
# sys.stderr.write('hola\n')
time.sleep(1)
#counter.apply_async(10, '/tmp/kakas')
class SettingFileView(generic.TemplateView):
template_name = 'admin/settings/view.html'

103
orchestra/contrib/tasks/__init__.py Normal file
View File

@ -0,0 +1,103 @@
import traceback
from functools import partial, wraps, update_wrapper
from multiprocessing import Process
from uuid import uuid4
from threading import Thread
from celery import shared_task as celery_shared_task
from celery import states
from celery.decorators import periodic_task as celery_periodic_task
from django.utils import timezone
from orchestra.utils.db import close_connection
from orchestra.utils.python import AttrDict, OrderedSet
def get_id():
return str(uuid4())
def get_name(fn):
return '.'.join((fn.__module__, fn.__name__))
def keep_state(fn):
""" logs task on djcelery's TaskState model """
@wraps(fn)
def wrapper(task_id, name, *args, **kwargs):
from djcelery.models import TaskState
now = timezone.now()
state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
kwargs=str(kwargs), tstamp=now)
try:
result = fn(*args, **kwargs)
except Exception as exc:
state.state = states.FAILURE
state.traceback = traceback.format_exc()
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return
# TODO send email
else:
state.state = states.SUCCESS
state.result = str(result)
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return result
return wrapper
def apply_async(fn, name=None, method='thread'):
""" replaces celery apply_async """
def inner(fn, name, method, *args, **kwargs):
task_id = get_id()
args = (task_id, name) + args
thread = Process(target=fn, args=args, kwargs=kwargs)
thread.start()
# Celery API compat
thread.request = AttrDict(id=task_id)
return thread
if name is None:
name = get_name(fn)
if method == 'thread':
method = Thread
elif method == 'process':
method = Process
else:
raise NotImplementedError("Support for %s concurrency method is not supported." % method)
fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method)
return fn
def apply_async_override(fn, name):
if fn is None:
def decorator(fn):
return update_wrapper(apply_async(fn), fn)
return decorator
return update_wrapper(apply_async(fn, name), fn)
def task(fn=None, **kwargs):
from . import settings
# register task
if fn is None:
fn = celery_shared_task(**kwargs)
else:
fn = celery_shared_task(fn)
if settings.TASKS_BACKEND in ('thread', 'process'):
name = kwargs.pop('name', None)
apply_async_override(fn, name)
return fn
def periodic_task(fn=None, **kwargs):
from . import settings
# register task
if fn is None:
fn = celery_periodic_task(**kwargs)
else:
fn = celery_periodic_task(fn)
if settings.TASKS_BACKEND in ('thread', 'process'):
name = kwargs.pop('name', None)
apply_async_override(fn, name)
return fn

9
orchestra/contrib/tasks/admin.py Normal file
View File

@ -0,0 +1,9 @@
from django.utils.translation import ugettext_lazy as _
from djcelery.admin import PeriodicTaskAdmin
from orchestra.admin.utils import admin_date
display_last_run_at = admin_date('last_run_at', short_description=_("Last run"))
PeriodicTaskAdmin.list_display = ('__unicode__', display_last_run_at, 'total_run_count', 'enabled')

43
orchestra/contrib/tasks/beat.py Normal file
View File

@ -0,0 +1,43 @@
import json
from celery import current_app
from celery.schedules import crontab_parser as CrontabParser
from django.utils import timezone
from djcelery.models import PeriodicTask
from . import apply_async
def is_due(task, time=None):
if time is None:
time = timezone.now()
crontab = task.crontab
parts = map(int, time.strftime("%M %H %w %d %m").split())
n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = parts
return bool(
n_minute in CrontabParser(60).parse(crontab.minute) and
n_hour in CrontabParser(24).parse(crontab.hour) and
n_day_of_week in CrontabParser(7).parse(crontab.day_of_week) and
n_day_of_month in CrontabParser(31, 1).parse(crontab.day_of_month) and
n_month_of_year in CrontabParser(12, 1).parse(crontab.month_of_year)
)
def run_task(task, thread=True, process=False, async=False):
args = json.loads(task.args)
kwargs = json.loads(task.kwargs)
task_fn = current_app.tasks.get(task.task)
if async:
method = 'process' if process else 'thread'
return apply_async(task_fn, method=method).apply_async(*args, **kwargs)
return task_fn(*args, **kwargs)
def run():
now = timezone.now()
procs = []
for task in PeriodicTask.objects.enabled().select_related('crontab'):
if is_due(task, now):
proc = run_task(task, process=True, async=True)
procs.append(proc)
[proc.join() for proc in procs]

83
orchestra/contrib/tasks/bin/orchestra-beat Executable file
View File

@ -0,0 +1,83 @@
#!/usr/bin/env python3
# High performance alternative to beat management command
#
# USAGE: beat /path/to/project/manage.py
import json
import os
import re
import sys
from datetime import datetime
from orchestra.utils import db
from orchestra.utils.python import import_class
from orchestra.utils.sys import run, join
from celery.schedules import crontab_parser as CrontabParser
def get_settings_file(manage):
with open(manage, 'r') as handler:
regex = re.compile(r'"DJANGO_SETTINGS_MODULE"\s*,\s*"([^"]+)"')
for line in handler.readlines():
match = regex.search(line)
if match:
settings_module = match.groups()[0]
settings_file = os.path.join(*settings_module.split('.')) + '.py'
settings_file = os.path.join(os.path.dirname(manage), settings_file)
return settings_file
raise ValueError("settings module not found in %s" % manage)
def get_tasks(manage):
settings_file = get_settings_file(manage)
settings = db.get_settings(settings_file)
try:
conn = db.get_connection(settings)
except:
sys.stdout.write("ERROR")
sys.stderr.write("I am unable to connect to the database\n")
sys.exit(1)
script, settings_file = sys.argv[:2]
query = (
"SELECT c.minute, c.hour, c.day_of_week, c.day_of_month, c.month_of_year, p.id "
"FROM djcelery_periodictask as p, djcelery_crontabschedule as c "
"WHERE p.crontab_id = c.id AND p.enabled = True"
)
tasks = db.run_query(conn, query)
conn.close()
return tasks
def is_due(now, minute, hour, day_of_week, day_of_month, month_of_year):
n_minute, n_hour, n_day_of_week, n_day_of_month, n_month_of_year = now
return (
n_minute in CrontabParser(60).parse(minute) and
n_hour in CrontabParser(24).parse(hour) and
n_day_of_week in CrontabParser(7).parse(day_of_week) and
n_day_of_month in CrontabParser(31, 1).parse(day_of_month) and
n_month_of_year in CrontabParser(12, 1).parse(month_of_year)
)
if __name__ == "__main__":
manage = sys.argv[1]
now = datetime.utcnow()
now = tuple(map(int, now.strftime("%M %H %w %d %m").split()))
procs = []
for minute, hour, day_of_week, day_of_month, month_of_year, task_id in get_tasks(manage):
if is_due(now, minute, hour, day_of_week, day_of_month, month_of_year):
command = 'python3 -W ignore::DeprecationWarning {manage} runtask {task_id}'.format(
manage=manage, task_id=task_id)
proc = run(command, async=True)
procs.append(proc)
code = 0
for proc in procs:
result = join(proc)
sys.stdout.write(result.stdout.decode('utf8'))
sys.stderr.write(result.stderr.decode('utf8'))
if result.return_code != 0:
code = result.return_code
sys.exit(code)

10
orchestra/contrib/tasks/management/commands/beat.py Normal file
View File

@ -0,0 +1,10 @@
from django.core.management.base import BaseCommand, CommandError
from ... import beat
class Command(BaseCommand):
help = 'Runs periodic tasks.'
def handle(self, *args, **options):
beat.run()

32
orchestra/contrib/tasks/management/commands/runfunction.py Normal file
View File

@ -0,0 +1,32 @@
from django.core.management.base import BaseCommand, CommandError
from orchestra.utils.python import import_class
from ... import keep_state, get_id, get_name
class Command(BaseCommand):
help = 'Runs Orchestra method.'
def add_arguments(self, parser):
parser.add_argument('method',
help='Python path to the method to execute.')
parser.add_argument('args', nargs='*',
help='Additional arguments passed to the method.')
def handle(self, *args, **options):
method = import_class(options['method'])
kwargs = {}
arguments = []
for arg in args:
if '=' in args:
name, value = arg.split('=')
if value.isdigit():
value = int(value)
kwargs[name] = value
else:
if arg.isdigit():
arg = int(arg)
arguments.append(arg)
args = arguments
keep_state(method)(get_id(), get_name(method), *args, **kwargs)

49
orchestra/contrib/tasks/management/commands/runtask.py Normal file
View File

@ -0,0 +1,49 @@
import json
from celery import current_app
from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from djcelery.models import PeriodicTask
from ... import keep_state, get_id, get_name
class Command(BaseCommand):
help = 'Runs Orchestra method.'
def add_arguments(self, parser):
parser.add_argument('task',
help='Periodic task ID or task name.')
parser.add_argument('args', nargs='*',
help='Additional arguments passed to the task, when task name is used.')
def handle(self, *args, **options):
task = options.get('task')
if task.isdigit():
# periodic task
ptask = PeriodicTask.objects.get(pk=int(task))
task = current_app.tasks[ptask.task]
args = json.loads(ptask.args)
kwargs = json.loads(ptask.kwargs)
ptask.last_run_at = timezone.now()
ptask.total_run_count += 1
ptask.save()
else:
# task name
task = current_app.tasks[task]
kwargs = {}
arguments = []
for arg in args:
if '=' in args:
name, value = arg.split('=')
if value.isdigit():
value = int(value)
kwargs[name] = value
else:
if arg.isdigit():
arg = int(arg)
arguments.append(arg)
args = arguments
# Run task synchronously, but logging TaskState
keep_state(task)(get_id(), get_name(task), *args, **kwargs)

2
orchestra/contrib/tasks/management/commands/syncperiodictasks.py Normal file
View File

@ -0,0 +1,2 @@
# create crontab entries for defines periodic tasks

61
orchestra/contrib/tasks/parser.py Normal file
View File

@ -0,0 +1,61 @@
import os
# Rename module to handler.py
class CronHandler(object):
def __init__(self, filename):
self.content = None
self.filename = filename
def read(self):
comments = []
self.content = []
with open(self.filename, 'r') as handler:
for line in handler.readlines():
line = line.strip()
if line.startswith('#'):
comments.append(line)
else:
schedule = line.split()[:5]
command = ' '.join(line.split()[5:]).strip()
self.content.append((schedule, command, comments))
comments = []
def save(self, backup=True):
if self.content is None:
raise Exception("First read() the cron file!")
if backup:
os.rename(self.filename, self.filename + '.backup')
with open(self.filename, 'w') as handler:
handler.write('\n'.join(self.content))
handler.truncate()
self.reload()
def reload(self):
pass
# TODO
def remove(self, command):
if self.content is None:
raise Exception("First read() the cron file!")
new_content = []
for c_schedule, c_command, c_comments in self.content:
if command != c_command:
new_content.append((c_schedule, c_command, c_comments))
self.content = new_content
def add_or_update(self, schedule, command, comments=None):
""" if content contains an equal command, its schedule is updated """
if self.content is None:
raise Exception("First read() the cron file!")
new_content = []
replaced = False
for c_schedule, c_command, c_comments in self.content:
if command == c_command:
replaced = True
new_content.append((schedule, command, comments or c_comments))
else:
new_content.append((c_schedule, c_command, c_comments))
if not replaced:
new_content.append((schedule, command, comments or []))
self.content = new_content

119
orchestra/contrib/tasks/schedules.py Normal file
View File

@ -0,0 +1,119 @@
#import re
#class CronTab(object):
# pass
#class ParseException(Exception):
# """Raised by crontab_parser when the input can't be parsed."""
## https://github.com/celery/celery/blob/master/celery/schedules.py
#class CrontabParser(object):
# """Parser for crontab expressions. Any expression of the form 'groups'
# (see BNF grammar below) is accepted and expanded to a set of numbers.
# These numbers represent the units of time that the crontab needs to
# run on::
# digit :: '0'..'9'
# dow :: 'a'..'z'
# number :: digit+ | dow+
# steps :: number
# range :: number ( '-' number ) ?
# numspec :: '*' | range
# expr :: numspec ( '/' steps ) ?
# groups :: expr ( ',' expr ) *
# The parser is a general purpose one, useful for parsing hours, minutes and
# day_of_week expressions. Example usage::
# >>> minutes = crontab_parser(60).parse('*/15')
# [0, 15, 30, 45]
# >>> hours = crontab_parser(24).parse('*/4')
# [0, 4, 8, 12, 16, 20]
# >>> day_of_week = crontab_parser(7).parse('*')
# [0, 1, 2, 3, 4, 5, 6]
# It can also parse day_of_month and month_of_year expressions if initialized
# with an minimum of 1. Example usage::
# >>> days_of_month = crontab_parser(31, 1).parse('*/3')
# [1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31]
# >>> months_of_year = crontab_parser(12, 1).parse('*/2')
# [1, 3, 5, 7, 9, 11]
# >>> months_of_year = crontab_parser(12, 1).parse('2-12/2')
# [2, 4, 6, 8, 10, 12]
# The maximum possible expanded value returned is found by the formula::
# max_ + min_ - 1
# """
# ParseException = ParseException
# _range = r'(\w+?)-(\w+)'
# _steps = r'/(\w+)?'
# _star = r'\*'
# def __init__(self, max_=60, min_=0):
# self.max_ = max_
# self.min_ = min_
# self.pats = (
# (re.compile(self._range + self._steps), self._range_steps),
# (re.compile(self._range), self._expand_range),
# (re.compile(self._star + self._steps), self._star_steps),
# (re.compile('^' + self._star + '$'), self._expand_star),
# )
# def parse(self, spec):
# acc = set()
# for part in spec.split(','):
# if not part:
# raise self.ParseException('empty part')
# acc |= set(self._parse_part(part))
# return acc
# def _parse_part(self, part):
# for regex, handler in self.pats:
# m = regex.match(part)
# if m:
# return handler(m.groups())
# return self._expand_range((part, ))
# def _expand_range(self, toks):
# fr = self._expand_number(toks[0])
# if len(toks) > 1:
# to = self._expand_number(toks[1])
# if to < fr: # Wrap around max_ if necessary
# return (list(range(fr, self.min_ + self.max_)) +
# list(range(self.min_, to + 1)))
# return list(range(fr, to + 1))
# return [fr]
# def _range_steps(self, toks):
# if len(toks) != 3 or not toks[2]:
# raise self.ParseException('empty filter')
# return self._expand_range(toks[:2])[::int(toks[2])]
# def _star_steps(self, toks):
# if not toks or not toks[0]:
# raise self.ParseException('empty filter')
# return self._expand_star()[::int(toks[0])]
# def _expand_star(self, *args):
# return list(range(self.min_, self.max_ + self.min_))
# def _expand_number(self, s):
# if isinstance(s, str) and s[0] == '-':
# raise self.ParseException('negative numbers not supported')
# try:
# i = int(s)
# except ValueError:
# try:
# i = weekday(s)
# except KeyError:
# raise ValueError('Invalid weekday literal {0!r}.'.format(s))
# max_val = self.min_ + self.max_ - 1
# if i > max_val:
# raise ValueError(
# 'Invalid end range: {0} > {1}.'.format(i, max_val))
# if i < self.min_:
# raise ValueError(
# 'Invalid beginning range: {0} < {1}.'.format(i, self.min_))
# return i

11
orchestra/contrib/tasks/settings.py Normal file
View File

@ -0,0 +1,11 @@
from orchestra.settings import Setting
TASKS_BACKEND = Setting('TASKS_BACKEND',
'thread',
choices=(
('thread', "threading.Thread (no queue)"),
('process', "multiprocess.Process (no queue)"),
('celery', "Celery (with queue)"),
)
)

12
orchestra/contrib/tasks/utils.py Normal file
View File

@ -0,0 +1,12 @@
import threading
from orchestra.utils.db import close_connection
# TODO import as_task
def run(method, *args, **kwargs):
async = kwargs.pop('async', True)
thread = threading.Thread(target=close_connection(method), args=args, kwargs=kwargs)
thread = Process(target=close_connection(counter))
thread.start()

2
orchestra/forms/widgets.py
View File

@ -29,7 +29,7 @@ class SpanWidget(forms.Widget):
return mark_safe('<img src="%s" alt="%s">' % (icon, str(display)))
tag = self.tag[:-1]
endtag = '/'.join((self.tag[0], self.tag[1:]))
return mark_safe('%s%s >%s%s' % (tag, forms.util.flatatt(final_attrs), display, endtag))
return mark_safe('%s%s >%s%s' % (tag, forms.utils.flatatt(final_attrs), display, endtag))
def value_from_datadict(self, data, files, name):
return self.original

6
orchestra/models/utils.py
View File

@ -1,18 +1,18 @@
from django.conf import settings
from django.db.models import loading
from django.apps import apps
import importlib
def get_model(label, import_module=True):
app_label, model_name = label.split('.')
model = loading.get_model(app_label, model_name)
model = apps.get_model(app_label, model_name)
if model is None:
# Sometimes the models module is not yet imported
for app in settings.INSTALLED_APPS:
if app.endswith(app_label):
app_label = app
importlib.import_module('%s.%s' % (app_label, 'admin'))
return loading.get_model(*label.split('.'))
return apps.get_model(*label.split('.'))
return model

24
orchestra/settings.py
View File

@ -3,6 +3,7 @@ import sys
from collections import OrderedDict
from django.conf import settings
from django.core.checks import register, Error
from django.core.exceptions import ValidationError, AppRegistryNotReady
from django.core.validators import validate_email
from django.db.models import get_model
@ -42,16 +43,6 @@ class Setting(object):
for name, value in kwargs.items():
setattr(self, name, value)
self.value = self.get_value(self.name, self.default)
try:
self.validate_value(self.value)
except ValidationError as exc:
# Init time warning
sys.stderr.write("Error validating setting %s with value %s\n" % (self.name, self.value))
sys.stderr.write(format_exception(exc))
raise exc
except AppRegistryNotReady:
# lazy bastards
pass
self.settings[name] = self
@classmethod
@ -117,6 +108,19 @@ class Setting(object):
return getattr(cls.conf_settings, name, default)
@register()
def check_settings(app_configs, **kwargs):
""" perfroms all the validation """
messages = []
for name, setting in Setting.settings.items():
try:
setting.validate_value(setting.value)
except ValidationError as exc:
msg = "Error validating setting with value %s: %s" % (setting.value, str(exc))
messages.append(Error(msg, obj=name, id='settings.E001'))
return messages
ORCHESTRA_BASE_DOMAIN = Setting('ORCHESTRA_BASE_DOMAIN',
'orchestra.lan',
help_text=("Base domain name used for other settings.<br>"

49
orchestra/utils/db.py Normal file
View File

@ -0,0 +1,49 @@
import ast
from django import db
def close_connection(execute):
""" Threads have their own connection pool, closing it when finishing """
def wrapper(*args, **kwargs):
try:
log = execute(*args, **kwargs)
except Exception as e:
pass
else:
wrapper.log = log
finally:
db.connection.close()
return wrapper
def get_settings(settings_file):
""" get db settings from settings.py file without importing """
settings = {}
with open(settings_file, 'r') as handler:
body = ast.parse(handler.read()).body
for var in body:
targets = getattr(var, 'targets', None)
if targets and targets[0].id == 'DATABASES':
keys = var.value.values[0].keys
values = var.value.values[0].values
for key, value in zip(keys, values):
if key.s == 'ENGINE':
if not 'postgresql' in value.s:
raise ValueError("%s engine not supported." % value)
settings[key.s] = getattr(value, 's', None)
return settings
def get_connection(settings):
import psycopg2
conn = psycopg2.connect("dbname='{NAME}' user='{USER}' host='{HOST}' password='{PASSWORD}'".format(**settings))
return conn
def run_query(conn, query):
cur = conn.cursor()
cur.execute(query)
result = cur.fetchall()
cur.close()
return result

22
orchestra/utils/sys.py
View File

@ -45,7 +45,7 @@ def read_async(fd):
return ''
def runiterator(command, display=False, error_codes=[0], silent=False, stdin=b''):
def runiterator(command, display=False, stdin=b''):
""" Subprocess wrapper for running commands concurrently """
if display:
sys.stderr.write("\n\033[1m $ %s\033[0m\n" % command)
@ -83,6 +83,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin=b''
state = _Attribute(stdout)
state.stderr = stderr
state.return_code = p.poll()
state.command = command
yield state
if state.return_code != None:
@ -90,13 +91,8 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin=b''
p.stderr.close()
raise StopIteration
def run(command, display=False, error_codes=[0], silent=False, stdin=b'', async=False):
iterator = runiterator(command, display, error_codes, silent, stdin)
next(iterator)
if async:
return iterator
def join(iterator, display=False, silent=False, error_codes=[0]):
""" joins the iterator process """
stdout = b''
stderr = b''
for state in iterator:
@ -114,7 +110,7 @@ def run(command, display=False, error_codes=[0], silent=False, stdin=b'', async=
if return_code not in error_codes:
out.failed = True
msg = "\nrun() encountered an error (return code %s) while executing '%s'\n"
msg = msg % (return_code, command)
msg = msg % (return_code, state.command)
if display:
sys.stderr.write("\n\033[1;31mCommandError: %s %s\033[m\n" % (msg, err))
if not silent:
@ -124,6 +120,14 @@ def run(command, display=False, error_codes=[0], silent=False, stdin=b'', async=
return out
def run(command, display=False, error_codes=[0], silent=False, stdin=b'', async=False):
iterator = runiterator(command, display, stdin)
next(iterator)
if async:
return iterator
return join(iterator, display=display, silent=silent, error_codes=error_codes)
def sshrun(addr, command, *args, **kwargs):
command = command.replace("'", """'"'"'""")
cmd = "ssh -o stricthostkeychecking=no -C root@%s '%s'" % (addr, command)

2
requirements.txt
View File

@ -1,6 +1,6 @@
cracklib
psycopg2
django==1.8
django==1.8.1
django-celery-email==1.0.4
https://github.com/glic3rinu/django-fluent-dashboard/archive/master.zip
https://bitbucket.org/izi/django-admin-tools/get/a0abfffd76a0.zip

5
setup.py
View File

@ -27,7 +27,10 @@ setup(
"The goal of this project is to provide the tools for easily build a fully "
"featured control panel that fits any service architecture."),
include_package_data = True,
scripts=['orchestra/bin/orchestra-admin'],
scripts=[
'orchestra/bin/orchestra-admin',
'orchestra/contrib/tasks/bin/orchestra-beat',
],
packages = packages,
classifiers = [
'Development Status :: 1 - Alpha',