pangea/django-orchestra
7
0
Fork 0
Code Issues 1 Pull requests 2 Packages Projects Releases Wiki Activity

Added create initial superuser prompt on accounts post migrate

Browse source
This commit is contained in:
Marc Aymerich 2015-05-01 18:05:34 +00:00
parent 94941a633f
commit f5e80d680c
20 changed files with 169 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
TODO.md
View file

@ -300,30 +300,31 @@ https://code.djangoproject.com/ticket/24576
# accounts.migrations link to last auth migration instead of first
# DNS allow transfer other NS servers instead of masters and slaves!
Replace celery by a custom solution?
# TODO create periodic task like settings, but parsing cronfiles!
# TODO create decorator wrapper that abstract the task away from the backen (cron/celery)
# TODO crontab model localhost/autoadded attribute
* No more jumbo dependencies and wierd bugs
1) Periodic Monitoring:
* runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system)
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument('method', help='')
parser.add_argument('args', nargs='*', help='')
def handle(self, *args, **options):
method = import_class(options['method'])
kwargs = {}
arguments = []
for arg in args:
if '=' in args:
name, value = arg.split('=')
kwargs[name] = value
else:
arguments.append(arg)
args = arguments
method(*args, **kwargs)
2) Single time shot:
sys.run("python3 manage.py runtas 'task' args")
3) Emails:
Custom backend that distinguishes between priority and bulk mail
priority: custom Thread backend
bulk: wrapper arround django-mailer to avoid loading django system
*priority: custom Thread backend
*bulk: wrapper arround django-mailer to avoid loading django system
# uwsgi enable threads
# Create superuser on migrate
# register signals in app ready()
def ready(self):
if self.has_attr('ready_run'): return
self.ready_run = True
# database_ready(): connect to the database or inspect django connection
# beat.sh
# do settings validation on orchestra.apps.ready(), not during startime

8
orchestra/admin/utils.py
View file

@ -16,6 +16,7 @@ from orchestra.models.utils import get_field_value
from orchestra.utils import humanize
from .decorators import admin_field
from .html import monospace_format
def get_modeladmin(model, import_module=True):
@ -153,3 +154,10 @@ def get_object_from_url(modeladmin, request):
return None
else:
return modeladmin.model.objects.get(pk=object_id)
def display_mono(field):
def display(self, log):
return monospace_format(escape(getattr(log, field)))
display.short_description = field
return display

2
orchestra/conf/base_settings.py
View file

@ -89,6 +89,7 @@ INSTALLED_APPS = (
'orchestra.contrib.miscellaneous',
'orchestra.contrib.bills',
'orchestra.contrib.payments',
'orchestra.contrib.tasks',
# Third-party apps
'django_extensions',
@ -103,6 +104,7 @@ INSTALLED_APPS = (
'rest_framework.authtoken',
'passlib.ext.django',
'django_countries',
'django_mailer',
# Django.contrib
'django.contrib.auth',

1
orchestra/contrib/accounts/__init__.py
View file

@ -0,0 +1 @@
default_app_config = 'orchestra.contrib.accounts.apps.AccountConfig'

14
orchestra/contrib/accounts/apps.py Normal file
View file

@ -0,0 +1,14 @@
from django.apps import AppConfig
from django.db.models.signals import post_migrate
from django.utils.translation import ugettext_lazy as _
from .management import create_initial_superuser
class AccountConfig(AppConfig):
name = 'orchestra.contrib.accounts'
verbose_name = _("Accounts")
def ready(self):
post_migrate.connect(create_initial_superuser,
dispatch_uid="orchestra.contrib.accounts.management.createsuperuser")

19
orchestra/contrib/accounts/management/__init__.py Normal file
View file

@ -0,0 +1,19 @@
import sys
import textwrap
from django.contrib.auth import get_user_model
from django.core.management import execute_from_command_line
def create_initial_superuser(**kwargs):
if '--noinput' not in sys.argv and '--fake' not in sys.argv and '--fake-initial' not in sys.argv and 'accounts' in sys.argv:
model = get_user_model()
if not model.objects.filter(is_superuser=True).exists():
sys.stdout.write(textwrap.dedent("""
It appears that you just installed Accounts application.
You can now create a superuser:
""")
)
manager = sys.argv[0]
execute_from_command_line(argv=[manager, 'createsuperuser'])

39
orchestra/contrib/domains/backends.py
View file

@ -1,4 +1,5 @@
import re
import socket
import textwrap
from django.utils.translation import ugettext_lazy as _
@ -8,12 +9,13 @@ from orchestra.contrib.orchestration import Operation
from orchestra.utils.python import OrderedSet
from . import settings
from .models import Record, Domain
class Bind9MasterDomainBackend(ServiceController):
"""
Bind9 zone and config generation.
It auto-discovers slave Bind9 servers based on your routing configuration or you can use DOMAINS_SLAVES to explicitly configure the slaves.
It auto-discovers slave Bind9 servers based on your routing configuration and NS servers.
"""
CONF_PATH = settings.DOMAINS_MASTERS_PATH
@ -25,7 +27,7 @@ class Bind9MasterDomainBackend(ServiceController):
)
ignore_fields = ['serial']
doc_settings = (settings,
('DOMAINS_SLAVES', 'DOMAINS_MASTERS_PATH')
('DOMAINS_MASTERS_PATH',)
)
@classmethod
@ -100,10 +102,32 @@ class Bind9MasterDomainBackend(ServiceController):
servers.append(server.get_ip())
return servers
def get_masters(self, domain):
ips = list(settings.DOMAINS_MASTERS)
if not ips:
ips += self.get_servers(domain, Bind9MasterDomainBackend)
return OrderedSet(sorted(ips))
def get_slaves(self, domain):
ips = list(settings.DOMAINS_SLAVES)
ips += self.get_servers(domain, Bind9SlaveDomainBackend)
return OrderedSet(ips)
ips = []
masters = self.get_masters(domain)
for ns in domain.records.filter(type=Record.NS):
hostname = ns.value.rstrip('.')
# First try with a DNS query, a more reliable source
try:
addr = socket.gethostbyname(hostname)
except socket.gaierror:
# check if domain is declared
try:
domain = Domain.objects.get(name=ns)
except Domain.DoesNotExist:
continue
else:
a_record = domain.records.filter(name=Record.A) or [settings.DOMAINS_DEFAULT_NS]
addr = a_record[0]
if addr not in masters:
ips.append(addr)
return OrderedSet(sorted(ips))
def get_context(self, domain):
slaves = self.get_slaves(domain)
@ -154,11 +178,6 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend):
""" ideally slave should be restarted after master """
self.append('if [[ $UPDATED == 1 ]]; then { sleep 1 && service bind9 reload; } & fi')
def get_masters(self, domain):
ips = list(settings.DOMAINS_MASTERS)
ips += self.get_servers(domain, Bind9MasterDomainBackend)
return OrderedSet(ips)
def get_context(self, domain):
context = {
'name': domain.name,

7
orchestra/contrib/domains/settings.py
View file

@ -121,10 +121,3 @@ DOMAINS_MASTERS = Setting('DOMAINS_MASTERS',
validators=[lambda masters: map(validate_ip_address, masters)],
help_text="Additional master server ip addresses other than autodiscovered by router.get_servers()."
)
DOMAINS_SLAVES = Setting('DOMAINS_SLAVES',
(),
validators=[lambda slaves: map(validate_ip_address, slaves)],
help_text="Additional slave server ip addresses other than autodiscovered by router.get_servers()."
)

16
orchestra/contrib/orchestration/admin.py
View file

@ -3,8 +3,7 @@ from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from orchestra.admin.html import monospace_format
from orchestra.admin.utils import admin_link, admin_date, admin_colored
from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono
from . import settings, helpers
from .backends import ServiceBackend
@ -109,13 +108,6 @@ class BackendOperationInline(admin.TabularInline):
return queryset.prefetch_related('instance')
def display_mono(field):
def display(self, log):
return monospace_format(escape(getattr(log, field)))
display.short_description = _(field)
return display
class BackendLogAdmin(admin.ModelAdmin):
list_display = (
'id', 'backend', 'server_link', 'display_state', 'exit_code',
@ -123,12 +115,12 @@ class BackendLogAdmin(admin.ModelAdmin):
)
list_display_links = ('id', 'backend')
list_filter = ('state', 'backend')
inlines = [BackendOperationInline]
fields = [
inlines = (BackendOperationInline,)
fields = (
'backend', 'server_link', 'state', 'mono_script', 'mono_stdout',
'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created',
'execution_time'
]
)
readonly_fields = fields
server_link = admin_link('server')

2
orchestra/contrib/orchestration/management/commands/orchestrate.py
View file

@ -1,5 +1,3 @@
import sys
from django.core.management.base import BaseCommand, CommandError
from django.db.models.loading import get_model

16
orchestra/contrib/orchestration/manager.py
View file

@ -3,9 +3,9 @@ import threading
import traceback
from collections import OrderedDict
from django import db
from django.core.mail import mail_admins
from orchestra.utils.db import close_connection
from orchestra.utils.python import import_class, OrderedSet
from . import settings, Operation
@ -42,20 +42,6 @@ def as_task(execute):
return wrapper
def close_connection(execute):
""" Threads have their own connection pool, closing it when finishing """
def wrapper(*args, **kwargs):
try:
log = execute(*args, **kwargs)
except Exception as e:
pass
else:
wrapper.log = log
finally:
db.connection.close()
return wrapper
def generate(operations):
scripts = OrderedDict()
cache = {}

8
orchestra/contrib/orchestration/models.py
View file

@ -68,13 +68,11 @@ class BackendLog(models.Model):
)
backend = models.CharField(_("backend"), max_length=256)
state = models.CharField(_("state"), max_length=16, choices=STATES,
default=RECEIVED)
server = models.ForeignKey(Server, verbose_name=_("server"),
related_name='execution_logs')
state = models.CharField(_("state"), max_length=16, choices=STATES, default=RECEIVED)
server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs')
script = models.TextField(_("script"))
stdout = models.TextField(_("stdout"))
stderr = models.TextField(_("stdin"))
stderr = models.TextField(_("stderr"))
traceback = models.TextField(_("traceback"))
exit_code = models.IntegerField(_("exit code"), null=True)
task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True,

1
orchestra/contrib/resources/actions.py
View file

@ -16,6 +16,7 @@ def run_monitor(modeladmin, request, queryset):
modeladmin.log_change(request, resource, _("Run monitors"))
if async:
num = len(queryset)
# TODO listfilter by uuid: task.request.id + ?task_id__in=ids
link = reverse('admin:djcelery_taskstate_changelist')
msg = ungettext(
_("One selected resource has been <a href='%s'>scheduled for monitoring</a>.") % link,

32
orchestra/contrib/resources/models.py
View file

@ -6,7 +6,7 @@ from django.db.models.loading import get_model
from django.utils import timezone
from django.utils.functional import cached_property
from django.utils.translation import ugettext_lazy as _
from djcelery.models import PeriodicTask, CrontabSchedule
from djcelery.models import CrontabSchedule
from orchestra.core import validators
from orchestra.models import queryset, fields
@ -14,7 +14,7 @@ from orchestra.models.utils import get_model_field_path
from orchestra.utils.paths import get_project_dir
from orchestra.utils.sys import run
from . import tasks
from . import tasks, settings
from .backends import ServiceMonitor
from .aggregations import Aggregation
from .validators import validate_scale
@ -129,6 +129,12 @@ class Resource(models.Model):
def delete(self, *args, **kwargs):
super(Resource, self).delete(*args, **kwargs)
name = 'monitor.%s' % str(self)
self.sync_periodic_task()
def sync_periodic_task(self):
name = 'monitor.%s' % str(self)
sync = import_class(settings.RESOURCES_TASK_BACKEND)
return sync(self, name)
def get_model_path(self, monitor):
""" returns a model path between self.content_type and monitor.model """
@ -136,28 +142,6 @@ class Resource(models.Model):
monitor_model = ServiceMonitor.get_backend(monitor).model_class()
return get_model_field_path(monitor_model, resource_model)
def sync_periodic_task(self):
name = 'monitor.%s' % str(self)
if self.pk and self.crontab:
try:
task = PeriodicTask.objects.get(name=name)
except PeriodicTask.DoesNotExist:
if self.is_active:
PeriodicTask.objects.create(
name=name,
task='resources.Monitor',
args=[self.pk],
crontab=self.crontab
)
else:
if task.crontab != self.crontab:
task.crontab = self.crontab
task.save(update_fields=['crontab'])
else:
PeriodicTask.objects.filter(
name=name,
).delete()
def get_scale(self):
return eval(self.scale)

6
orchestra/contrib/resources/settings.py Normal file
View file

@ -0,0 +1,6 @@
from orchestra.settings import Setting
RESOURCES_TASK_BACKEND = Setting('RESOURCES_TASK_BACKEND',
'orchestra.contrib.resources.utils.cron_sync'
)

1
orchestra/contrib/resources/tasks.py
View file

@ -30,6 +30,7 @@ def monitor(resource_id, ids=None, async=True):
op = Operation(backend, obj, Operation.MONITOR)
monitorings.append(op)
# TODO async=True only when running with celery
# monitor.request.id
logs += Operation.execute(monitorings, async=async)
kwargs = {'id__in': ids} if ids else {}

41
orchestra/contrib/resources/utils.py Normal file
View file

@ -0,0 +1,41 @@
from orchestra.contrib.crons.utils import apply_local
from . import settings
def celery_sync(resource, name):
from djcelery.models import PeriodicTask
if resource.pk and resource.crontab:
try:
task = PeriodicTask.objects.get(name=name)
except PeriodicTask.DoesNotExist:
if resource.is_active:
PeriodicTask.objects.create(
name=name,
task='resources.Monitor',
args=[resource.pk],
crontab=resource.crontab
)
else:
if task.crontab != resource.crontab:
task.crontab = resource.crontab
task.save(update_fields=['crontab'])
else:
PeriodicTask.objects.filter(
name=name,
).delete()
def cron_sync(resource, name):
if resource.pk and resource.crontab:
context = {
'manager': os.path.join(paths.get_project_dir(), 'manage.py'),
'id': resource.pk,
}
apply_local(resource.crontab,
'python3 %(manager)s runmethod orchestra.contrib.resources.tasks.monitor %(id)s',
'orchestra', # TODO
name
)
else:
apply_local(resource.crontab, '', 'orchestra', name, action='delete')

11
orchestra/contrib/settings/admin.py
View file

@ -91,6 +91,16 @@ class SettingFileView(generic.TemplateView):
template_name = 'admin/settings/view.html'
def get_context_data(self, **kwargs):
from orchestra.contrib.tasks import shared_task
import time
@shared_task(name='rata')
def counter(num, log):
for i in range(1, num):
with open(log, 'a') as handler:
handler.write(str(i))
time.sleep(1)
counter.apply_async(10, '/tmp/kakas')
context = super(SettingFileView, self).get_context_data(**kwargs)
settings_file = parser.get_settings_file()
with open(settings_file, 'r') as handler:
@ -106,4 +116,3 @@ class SettingFileView(generic.TemplateView):
admin.site.register_url(r'^settings/setting/view/$', SettingFileView.as_view(), 'settings_setting_view')
admin.site.register_url(r'^settings/setting/$', SettingView.as_view(), 'settings_setting_change')
OrchestraIndexDashboard.register_link('Administration', 'settings_setting_change', _("Settings"))

1
orchestra/contrib/settings/parser.py
View file

@ -58,6 +58,7 @@ def get_eval_context():
'_': _,
}
def serialize(obj, init=True):
if isinstance(obj, NotSupported):
return obj

3
orchestra/utils/options.py
View file

@ -50,8 +50,7 @@ def database_ready():
# Celerybeat has yet to stablish a connection at AppConf.ready()
'celerybeat' not in sys.argv and
# Allow to run python manage.py without a database
len(sys.argv) <= 1 and
'--help' not in sys.argv)
sys.argv != ['manage.py'] and '--help' not in sys.argv)
def dict_setting_to_choices(choices):