Added create initial superuser prompt on accounts post migrate

TODO.md

@@ -300,30 +300,31 @@ https://code.djangoproject.com/ticket/24576
 # accounts.migrations link to last auth migration instead of first
 
 
+# DNS allow transfer other NS servers instead of masters and slaves!
 
 Replace celery by a custom solution?
+    # TODO create periodic task like settings, but parsing cronfiles!
+    # TODO create decorator wrapper that abstract the task away from the backen (cron/celery)
+    # TODO crontab model localhost/autoadded attribute
     * No more jumbo dependencies and wierd bugs
     1) Periodic Monitoring:
         * runtask management command + crontab scheduling or high performance beat crontab (not loading bloated django system)
-        class Command(BaseCommand):
-            def add_arguments(self, parser):
-                parser.add_argument('method', help='')
-                parser.add_argument('args', nargs='*', help='')
-            def handle(self, *args, **options):
-                method = import_class(options['method'])
-                kwargs = {}
-                arguments = []
-                for arg in args:
-                    if '=' in args:
-                        name, value = arg.split('=')
-                        kwargs[name] = value
-                    else:
-                        arguments.append(arg)
-                args = arguments
-                method(*args, **kwargs)
     2) Single time shot:
         sys.run("python3 manage.py runtas 'task' args")
     3) Emails:
         Custom backend that distinguishes between priority and bulk mail
-            priority: custom Thread backend
+            *priority: custom Thread backend
-            bulk: wrapper arround django-mailer to avoid loading django system
+            *bulk: wrapper arround django-mailer to avoid loading django system
+
+
+# uwsgi enable threads
+# Create superuser on migrate
+# register signals in app ready()
+def ready(self):
+    if self.has_attr('ready_run'): return
+    self.ready_run = True
+
+# database_ready(): connect to the database or inspect django connection
+# beat.sh
+
+# do settings validation on orchestra.apps.ready(), not during startime

orchestra/admin/utils.py

@@ -16,6 +16,7 @@ from orchestra.models.utils import get_field_value
 from orchestra.utils import humanize
 
 from .decorators import admin_field
+from .html import monospace_format
 
 
 def get_modeladmin(model, import_module=True):
@@ -153,3 +154,10 @@ def get_object_from_url(modeladmin, request):
         return None
     else:
         return modeladmin.model.objects.get(pk=object_id)
+
+
+def display_mono(field):
+    def display(self, log):
+        return monospace_format(escape(getattr(log, field)))
+    display.short_description = field
+    return display

orchestra/conf/base_settings.py

@@ -89,6 +89,7 @@ INSTALLED_APPS = (
     'orchestra.contrib.miscellaneous',
     'orchestra.contrib.bills',
     'orchestra.contrib.payments',
+    'orchestra.contrib.tasks',
     
     # Third-party apps
     'django_extensions',
@@ -103,6 +104,7 @@ INSTALLED_APPS = (
     'rest_framework.authtoken',
     'passlib.ext.django',
     'django_countries',
+    'django_mailer',
     
     # Django.contrib
     'django.contrib.auth',

orchestra/contrib/accounts/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'orchestra.contrib.accounts.apps.AccountConfig'

orchestra/contrib/accounts/apps.py

@@ -0,0 +1,14 @@
+from django.apps import AppConfig
+from django.db.models.signals import post_migrate
+from django.utils.translation import ugettext_lazy as _
+
+from .management import create_initial_superuser
+
+
+class AccountConfig(AppConfig):
+    name = 'orchestra.contrib.accounts'
+    verbose_name = _("Accounts")
+    
+    def ready(self):
+        post_migrate.connect(create_initial_superuser,
+            dispatch_uid="orchestra.contrib.accounts.management.createsuperuser")

orchestra/contrib/accounts/management/__init__.py

@@ -0,0 +1,19 @@
+import sys
+import textwrap
+
+from django.contrib.auth import get_user_model
+from django.core.management import execute_from_command_line
+
+
+def create_initial_superuser(**kwargs):
+    if '--noinput' not in sys.argv and '--fake' not in sys.argv and '--fake-initial' not in sys.argv and 'accounts' in sys.argv:
+        model = get_user_model()
+        if not model.objects.filter(is_superuser=True).exists():
+            sys.stdout.write(textwrap.dedent("""
+                It appears that you just installed Accounts application.
+                You can now create a superuser:
+                
+                """)
+            )
+            manager = sys.argv[0]
+            execute_from_command_line(argv=[manager, 'createsuperuser'])

orchestra/contrib/domains/backends.py

@@ -1,4 +1,5 @@
 import re
+import socket
 import textwrap
 
 from django.utils.translation import ugettext_lazy as _
@@ -8,12 +9,13 @@ from orchestra.contrib.orchestration import Operation
 from orchestra.utils.python import OrderedSet
 
 from . import settings
+from .models import Record, Domain
 
 
 class Bind9MasterDomainBackend(ServiceController):
     """
     Bind9 zone and config generation.
-    It auto-discovers slave Bind9 servers based on your routing configuration or you can use DOMAINS_SLAVES to explicitly configure the slaves.
+    It auto-discovers slave Bind9 servers based on your routing configuration and NS servers.
     """
     CONF_PATH = settings.DOMAINS_MASTERS_PATH
     
@@ -25,7 +27,7 @@ class Bind9MasterDomainBackend(ServiceController):
     )
     ignore_fields = ['serial']
     doc_settings = (settings,
-        ('DOMAINS_SLAVES', 'DOMAINS_MASTERS_PATH')
+        ('DOMAINS_MASTERS_PATH',)
     )
     
     @classmethod
@@ -100,10 +102,32 @@ class Bind9MasterDomainBackend(ServiceController):
             servers.append(server.get_ip())
         return servers
     
+    def get_masters(self, domain):
+        ips = list(settings.DOMAINS_MASTERS)
+        if not ips:
+            ips += self.get_servers(domain, Bind9MasterDomainBackend)
+        return OrderedSet(sorted(ips))
+    
     def get_slaves(self, domain):
-        ips = list(settings.DOMAINS_SLAVES)
+        ips = []
-        ips += self.get_servers(domain, Bind9SlaveDomainBackend)
+        masters = self.get_masters(domain)
-        return OrderedSet(ips)
+        for ns in domain.records.filter(type=Record.NS):
+            hostname = ns.value.rstrip('.')
+            # First try with a DNS query, a more reliable source
+            try:
+                addr = socket.gethostbyname(hostname)
+            except socket.gaierror:
+                # check if domain is declared
+                try:
+                    domain = Domain.objects.get(name=ns)
+                except Domain.DoesNotExist:
+                    continue
+                else:
+                    a_record = domain.records.filter(name=Record.A) or [settings.DOMAINS_DEFAULT_NS]
+                    addr = a_record[0]
+            if addr not in masters:
+                ips.append(addr)
+        return OrderedSet(sorted(ips))
     
     def get_context(self, domain):
         slaves = self.get_slaves(domain)
@@ -154,11 +178,6 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend):
         """ ideally slave should be restarted after master """
         self.append('if [[ $UPDATED == 1 ]]; then { sleep 1 && service bind9 reload; } & fi')
     
-    def get_masters(self, domain):
-        ips = list(settings.DOMAINS_MASTERS)
-        ips += self.get_servers(domain, Bind9MasterDomainBackend)
-        return OrderedSet(ips)
-    
     def get_context(self, domain):
         context = {
             'name': domain.name,

orchestra/contrib/domains/settings.py

@@ -121,10 +121,3 @@ DOMAINS_MASTERS = Setting('DOMAINS_MASTERS',
     validators=[lambda masters: map(validate_ip_address, masters)],
     help_text="Additional master server ip addresses other than autodiscovered by router.get_servers()."
 )
-
-
-DOMAINS_SLAVES = Setting('DOMAINS_SLAVES',
-    (),
-    validators=[lambda slaves: map(validate_ip_address, slaves)],
-    help_text="Additional slave server ip addresses other than autodiscovered by router.get_servers()."
-)

orchestra/contrib/orchestration/admin.py

@@ -3,8 +3,7 @@ from django.utils.html import escape
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.admin.html import monospace_format
+from orchestra.admin.utils import admin_link, admin_date, admin_colored, display_mono
-from orchestra.admin.utils import admin_link, admin_date, admin_colored
 
 from . import settings, helpers
 from .backends import ServiceBackend
@@ -109,13 +108,6 @@ class BackendOperationInline(admin.TabularInline):
         return queryset.prefetch_related('instance')
 
 
-def display_mono(field):
-    def display(self, log):
-        return monospace_format(escape(getattr(log, field)))
-    display.short_description = _(field)
-    return display
-
-
 class BackendLogAdmin(admin.ModelAdmin):
     list_display = (
         'id', 'backend', 'server_link', 'display_state', 'exit_code',
@@ -123,12 +115,12 @@ class BackendLogAdmin(admin.ModelAdmin):
     )
     list_display_links = ('id', 'backend')
     list_filter = ('state', 'backend')
-    inlines = [BackendOperationInline]
+    inlines = (BackendOperationInline,)
-    fields = [
+    fields = (
         'backend', 'server_link', 'state', 'mono_script', 'mono_stdout',
         'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created',
         'execution_time'
-    ]
+    )
     readonly_fields = fields
     
     server_link = admin_link('server')

orchestra/contrib/orchestration/management/commands/orchestrate.py

@@ -1,5 +1,3 @@
-import sys
-
 from django.core.management.base import BaseCommand, CommandError
 from django.db.models.loading import get_model
 

orchestra/contrib/orchestration/manager.py

@@ -3,9 +3,9 @@ import threading
 import traceback
 from collections import OrderedDict
 
-from django import db
 from django.core.mail import mail_admins
 
+from orchestra.utils.db import close_connection
 from orchestra.utils.python import import_class, OrderedSet
 
 from . import settings, Operation
@@ -42,20 +42,6 @@ def as_task(execute):
     return wrapper
 
 
-def close_connection(execute):
-    """ Threads have their own connection pool, closing it when finishing """
-    def wrapper(*args, **kwargs):
-        try:
-            log = execute(*args, **kwargs)
-        except Exception as e:
-            pass
-        else:
-            wrapper.log = log
-        finally:
-            db.connection.close()
-    return wrapper
-
-
 def generate(operations):
     scripts = OrderedDict()
     cache = {}

orchestra/contrib/orchestration/models.py

@@ -68,13 +68,11 @@ class BackendLog(models.Model):
     )
     
     backend = models.CharField(_("backend"), max_length=256)
-    state = models.CharField(_("state"), max_length=16, choices=STATES,
+    state = models.CharField(_("state"), max_length=16, choices=STATES, default=RECEIVED)
-        default=RECEIVED)
+    server = models.ForeignKey(Server, verbose_name=_("server"), related_name='execution_logs')
-    server = models.ForeignKey(Server, verbose_name=_("server"),
-        related_name='execution_logs')
     script = models.TextField(_("script"))
     stdout = models.TextField(_("stdout"))
-    stderr = models.TextField(_("stdin"))
+    stderr = models.TextField(_("stderr"))
     traceback = models.TextField(_("traceback"))
     exit_code = models.IntegerField(_("exit code"), null=True)
     task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True,

orchestra/contrib/resources/actions.py

@@ -16,6 +16,7 @@ def run_monitor(modeladmin, request, queryset):
         modeladmin.log_change(request, resource, _("Run monitors"))
     if async:
         num = len(queryset)
+        # TODO listfilter by uuid: task.request.id + ?task_id__in=ids
         link = reverse('admin:djcelery_taskstate_changelist')
         msg = ungettext(
             _("One selected resource has been <a href='%s'>scheduled for monitoring</a>.") % link,

orchestra/contrib/resources/models.py

@@ -6,7 +6,7 @@ from django.db.models.loading import get_model
 from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
-from djcelery.models import PeriodicTask, CrontabSchedule
+from djcelery.models import CrontabSchedule
 
 from orchestra.core import validators
 from orchestra.models import queryset, fields
@@ -14,7 +14,7 @@ from orchestra.models.utils import get_model_field_path
 from orchestra.utils.paths import get_project_dir
 from orchestra.utils.sys import run
 
-from . import tasks
+from . import tasks, settings
 from .backends import ServiceMonitor
 from .aggregations import Aggregation
 from .validators import validate_scale
@@ -129,6 +129,12 @@ class Resource(models.Model):
     def delete(self, *args, **kwargs):
         super(Resource, self).delete(*args, **kwargs)
         name = 'monitor.%s' % str(self)
+        self.sync_periodic_task()
+    
+    def sync_periodic_task(self):
+        name = 'monitor.%s' % str(self)
+        sync = import_class(settings.RESOURCES_TASK_BACKEND)
+        return sync(self, name)
     
     def get_model_path(self, monitor):
         """ returns a model path between self.content_type and monitor.model """
@@ -136,28 +142,6 @@ class Resource(models.Model):
         monitor_model = ServiceMonitor.get_backend(monitor).model_class()
         return get_model_field_path(monitor_model, resource_model)
     
-    def sync_periodic_task(self):
-        name = 'monitor.%s' % str(self)
-        if self.pk and self.crontab:
-            try:
-                task = PeriodicTask.objects.get(name=name)
-            except PeriodicTask.DoesNotExist:
-                if self.is_active:
-                    PeriodicTask.objects.create(
-                        name=name,
-                        task='resources.Monitor',
-                        args=[self.pk],
-                        crontab=self.crontab
-                    )
-            else:
-                if task.crontab != self.crontab:
-                    task.crontab = self.crontab
-                    task.save(update_fields=['crontab'])
-        else:
-            PeriodicTask.objects.filter(
-                name=name,
-            ).delete()
-    
     def get_scale(self):
         return eval(self.scale)
     

orchestra/contrib/resources/settings.py

@@ -0,0 +1,6 @@
+from orchestra.settings import Setting
+
+
+RESOURCES_TASK_BACKEND = Setting('RESOURCES_TASK_BACKEND',
+    'orchestra.contrib.resources.utils.cron_sync'
+)

orchestra/contrib/resources/tasks.py

@@ -30,6 +30,7 @@ def monitor(resource_id, ids=None, async=True):
             op = Operation(backend, obj, Operation.MONITOR)
             monitorings.append(op)
         # TODO async=True only when running with celery
+        # monitor.request.id
         logs += Operation.execute(monitorings, async=async)
     
     kwargs = {'id__in': ids} if ids else {}

orchestra/contrib/resources/utils.py

@@ -0,0 +1,41 @@
+from orchestra.contrib.crons.utils import apply_local
+
+from . import settings
+
+
+def celery_sync(resource, name):
+    from djcelery.models import PeriodicTask
+    if resource.pk and resource.crontab:
+        try:
+            task = PeriodicTask.objects.get(name=name)
+        except PeriodicTask.DoesNotExist:
+            if resource.is_active:
+                PeriodicTask.objects.create(
+                    name=name,
+                    task='resources.Monitor',
+                    args=[resource.pk],
+                    crontab=resource.crontab
+                )
+        else:
+            if task.crontab != resource.crontab:
+                task.crontab = resource.crontab
+                task.save(update_fields=['crontab'])
+    else:
+        PeriodicTask.objects.filter(
+            name=name,
+        ).delete()
+
+
+def cron_sync(resource, name):
+    if resource.pk and resource.crontab:
+        context = {
+            'manager': os.path.join(paths.get_project_dir(), 'manage.py'),
+            'id': resource.pk,
+        }
+        apply_local(resource.crontab,
+            'python3 %(manager)s runmethod orchestra.contrib.resources.tasks.monitor %(id)s',
+            'orchestra', # TODO
+            name
+        )
+    else:
+        apply_local(resource.crontab, '', 'orchestra', name, action='delete')

orchestra/contrib/settings/admin.py

@@ -91,6 +91,16 @@ class SettingFileView(generic.TemplateView):
     template_name = 'admin/settings/view.html'
     
     def get_context_data(self, **kwargs):
+        from orchestra.contrib.tasks import shared_task
+        import time
+        @shared_task(name='rata')
+        def counter(num, log):
+            for i in range(1, num):
+                with open(log, 'a') as handler:
+                    handler.write(str(i))
+                time.sleep(1)
+        counter.apply_async(10, '/tmp/kakas')
+        
         context = super(SettingFileView, self).get_context_data(**kwargs)
         settings_file = parser.get_settings_file()
         with open(settings_file, 'r') as handler:
@@ -106,4 +116,3 @@ class SettingFileView(generic.TemplateView):
 admin.site.register_url(r'^settings/setting/view/$', SettingFileView.as_view(), 'settings_setting_view')
 admin.site.register_url(r'^settings/setting/$', SettingView.as_view(), 'settings_setting_change')
 OrchestraIndexDashboard.register_link('Administration', 'settings_setting_change', _("Settings"))
-

orchestra/contrib/settings/parser.py

@@ -58,6 +58,7 @@ def get_eval_context():
         '_': _,
     }
 
+
 def serialize(obj, init=True):
     if isinstance(obj, NotSupported):
         return obj

orchestra/utils/options.py

@@ -50,8 +50,7 @@ def database_ready():
             # Celerybeat has yet to stablish a connection at AppConf.ready()
             'celerybeat' not in sys.argv and
             # Allow to run python manage.py without a database
-            len(sys.argv) <= 1 and
+            sys.argv != ['manage.py'] and '--help' not in sys.argv)
-            '--help' not in sys.argv)
 
 
 def dict_setting_to_choices(choices):