diff --git a/INSTALL.md b/INSTALL.md index f501d473..7ef1fe56 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -61,6 +61,10 @@ Django-orchestra can be installed on any Linux system, however it is **strongly sudo python3 manage.py setupcelery --username orchestra ``` +7. (Optional) Configure logging + ```bash + sudo python3 manage.py setuplog + ``` 8. Configure the web server: ```bash @@ -69,6 +73,7 @@ Django-orchestra can be installed on any Linux system, however it is **strongly sudo python3 manage.py setupnginx --user orchestra ``` + 9. Start all services: ```bash sudo python manage.py startservices diff --git a/TODO.md b/TODO.md index bada898b..70b24f3c 100644 --- a/TODO.md +++ b/TODO.md @@ -15,8 +15,6 @@ * help_text on readonly_fields specialy Bill.state. (eg. A bill is in OPEN state when bla bla ) -* create log file at /var/log/orchestra.log and rotate - * order.register_at @property def register_on(self): @@ -363,8 +361,6 @@ serailzer self.instance on create. * check certificate: websites directive ssl + domains search on miscellaneous -# ValueError: Unable to configure handler 'file': [Errno 13] Permission denied: '/home/orchestra/panel/orchestra.log' - # billing invoice link on related invoices not overflow nginx GET vars * backendLog store method and language... and use it for display_script with correct lexer @@ -410,3 +406,9 @@ Case # messages SMTP errors: temporary->deferre else Failed # Don't enforce one contact per account? remove account.email in favour of contacts? + +#change class LogEntry(models.Model): + action_time = models.DateTimeField(_('action time'), auto_now=True) to auto_now_add + +# Model operations on Manager instead of model method +# Mailer: mark as sent diff --git a/orchestra/admin/__init__.py b/orchestra/admin/__init__.py index 75b17e58..b61f7980 100644 --- a/orchestra/admin/__init__.py +++ b/orchestra/admin/__init__.py @@ -1,6 +1,6 @@ from functools import update_wrapper -from django.contrib.admin import site +from django.contrib import admin from .dashboard import * from .options import * @@ -12,15 +12,15 @@ urls = [] def register_url(pattern, view, name=""): global urls urls.append((pattern, view, name)) -site.register_url = register_url +admin.site.register_url = register_url -site_get_urls = site.get_urls +site_get_urls = admin.site.get_urls def get_urls(): def wrap(view, cacheable=False): def wrapper(*args, **kwargs): - return site.admin_view(view, cacheable)(*args, **kwargs) - wrapper.admin_site = site + return admin.site.admin_view(view, cacheable)(*args, **kwargs) + wrapper.admin_site = admin.site return update_wrapper(wrapper, view) global urls extra_patterns = [] @@ -29,4 +29,4 @@ def get_urls(): url(pattern, wrap(view), name=name) ) return site_get_urls() + extra_patterns -site.get_urls = get_urls +admin.site.get_urls = get_urls diff --git a/orchestra/contrib/history/__init__.py b/orchestra/contrib/history/__init__.py new file mode 100644 index 00000000..d39042d6 --- /dev/null +++ b/orchestra/contrib/history/__init__.py @@ -0,0 +1 @@ +default_app_config = 'orchestra.contrib.history.apps.HistoryConfig' diff --git a/orchestra/contrib/history/admin.py b/orchestra/contrib/history/admin.py new file mode 100644 index 00000000..a6d9e738 --- /dev/null +++ b/orchestra/contrib/history/admin.py @@ -0,0 +1,94 @@ +from django.contrib import admin +from django.utils.translation import ugettext_lazy as _ +from django.core.urlresolvers import reverse, NoReverseMatch +from django.contrib.admin.templatetags.admin_urls import add_preserved_filters +from django.http import HttpResponseRedirect +from django.contrib.admin.utils import unquote + +from orchestra.admin.utils import admin_link, admin_date + + +class LogEntryAdmin(admin.ModelAdmin): + list_display = ( + '__str__', 'display_action_time', 'user_link', + ) + list_filter = ('action_flag', 'content_type',) + date_hierarchy = 'action_time' + search_fields = ('object_repr', 'change_message') + fields = ( + 'user_link', 'content_object_link', 'display_action_time', 'display_action', 'change_message' + ) + readonly_fields = ( + 'user_link', 'content_object_link', 'display_action_time', 'display_action', + ) + actions = None + + user_link = admin_link('user') + display_action_time = admin_date('action_time', short_description=_("Time")) + + def display_action(self, log): + if log.is_addition(): + return _("Added") + elif log.is_change(): + return _("Changed") + return _("Deleted") + display_action.short_description = _("Action") + display_action.admin_order_field = 'action_flag' + + def content_object_link(self, log): + ct = log.content_type + try: + url = reverse('admin:%s_%s_change' % (ct.app_label, ct.model), args=(log.object_id,)) + except NoReverseMatch: + return log.object_repr + return '%s' % (url, log.object_repr) + content_object_link.short_description = _("Content object") + content_object_link.admin_order_field = 'object_repr' + content_object_link.allow_tags = True + + def changeform_view(self, request, object_id=None, form_url='', extra_context=None): + """ Add rel_opts and object to context """ + context = {} + if 'edit' in request.GET.urlencode(): + obj = self.get_object(request, unquote(object_id)) + context = { + 'rel_opts': obj.content_type.model_class()._meta, + 'object': obj, + } + context.update(extra_context or {}) + return super(LogEntryAdmin, self).changeform_view(request, object_id, form_url, extra_context=context) + + def response_change(self, request, obj): + """ save and continue preserve edit query string """ + response = super(LogEntryAdmin, self).response_change(request, obj) + if 'edit' in request.GET.urlencode() and 'edit' not in response.url: + return HttpResponseRedirect(response.url + '?edit=True') + return response + + def response_post_save_change(self, request, obj): + """ save redirect to object history """ + if 'edit' in request.GET.urlencode(): + opts = obj.content_type.model_class()._meta + post_url = reverse('admin:%s_%s_history' % (opts.app_label, opts.model_name), args=(obj.object_id,)) + preserved_filters = self.get_preserved_filters(request) + post_url = add_preserved_filters({'preserved_filters': preserved_filters, 'opts': opts}, post_url) + return HttpResponseRedirect(post_url) + return super(LogEntryAdmin, self).response_post_save_change(request, obj) + + def has_add_permission(self, *args, **kwargs): + return False + + def has_delete_permission(self, *args, **kwargs): + return False + + def log_addition(self, *args, **kwargs): + pass + + def log_change(self, *args, **kwargs): + pass + + def log_deletion(self, *args, **kwargs): + pass + + +admin.site.register(admin.models.LogEntry, LogEntryAdmin) diff --git a/orchestra/contrib/history/apps.py b/orchestra/contrib/history/apps.py new file mode 100644 index 00000000..04d2233c --- /dev/null +++ b/orchestra/contrib/history/apps.py @@ -0,0 +1,19 @@ +from django import db +from django.apps import AppConfig + +from orchestra.core import administration + + +class HistoryConfig(AppConfig): + name = 'orchestra.contrib.history' + verbose_name = 'History' + + def ready(self): + from django.contrib.admin.models import LogEntry + administration.register( + LogEntry, verbose_name='History', verbose_name_plural='History', icon='History.png' + ) + # prevent loosing creation time on log entry edition + action_time = LogEntry._meta.get_field_by_name('action_time')[0] + action_time.auto_now = False + action_time.auto_now_add = True diff --git a/orchestra/contrib/history/templates/admin/admin/logentry/change_form.html b/orchestra/contrib/history/templates/admin/admin/logentry/change_form.html new file mode 100644 index 00000000..95c693c0 --- /dev/null +++ b/orchestra/contrib/history/templates/admin/admin/logentry/change_form.html @@ -0,0 +1,22 @@ +{% extends "admin/change_form.html" %} +{% load i18n admin_urls %} + +{% block object-tools-items %} +{% endblock %} + + +{% block breadcrumbs %} +{% if rel_opts %} +
+{% trans 'Home' %} +› {{ rel_opts.app_config.verbose_name }} +› {{ rel_opts.model_name|capfirst }} +› {{ object.object_repr|truncatewords:"18" }} +› {% trans 'History' %} +› {% trans 'Edit' %} +
+{% else %} +{{ block.super }} +{% endif %} +{% endblock %} + diff --git a/orchestra/contrib/history/templates/admin/object_history.html b/orchestra/contrib/history/templates/admin/object_history.html new file mode 100644 index 00000000..26eed2d0 --- /dev/null +++ b/orchestra/contrib/history/templates/admin/object_history.html @@ -0,0 +1,43 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls static %} + +{% block breadcrumbs %} +
+{% trans 'Home' %} +› {{ opts.app_config.verbose_name }} +› {{ module_name }} +› {{ object|truncatewords:"18" }} +› {% trans 'History' %} +
+{% endblock %} + +{% block content %} +
+
+ +{% if action_list %} + + + + + + + + + + {% for action in action_list %} + + + + + + {% endfor %} + +
{% trans 'Date/time' %}{% trans 'User' %}{% trans 'Action' %}
{{ action.action_time|date:"DATETIME_FORMAT" }}{{ action.user.get_username }}{% if action.user.get_full_name %} ({{ action.user.get_full_name }}){% endif %}{% if action.is_addition and not action.change_message %}{% trans 'Added' %}{% else %}{{ action.change_message }}{% endif %}
+{% else %} +

{% trans "This object doesn't have a change history. It probably wasn't added via this admin site." %}

+{% endif %} +
+
+{% endblock %} + diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index c9018973..54c0c2c3 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -24,12 +24,12 @@ class SieveFilteringMixin(object): context['box'] = box self.append(textwrap.dedent(""" # Create %(box)s mailbox - su $user --shell /bin/bash << 'EOF' + su %(user)s --shell /bin/bash << 'EOF' mkdir -p "%(maildir)s/.%(box)s" EOF if [[ ! $(grep '%(box)s' %(maildir)s/subscriptions) ]]; then echo '%(box)s' >> %(maildir)s/subscriptions - chown $user:$user %(maildir)s/subscriptions + chown %(user)s:%(user)s %(maildir)s/subscriptions fi """) % context ) @@ -39,7 +39,7 @@ class SieveFilteringMixin(object): context['filtering'] = ('# %(banner)s\n' + content) % context self.append(textwrap.dedent("""\ # Create and compile orchestra sieve filtering - su $user --shell /bin/bash << 'EOF' + su %(user)s --shell /bin/bash << 'EOF' mkdir -p $(dirname "%(filtering_path)s") cat << ' EOF' > %(filtering_path)s %(filtering)s @@ -50,7 +50,7 @@ class SieveFilteringMixin(object): ) else: self.append("echo '' > %(filtering_path)s" % context) - self.append('chown $user:$group %(filtering_path)s' % context) + self.append('chown %(user)s:%(group)s %(filtering_path)s' % context) class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController): @@ -97,7 +97,7 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController): #unit_to_bytes(mailbox.resources.disk.unit) self.append(textwrap.dedent(""" # Set Maildir quota for %(user)s - su $user --shell /bin/bash << 'EOF' + su %(user)s --shell /bin/bash << 'EOF' mkdir -p %(maildir)s EOF if [[ ! -f %(maildir)s/maildirsize ]]; then diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py index 6ee69e21..5ed378da 100644 --- a/orchestra/contrib/mailer/admin.py +++ b/orchestra/contrib/mailer/admin.py @@ -32,6 +32,7 @@ class MessageAdmin(admin.ModelAdmin): ) list_filter = ('state', 'priority', 'retries') list_prefetch_related = ('logs__id') + search_fields = ('to_address', 'from_address', 'subject',) fieldsets = ( (None, { 'fields': ('state', 'priority', ('retries', 'last_try_delta', 'created_at_delta'), diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py index ddf06c2f..14f65c49 100644 --- a/orchestra/contrib/mailer/engine.py +++ b/orchestra/contrib/mailer/engine.py @@ -13,13 +13,9 @@ from . import settings from .models import Message -def send_message(message, num=0, connection=None, bulk=settings.MAILER_BULK_MESSAGES): - if num >= bulk and connection is not None: - connection.close() - connection = None +def send_message(message, connection=None, bulk=settings.MAILER_BULK_MESSAGES): if connection is None: # Reset connection with django - num = 0 connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend') connection.open() error = None @@ -47,20 +43,28 @@ def send_pending(bulk=settings.MAILER_BULK_MESSAGES): try: with LockFile('/dev/shm/mailer.send_pending.lock'): connection = None - num = 0 - for message in Message.objects.filter(state=Message.QUEUED).order_by('priority'): - connection = send_message(message, num, connection, bulk) - num += 1 + cur, total = 0, 0 + for message in Message.objects.filter(state=Message.QUEUED).order_by('priority', 'last_try', 'created_at'): + if cur >= bulk and connection is not None: + connection.close() + cur = 0 + connection = send_message(message, connection, bulk) + cur += 1 + total += 1 now = timezone.now() qs = Q() for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS): delta = timedelta(seconds=seconds) qs = qs | Q(retries=retries, last_try__lte=now-delta) - for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'): - connection = send_message(message, num, connection, bulk) - num += 1 + for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority', 'last_try'): + if cur >= bulk and connection is not None: + connection.close() + cur = 0 + connection = send_message(message, connection, bulk) + cur += 1 + total += 1 if connection is not None: connection.close() - return num + return total except OperationLocked: pass diff --git a/orchestra/contrib/orchestration/admin.py b/orchestra/contrib/orchestration/admin.py index bae2e228..c7242b96 100644 --- a/orchestra/contrib/orchestration/admin.py +++ b/orchestra/contrib/orchestration/admin.py @@ -119,6 +119,7 @@ class BackendLogAdmin(admin.ModelAdmin): ) list_display_links = ('id', 'backend') list_filter = ('state', 'backend', 'server') + search_fields = ('script',) date_hierarchy = 'created_at' inlines = (BackendOperationInline,) fields = ( diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py index 67a708ac..b12b4a0c 100644 --- a/orchestra/contrib/orchestration/management/commands/orchestrate.py +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -6,6 +6,7 @@ from orchestra.contrib.orchestration import manager, Operation from orchestra.contrib.orchestration.models import Server from orchestra.contrib.orchestration.backends import ServiceBackend from orchestra.utils.python import import_class, OrderedSet, AttrDict +from orchestra.utils.sys import confirm class Command(BaseCommand): @@ -91,15 +92,8 @@ class Command(BaseCommand): context = { 'servers': ', '.join(servers), } - msg = ("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context) - confirm = input(msg) - while 1: - if confirm not in ('yes', 'no'): - confirm = input('Please enter either "yes" or "no": ') - continue - if confirm == 'no': - return - break + if not confirm("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context) + return if not dry: logs = manager.execute(scripts, serialize=serialize, async=True) running = list(logs) diff --git a/orchestra/contrib/systemusers/admin.py b/orchestra/contrib/systemusers/admin.py index c27577b3..6acf85fc 100644 --- a/orchestra/contrib/systemusers/admin.py +++ b/orchestra/contrib/systemusers/admin.py @@ -42,7 +42,7 @@ class SystemUserAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, Extende add_form = SystemUserCreationForm form = SystemUserChangeForm ordering = ('-id',) - change_view_actions = (set_permission, disable) + change_view_actions = (set_permission,) actions = (delete_selected, list_accounts) + change_view_actions def display_main(self, user): diff --git a/orchestra/management/commands/setuplog.py b/orchestra/management/commands/setuplog.py new file mode 100644 index 00000000..4092b9c8 --- /dev/null +++ b/orchestra/management/commands/setuplog.py @@ -0,0 +1,82 @@ +import os +import textwrap + +from django.core.management.base import BaseCommand + +from orchestra.contrib.settings import parser as settings_parser +from orchestra.utils.paths import get_project_dir, get_site_dir, get_project_name +from orchestra.utils.sys import run, check_root, confirm + + +class Command(BaseCommand): + help = 'Configures LOGGING setting, creates logging dir and configures logrotate.' + + def add_arguments(self, parser): + parser.add_argument('--noinput', action='store_false', dest='interactive', default=True, + help='Tells Django to NOT prompt the user for input of any kind.') + + @check_root + def handle(self, *args, **options): + interactive = options.get('interactive') + context = { + 'site_dir': get_site_dir(), + 'settings_path': os.path.join(get_project_dir(), 'settings.py'), + 'project_name': get_project_name(), + 'log_dir': os.path.join(get_site_dir(), 'log'), + 'log_path': os.path.join(get_site_dir(), 'log', 'orchestra.log') + } + has_logging = not run('grep "^LOGGING\s*=" %(settings_path)s' % context, valid_codes=(0,1)).exit_code + if has_logging: + if not interactive: + self.stderr.write("Project settings already defines LOGGING setting, doing nothing.") + return + msg = ("\n\nYour project settings file already defines a LOGGING setting.\n" + "Do you want to override it? (yes/no): ") + if not confirm(msg): + return + settings_parser.save({ + 'LOGGING': settings_parser.Remove(), + }) + setuplogrotate = textwrap.dedent("""\ + mkdir %(log_dir)s && chown $(ls -dl %(site_dir)s|awk {'print $3":"$4'}) %(log_dir)s + echo '%(log_dir)s/*.log { + copytruncate + daily + rotate 5 + compress + delaycompress + missingok + notifempty + }' > /etc/logrotate.d/orchestra.%(project_name)s + cat << 'EOF' >> %(settings_path)s + + LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'simple': { + 'format': '%%(asctime)s %%(name)s %%(levelname)s %%(message)s' + }, + }, + 'handlers': { + 'file': { + 'class': 'logging.FileHandler', + 'filename': '%(log_path)s', + 'formatter': 'simple' + }, + }, + 'loggers': { + 'orchestra': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + 'orm': { + 'handlers': ['file'], + 'level': 'INFO', + 'propagate': True, + }, + }, + } + EOF""") % context + run(setuplogrotate, display=True) diff --git a/orchestra/management/commands/setupnginx.py b/orchestra/management/commands/setupnginx.py index 0ade3e44..0797d83d 100644 --- a/orchestra/management/commands/setupnginx.py +++ b/orchestra/management/commands/setupnginx.py @@ -7,7 +7,7 @@ from django.conf import settings from django.core.management.base import BaseCommand, CommandError from orchestra.utils import paths -from orchestra.utils.sys import run, check_root +from orchestra.utils.sys import run, check_root, confirm class Command(BaseCommand): @@ -230,16 +230,9 @@ class Command(BaseCommand): elif diff.exit_code == 1: # File is different, save the old one if interactive: - msg = ("\n\nFile %(file)s be updated, do you like to overide " - "it? (yes/no): " % context) - confirm = input(msg) - while 1: - if confirm not in ('yes', 'no'): - confirm = input('Please enter either "yes" or "no": ') - continue - if confirm == 'no': - return - break + if not confirm("\n\nFile %(file)s be updated, do you like to overide " + "it? (yes/no): " % context) + return run("cp %(file)s %(file)s.save" % context, display=True) run("cat << 'EOF' > %(file)s\n%(conf)s\nEOF" % context, display=True) self.stdout.write("\033[1;31mA new version of %(file)s has been installed.\n " diff --git a/orchestra/static/orchestra/icons/History.png b/orchestra/static/orchestra/icons/History.png new file mode 100644 index 00000000..855b4982 Binary files /dev/null and b/orchestra/static/orchestra/icons/History.png differ diff --git a/orchestra/static/orchestra/icons/History.svg b/orchestra/static/orchestra/icons/History.svg new file mode 100644 index 00000000..623c89f5 --- /dev/null +++ b/orchestra/static/orchestra/icons/History.svg @@ -0,0 +1,554 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/orchestra/utils/sys.py b/orchestra/utils/sys.py index 734ee474..bd98409b 100644 --- a/orchestra/utils/sys.py +++ b/orchestra/utils/sys.py @@ -32,6 +32,17 @@ def check_non_root(func): return wrapped +def confirm(msg): + confirmation = input(msg) + while True: + if confirmation not in ('yes', 'no'): + confirmation = input('Please enter either "yes" or "no": ') + continue + if confirmation == 'no': + return False + return True + + class _Attribute(object): """ Simple string subclass to allow arbitrary attribute access. """ def __init__(self, stdout): diff --git a/scripts/container/deploy.sh b/scripts/container/deploy.sh index d894cc3d..d60a7f44 100755 --- a/scripts/container/deploy.sh +++ b/scripts/container/deploy.sh @@ -88,8 +88,7 @@ if [[ ! $(sudo su postgres -c "psql -lqt" | awk {'print $1'} | grep '^orchestra$ fi # create logfile -touch /home/orchestra/panel/orchestra.log -chown $USER:$USER /home/orchestra/panel/orchestra.log +surun "$PYTHON_BIN $MANAGE setuplog --noinput" # admin_tools needs accounts and does not have migrations