From d8c529f93666fec73c5e0b7ecefa9d44c56db8ac Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Tue, 4 Aug 2015 09:47:39 +0000 Subject: [PATCH] Added resource old monitored data aggregation for space efficiency --- TODO.md | 27 +++++++++- orchestra/contrib/databases/backends.py | 1 + orchestra/contrib/domains/models.py | 3 ++ orchestra/contrib/lists/backends.py | 2 + orchestra/contrib/mailboxes/backends.py | 29 +++++----- orchestra/contrib/resources/backends.py | 11 ++++ orchestra/contrib/resources/helpers.py | 53 +++++++++++++++++++ orchestra/contrib/resources/settings.py | 6 +++ orchestra/contrib/resources/tasks.py | 28 +++++++++- .../admin/resources/resourcedata/history.html | 3 ++ orchestra/contrib/systemusers/backends.py | 32 +++++++++-- orchestra/contrib/vps/backends.py | 1 + orchestra/contrib/websites/backends/apache.py | 2 + 13 files changed, 179 insertions(+), 19 deletions(-) create mode 100644 orchestra/contrib/resources/settings.py diff --git a/TODO.md b/TODO.md index 7359957c..2f3ad993 100644 --- a/TODO.md +++ b/TODO.md @@ -429,7 +429,30 @@ Case # Serie1 -# Pangea post-create: lorena no has afegit el webalizer -# cleanup monitor data +# cleanup monitor data: helpers.functions into ServiceMonitor methods # Add SPF record type + +# OVZ TRAFFIC ACCOUNTING!! + +# PHPlist cron, bounces and traffic (maybe specific mail script with sitename) + 'crontab': settings.SAAS_PHPLIST_CRONTAB.replace('$', '$$') +} +*/10 * * * * PHPLIST=%(php_list_path)s; export SITE="%(site)s"; php $PHPLIST/admin/index.php -c $PHPLIST/config/config.php -p processqueue > /dev/null +*/40 * * * * PHPLIST=%(php_list_path)s; export SITE="%(site)s"; php $PHPLIST/admin/index.php -c $PHPLIST/config/config.php -p processbounces > /dev/null + +if settings.SAAS_PHPLIST_CRONTAB: + self.append(textwrap.dedent(""" + # Configuring phpList crontabs +if [[ ! $(crontab -l | grep "^%(site)s") ]]; then +cat << EOF | crontab +$(crontab -l) +# %(banner)s +%(crontab)s +EOF +fi + +# use "su $user --shell /bin/bash" on backends for security : MKDIR -p... + + +# model.field.flatchoices diff --git a/orchestra/contrib/databases/backends.py b/orchestra/contrib/databases/backends.py index b1d49a3c..ad646d0f 100644 --- a/orchestra/contrib/databases/backends.py +++ b/orchestra/contrib/databases/backends.py @@ -120,6 +120,7 @@ class MysqlDisk(ServiceMonitor): """ model = 'databases.Database' verbose_name = _("MySQL disk") + delete_old_equal_values = True def exceeded(self, db): if db.type != db.MYSQL: diff --git a/orchestra/contrib/domains/models.py b/orchestra/contrib/domains/models.py index b1053311..f78b4306 100644 --- a/orchestra/contrib/domains/models.py +++ b/orchestra/contrib/domains/models.py @@ -259,6 +259,7 @@ class Record(models.Model): AAAA = 'AAAA' SRV = 'SRV' TXT = 'TXT' + SPF = 'SPF' SOA = 'SOA' TYPE_CHOICES = ( @@ -269,6 +270,7 @@ class Record(models.Model): (AAAA, _("AAAA (IPv6 address)")), (SRV, "SRV"), (TXT, "TXT"), + (SPF, "SPF"), (SOA, "SOA"), ) @@ -279,6 +281,7 @@ class Record(models.Model): AAAA: (validate_ipv6_address,), CNAME: (validators.validate_zone_label,), TXT: (validate_ascii, validators.validate_quoted_record), + SPF: (validate_ascii, validators.validate_quoted_record), SRV: (validators.validate_srv_record,), SOA: (validators.validate_soa_record,), } diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py index 5ee8f23d..0e759dad 100644 --- a/orchestra/contrib/lists/backends.py +++ b/orchestra/contrib/lists/backends.py @@ -228,6 +228,7 @@ class MailmanTraffic(ServiceMonitor): resource = ServiceMonitor.TRAFFIC verbose_name = _("Mailman traffic") script_executable = '/usr/bin/python' + monthly_sum_old_values = True doc_settings = (settings, ('LISTS_MAILMAN_POST_LOG_PATH',) ) @@ -332,6 +333,7 @@ class MailmanSubscribers(ServiceMonitor): """ model = 'lists.List' verbose_name = _("Mailman subscribers") + delete_old_equal_values = True def monitor(self, mail_list): context = self.get_context(mail_list) diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index ed3ee3e1..c9018973 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -24,10 +24,12 @@ class SieveFilteringMixin(object): context['box'] = box self.append(textwrap.dedent(""" # Create %(box)s mailbox - mkdir -p %(maildir)s/.%(box)s - chown %(user)s:%(group)s %(maildir)s/.%(box)s + su $user --shell /bin/bash << 'EOF' + mkdir -p "%(maildir)s/.%(box)s" + EOF if [[ ! $(grep '%(box)s' %(maildir)s/subscriptions) ]]; then echo '%(box)s' >> %(maildir)s/subscriptions + chown $user:$user %(maildir)s/subscriptions fi """) % context ) @@ -37,17 +39,18 @@ class SieveFilteringMixin(object): context['filtering'] = ('# %(banner)s\n' + content) % context self.append(textwrap.dedent("""\ # Create and compile orchestra sieve filtering - mkdir -p $(dirname '%(filtering_path)s') - cat << 'EOF' > %(filtering_path)s - %(filtering)s + su $user --shell /bin/bash << 'EOF' + mkdir -p $(dirname "%(filtering_path)s") + cat << ' EOF' > %(filtering_path)s + %(filtering)s + EOF + sievec %(filtering_path)s EOF - sievec %(filtering_path)s - chown %(user)s:%(group)s %(filtering_path)s - chown %(user)s:%(group)s %(filtering_cpath)s """) % context ) else: self.append("echo '' > %(filtering_path)s" % context) + self.append('chown $user:$group %(filtering_path)s' % context) class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController): @@ -94,8 +97,9 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController): #unit_to_bytes(mailbox.resources.disk.unit) self.append(textwrap.dedent(""" # Set Maildir quota for %(user)s - mkdir -p %(maildir)s - chown %(user)s:%(group)s %(maildir)s + su $user --shell /bin/bash << 'EOF' + mkdir -p %(maildir)s + EOF if [[ ! -f %(maildir)s/maildirsize ]]; then echo "%(quota)iS" > %(maildir)s/maildirsize chown %(user)s:%(group)s %(maildir)s/maildirsize @@ -110,7 +114,7 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController): self.append(textwrap.dedent(""" nohup bash -c '{ sleep 2 && killall -u %(user)s -s KILL; }' &> /dev/null & killall -u %(user)s || true - # Fucking postfix SASL caches credentials + # Restart because of Postfix SASL caches credentials userdel %(user)s || true && RESTART_POSTFIX=1 groupdel %(user)s || true""") % context ) @@ -141,7 +145,6 @@ class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceControl verbose_name = _("Dovecot-Postfix virtualuser") model = 'mailboxes.Mailbox' - # TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data def set_user(self, context): self.append(textwrap.dedent(""" @@ -394,6 +397,7 @@ class DovecotMaildirDisk(ServiceMonitor): model = 'mailboxes.Mailbox' resource = ServiceMonitor.DISK verbose_name = _("Dovecot Maildir size") + delete_old_equal_values = True doc_settings = (settings, ('MAILBOXES_MAILDIRSIZE_PATH',) ) @@ -428,6 +432,7 @@ class PostfixMailscannerTraffic(ServiceMonitor): resource = ServiceMonitor.TRAFFIC verbose_name = _("Postfix-Mailscanner traffic") script_executable = '/usr/bin/python' + monthly_sum_old_values = True doc_settings = (settings, ('MAILBOXES_MAIL_LOG_PATH',) ) diff --git a/orchestra/contrib/resources/backends.py b/orchestra/contrib/resources/backends.py index 9430f025..2b6ca374 100644 --- a/orchestra/contrib/resources/backends.py +++ b/orchestra/contrib/resources/backends.py @@ -7,6 +7,8 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.orchestration import ServiceBackend +from . import helpers + class ServiceMonitor(ServiceBackend): TRAFFIC = 'traffic' @@ -16,6 +18,8 @@ class ServiceMonitor(ServiceBackend): # TODO UNITS actions = ('monitor', 'exceeded', 'recovery') abstract = True + delete_old_equal_values = False + monthly_sum_old_values = False @classmethod def get_plugins(cls): @@ -81,3 +85,10 @@ class ServiceMonitor(ServiceBackend): log = super(ServiceMonitor, self).execute(*args, **kwargs) self.store(log) return log + + @classmethod + def aggregate(cls, dataset): + if cls.delete_old_equal_values: + return helpers.delete_old_equal_values(dataset) + elif cls.monthly_sum_old_values: + return helpers.monthly_sum_old_values(dataset) diff --git a/orchestra/contrib/resources/helpers.py b/orchestra/contrib/resources/helpers.py index 6b560fa8..da7cd447 100644 --- a/orchestra/contrib/resources/helpers.py +++ b/orchestra/contrib/resources/helpers.py @@ -1,3 +1,5 @@ +import decimal + from django.template.defaultfilters import date as date_format @@ -79,3 +81,54 @@ def get_history_data(queryset): else: result = [resource[0] for resource in resources.values()] return result + + +def delete_old_equal_values(dataset): + """ only first and last values of an equal serie (+-error) are kept """ + prev_value = None + prev_key = None + delete_count = 0 + error = decimal.Decimal('0.005') + third = False + for mdata in dataset.order_by('content_type_id', 'object_id', 'created_at'): + key = (mdata.content_type_id, mdata.object_id) + if prev_key == key: + if prev_value is not None and mdata.value*(1-error) < prev_value < mdata.value*(1+error): + if third: + prev.delete() + delete_count += 1 + else: + third = True + else: + third = False + prev_value = mdata.value + prev_key = key + else: + prev_value = None + prev_key = key + prev = mdata + return delete_count + + +def monthly_sum_old_values(dataset): + aggregated = 0 + prev_key = None + prev = None + to_delete = [] + delete_count = 0 + for mdata in dataset.order_by('content_type_id', 'object_id', 'created_at'): + key = (mdata.content_type_id, mdata.object_id, mdata.created_at.year, mdata.created_at.month) + if prev_key is not None and prev_key != key: + if prev.value != aggregated: + prev.value = aggregated + prev.save(update_fields=('value',)) + for obj in to_delete[:-1]: + obj.delete() + delete_count += 1 + aggregated = 0 + to_delete = [] + prev = mdata + prev_key = key + aggregated += mdata.value + to_delete.append(mdata) + return delete_count diff --git a/orchestra/contrib/resources/settings.py b/orchestra/contrib/resources/settings.py new file mode 100644 index 00000000..be4ac62d --- /dev/null +++ b/orchestra/contrib/resources/settings.py @@ -0,0 +1,6 @@ +from orchestra.contrib.settings import Setting + + +RESOURCES_OLD_MONITOR_DATA_DAYS = Setting('RESOURCES_OLD_MONITOR_DATA_DAYS', + 40, +) diff --git a/orchestra/contrib/resources/tasks.py b/orchestra/contrib/resources/tasks.py index f1da8395..38ab8220 100644 --- a/orchestra/contrib/resources/tasks.py +++ b/orchestra/contrib/resources/tasks.py @@ -1,8 +1,15 @@ +import datetime + +from celery.task.schedules import crontab +from django.db import transaction +from django.utils import timezone + from orchestra.contrib.orchestration import Operation -from orchestra.contrib.tasks import task +from orchestra.contrib.tasks import task, periodic_task from orchestra.models.utils import get_model_field_path from orchestra.utils.sys import LockFile +from . import settings from .backends import ServiceMonitor @@ -49,3 +56,22 @@ def monitor(resource_id, ids=None): triggers.append(op) Operation.execute(triggers) return logs + + +@periodic_task(run_every=crontab(hour=2, minute=30), name='resources.cleanup_old_monitors') +@transaction.atomic +def cleanup_old_monitors(queryset=None): + if queryset is None: + from .models import MonitorData + queryset = MonitorData.objects.filter() + delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS) + threshold = timezone.now() - delta + queryset = queryset.filter(created_at__lt=threshold) + delete_counts = [] + for monitor in ServiceMonitor.get_plugins(): + dataset = queryset.filter(monitor=monitor) + delete_count = monitor.aggregate(dataset) + delete_counts.append( + (monitor.get_name(), delete_count) + ) + return delete_counts diff --git a/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html b/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html index 2964b5a4..e98eb69c 100644 --- a/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html +++ b/orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html @@ -29,6 +29,9 @@ resource['aggregation'] + (div.indexOf('aggregate') > 0 ? ' (aggregated)': '') }, + xAxis: { + ordinal: false + }, yAxis: { labels: { formatter: function () { diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index cbe3fe9f..c02b169c 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -101,12 +101,11 @@ class UNIXUserBackend(ServiceController): context = self.get_context(user) context.update({ 'perm_action': user.set_perm_action, - 'perm_home': user.set_perm_base_home, 'perm_to': os.path.join(user.set_perm_base_home, user.set_perm_home_extension), }) exclude_acl = [] for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS: - context['exclude_acl'] = os.path.join(context['perm_home'], exclude) + context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude) exclude_acl.append('-not -path "%(exclude_acl)s"' % context) context['exclude_acl'] = ' \\\n -a '.join(exclude_acl) if exclude_acl else '' if user.set_perm_perms == 'rw': @@ -115,10 +114,32 @@ class UNIXUserBackend(ServiceController): context['perm_perms'] = 'r-x' if user.set_perm_action == 'grant' else '-wx' elif user.set_perm_perms == 'w': context['perm_perms'] = '-wx' if user.set_perm_action == 'grant' else 'r-x' + # Access paths + head = user.set_perm_base_home + relative = '' + access_paths = ["'%s'" % head] + import fnmatch + for tail in user.set_perm_home_extension.split(os.sep)[:-1]: + relative = os.path.join(relative, tail) + for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS: + if fnmatch.fnmatch(relative, exclude): + break + else: + # No match + head = os.path.join(head, tail) + access_paths.append("'%s'" % head) + context['access_paths'] = ' '.join(access_paths) if user.set_perm_action == 'grant': self.append(textwrap.dedent("""\ - # Home access - setfacl -m u:%(user)s:--x '%(perm_home)s' + # Grant execution permissions to every parent directory + for access_path in %(access_paths)s; do + # Preserve existing ACLs + acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && { + perms=$(echo "$acl" | cut -d':' -f3) + perms=$(echo "$perms" | cut -c 1,2)x + setfacl -m u:%(user)s:$perms "$access_path" + } || setfacl -m u:%(user)s:--x "$access_path" + done # Grant perms to existing and future files find '%(perm_to)s' %(exclude_acl)s \\ -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\; @@ -182,6 +203,7 @@ class UNIXUserDisk(ServiceMonitor): model = 'systemusers.SystemUser' resource = ServiceMonitor.DISK verbose_name = _('UNIX user disk') + delete_old_equal_values = True def prepare(self): super(UNIXUserDisk, self).prepare() @@ -211,6 +233,7 @@ class Exim4Traffic(ServiceMonitor): resource = ServiceMonitor.TRAFFIC verbose_name = _("Exim4 traffic") script_executable = '/usr/bin/python' + monthly_sum_old_values = True doc_settings = (settings, ('SYSTEMUSERS_MAIL_LOG_PATH',) ) @@ -296,6 +319,7 @@ class VsFTPdTraffic(ServiceMonitor): resource = ServiceMonitor.TRAFFIC verbose_name = _('VsFTPd traffic') script_executable = '/usr/bin/python' + monthly_sum_old_values = True doc_settings = (settings, ('SYSTEMUSERS_FTP_LOG_PATH',) ) diff --git a/orchestra/contrib/vps/backends.py b/orchestra/contrib/vps/backends.py index 29d21415..1dd12dbb 100644 --- a/orchestra/contrib/vps/backends.py +++ b/orchestra/contrib/vps/backends.py @@ -8,6 +8,7 @@ class OpenVZTraffic(ServiceMonitor): """ model = 'vps.VPS' resource = ServiceMonitor.TRAFFIC + monthly_sum_old_values = True def process(self, line): """ diff with last stored value """ diff --git a/orchestra/contrib/websites/backends/apache.py b/orchestra/contrib/websites/backends/apache.py index 4413de3d..2143a613 100644 --- a/orchestra/contrib/websites/backends/apache.py +++ b/orchestra/contrib/websites/backends/apache.py @@ -420,9 +420,11 @@ class Apache2Traffic(ServiceMonitor): model = 'websites.Website' resource = ServiceMonitor.TRAFFIC verbose_name = _("Apache 2 Traffic") + monthly_sum_old_values = True doc_settings = (settings, ('WEBSITES_TRAFFIC_IGNORE_HOSTS',) ) + def prepare(self): super(Apache2Traffic, self).prepare() ignore_hosts = '\\|'.join(settings.WEBSITES_TRAFFIC_IGNORE_HOSTS)