Added resource old monitored data aggregation for space efficiency

This commit is contained in:
Marc Aymerich 2015-08-04 09:47:39 +00:00
parent a8f4b17149
commit d8c529f936
13 changed files with 179 additions and 19 deletions

27
TODO.md
View File

@ -429,7 +429,30 @@ Case
# Serie1
# Pangea post-create: lorena no has afegit el webalizer
# cleanup monitor data
# cleanup monitor data: helpers.functions into ServiceMonitor methods
# Add SPF record type
# OVZ TRAFFIC ACCOUNTING!!
# PHPlist cron, bounces and traffic (maybe specific mail script with sitename)
'crontab': settings.SAAS_PHPLIST_CRONTAB.replace('$', '$$')
}
*/10 * * * * PHPLIST=%(php_list_path)s; export SITE="%(site)s"; php $PHPLIST/admin/index.php -c $PHPLIST/config/config.php -p processqueue > /dev/null
*/40 * * * * PHPLIST=%(php_list_path)s; export SITE="%(site)s"; php $PHPLIST/admin/index.php -c $PHPLIST/config/config.php -p processbounces > /dev/null
if settings.SAAS_PHPLIST_CRONTAB:
self.append(textwrap.dedent("""
# Configuring phpList crontabs
if [[ ! $(crontab -l | grep "^%(site)s") ]]; then
cat << EOF | crontab
$(crontab -l)
# %(banner)s
%(crontab)s
EOF
fi
# use "su $user --shell /bin/bash" on backends for security : MKDIR -p...
# model.field.flatchoices

View File

@ -120,6 +120,7 @@ class MysqlDisk(ServiceMonitor):
"""
model = 'databases.Database'
verbose_name = _("MySQL disk")
delete_old_equal_values = True
def exceeded(self, db):
if db.type != db.MYSQL:

View File

@ -259,6 +259,7 @@ class Record(models.Model):
AAAA = 'AAAA'
SRV = 'SRV'
TXT = 'TXT'
SPF = 'SPF'
SOA = 'SOA'
TYPE_CHOICES = (
@ -269,6 +270,7 @@ class Record(models.Model):
(AAAA, _("AAAA (IPv6 address)")),
(SRV, "SRV"),
(TXT, "TXT"),
(SPF, "SPF"),
(SOA, "SOA"),
)
@ -279,6 +281,7 @@ class Record(models.Model):
AAAA: (validate_ipv6_address,),
CNAME: (validators.validate_zone_label,),
TXT: (validate_ascii, validators.validate_quoted_record),
SPF: (validate_ascii, validators.validate_quoted_record),
SRV: (validators.validate_srv_record,),
SOA: (validators.validate_soa_record,),
}

View File

@ -228,6 +228,7 @@ class MailmanTraffic(ServiceMonitor):
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Mailman traffic")
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('LISTS_MAILMAN_POST_LOG_PATH',)
)
@ -332,6 +333,7 @@ class MailmanSubscribers(ServiceMonitor):
"""
model = 'lists.List'
verbose_name = _("Mailman subscribers")
delete_old_equal_values = True
def monitor(self, mail_list):
context = self.get_context(mail_list)

View File

@ -24,10 +24,12 @@ class SieveFilteringMixin(object):
context['box'] = box
self.append(textwrap.dedent("""
# Create %(box)s mailbox
mkdir -p %(maildir)s/.%(box)s
chown %(user)s:%(group)s %(maildir)s/.%(box)s
su $user --shell /bin/bash << 'EOF'
mkdir -p "%(maildir)s/.%(box)s"
EOF
if [[ ! $(grep '%(box)s' %(maildir)s/subscriptions) ]]; then
echo '%(box)s' >> %(maildir)s/subscriptions
chown $user:$user %(maildir)s/subscriptions
fi
""") % context
)
@ -37,17 +39,18 @@ class SieveFilteringMixin(object):
context['filtering'] = ('# %(banner)s\n' + content) % context
self.append(textwrap.dedent("""\
# Create and compile orchestra sieve filtering
mkdir -p $(dirname '%(filtering_path)s')
cat << 'EOF' > %(filtering_path)s
su $user --shell /bin/bash << 'EOF'
mkdir -p $(dirname "%(filtering_path)s")
cat << ' EOF' > %(filtering_path)s
%(filtering)s
EOF
sievec %(filtering_path)s
chown %(user)s:%(group)s %(filtering_path)s
chown %(user)s:%(group)s %(filtering_cpath)s
EOF
""") % context
)
else:
self.append("echo '' > %(filtering_path)s" % context)
self.append('chown $user:$group %(filtering_path)s' % context)
class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
@ -94,8 +97,9 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
#unit_to_bytes(mailbox.resources.disk.unit)
self.append(textwrap.dedent("""
# Set Maildir quota for %(user)s
su $user --shell /bin/bash << 'EOF'
mkdir -p %(maildir)s
chown %(user)s:%(group)s %(maildir)s
EOF
if [[ ! -f %(maildir)s/maildirsize ]]; then
echo "%(quota)iS" > %(maildir)s/maildirsize
chown %(user)s:%(group)s %(maildir)s/maildirsize
@ -110,7 +114,7 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
self.append(textwrap.dedent("""
nohup bash -c '{ sleep 2 && killall -u %(user)s -s KILL; }' &> /dev/null &
killall -u %(user)s || true
# Fucking postfix SASL caches credentials
# Restart because of Postfix SASL caches credentials
userdel %(user)s || true && RESTART_POSTFIX=1
groupdel %(user)s || true""") % context
)
@ -141,7 +145,6 @@ class DovecotPostfixPasswdVirtualUserBackend(SieveFilteringMixin, ServiceControl
verbose_name = _("Dovecot-Postfix virtualuser")
model = 'mailboxes.Mailbox'
# TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data
def set_user(self, context):
self.append(textwrap.dedent("""
@ -394,6 +397,7 @@ class DovecotMaildirDisk(ServiceMonitor):
model = 'mailboxes.Mailbox'
resource = ServiceMonitor.DISK
verbose_name = _("Dovecot Maildir size")
delete_old_equal_values = True
doc_settings = (settings,
('MAILBOXES_MAILDIRSIZE_PATH',)
)
@ -428,6 +432,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Postfix-Mailscanner traffic")
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('MAILBOXES_MAIL_LOG_PATH',)
)

View File

@ -7,6 +7,8 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceBackend
from . import helpers
class ServiceMonitor(ServiceBackend):
TRAFFIC = 'traffic'
@ -16,6 +18,8 @@ class ServiceMonitor(ServiceBackend):
# TODO UNITS
actions = ('monitor', 'exceeded', 'recovery')
abstract = True
delete_old_equal_values = False
monthly_sum_old_values = False
@classmethod
def get_plugins(cls):
@ -81,3 +85,10 @@ class ServiceMonitor(ServiceBackend):
log = super(ServiceMonitor, self).execute(*args, **kwargs)
self.store(log)
return log
@classmethod
def aggregate(cls, dataset):
if cls.delete_old_equal_values:
return helpers.delete_old_equal_values(dataset)
elif cls.monthly_sum_old_values:
return helpers.monthly_sum_old_values(dataset)

View File

@ -1,3 +1,5 @@
import decimal
from django.template.defaultfilters import date as date_format
@ -79,3 +81,54 @@ def get_history_data(queryset):
else:
result = [resource[0] for resource in resources.values()]
return result
def delete_old_equal_values(dataset):
""" only first and last values of an equal serie (+-error) are kept """
prev_value = None
prev_key = None
delete_count = 0
error = decimal.Decimal('0.005')
third = False
for mdata in dataset.order_by('content_type_id', 'object_id', 'created_at'):
key = (mdata.content_type_id, mdata.object_id)
if prev_key == key:
if prev_value is not None and mdata.value*(1-error) < prev_value < mdata.value*(1+error):
if third:
prev.delete()
delete_count += 1
else:
third = True
else:
third = False
prev_value = mdata.value
prev_key = key
else:
prev_value = None
prev_key = key
prev = mdata
return delete_count
def monthly_sum_old_values(dataset):
aggregated = 0
prev_key = None
prev = None
to_delete = []
delete_count = 0
for mdata in dataset.order_by('content_type_id', 'object_id', 'created_at'):
key = (mdata.content_type_id, mdata.object_id, mdata.created_at.year, mdata.created_at.month)
if prev_key is not None and prev_key != key:
if prev.value != aggregated:
prev.value = aggregated
prev.save(update_fields=('value',))
for obj in to_delete[:-1]:
obj.delete()
delete_count += 1
aggregated = 0
to_delete = []
prev = mdata
prev_key = key
aggregated += mdata.value
to_delete.append(mdata)
return delete_count

View File

@ -0,0 +1,6 @@
from orchestra.contrib.settings import Setting
RESOURCES_OLD_MONITOR_DATA_DAYS = Setting('RESOURCES_OLD_MONITOR_DATA_DAYS',
40,
)

View File

@ -1,8 +1,15 @@
import datetime
from celery.task.schedules import crontab
from django.db import transaction
from django.utils import timezone
from orchestra.contrib.orchestration import Operation
from orchestra.contrib.tasks import task
from orchestra.contrib.tasks import task, periodic_task
from orchestra.models.utils import get_model_field_path
from orchestra.utils.sys import LockFile
from . import settings
from .backends import ServiceMonitor
@ -49,3 +56,22 @@ def monitor(resource_id, ids=None):
triggers.append(op)
Operation.execute(triggers)
return logs
@periodic_task(run_every=crontab(hour=2, minute=30), name='resources.cleanup_old_monitors')
@transaction.atomic
def cleanup_old_monitors(queryset=None):
if queryset is None:
from .models import MonitorData
queryset = MonitorData.objects.filter()
delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS)
threshold = timezone.now() - delta
queryset = queryset.filter(created_at__lt=threshold)
delete_counts = []
for monitor in ServiceMonitor.get_plugins():
dataset = queryset.filter(monitor=monitor)
delete_count = monitor.aggregate(dataset)
delete_counts.append(
(monitor.get_name(), delete_count)
)
return delete_counts

View File

@ -29,6 +29,9 @@
resource['aggregation'] +
(div.indexOf('aggregate') > 0 ? ' (aggregated)': '')
},
xAxis: {
ordinal: false
},
yAxis: {
labels: {
formatter: function () {

View File

@ -101,12 +101,11 @@ class UNIXUserBackend(ServiceController):
context = self.get_context(user)
context.update({
'perm_action': user.set_perm_action,
'perm_home': user.set_perm_base_home,
'perm_to': os.path.join(user.set_perm_base_home, user.set_perm_home_extension),
})
exclude_acl = []
for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS:
context['exclude_acl'] = os.path.join(context['perm_home'], exclude)
context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude)
exclude_acl.append('-not -path "%(exclude_acl)s"' % context)
context['exclude_acl'] = ' \\\n -a '.join(exclude_acl) if exclude_acl else ''
if user.set_perm_perms == 'rw':
@ -115,10 +114,32 @@ class UNIXUserBackend(ServiceController):
context['perm_perms'] = 'r-x' if user.set_perm_action == 'grant' else '-wx'
elif user.set_perm_perms == 'w':
context['perm_perms'] = '-wx' if user.set_perm_action == 'grant' else 'r-x'
# Access paths
head = user.set_perm_base_home
relative = ''
access_paths = ["'%s'" % head]
import fnmatch
for tail in user.set_perm_home_extension.split(os.sep)[:-1]:
relative = os.path.join(relative, tail)
for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS:
if fnmatch.fnmatch(relative, exclude):
break
else:
# No match
head = os.path.join(head, tail)
access_paths.append("'%s'" % head)
context['access_paths'] = ' '.join(access_paths)
if user.set_perm_action == 'grant':
self.append(textwrap.dedent("""\
# Home access
setfacl -m u:%(user)s:--x '%(perm_home)s'
# Grant execution permissions to every parent directory
for access_path in %(access_paths)s; do
# Preserve existing ACLs
acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
perms=$(echo "$acl" | cut -d':' -f3)
perms=$(echo "$perms" | cut -c 1,2)x
setfacl -m u:%(user)s:$perms "$access_path"
} || setfacl -m u:%(user)s:--x "$access_path"
done
# Grant perms to existing and future files
find '%(perm_to)s' %(exclude_acl)s \\
-exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;
@ -182,6 +203,7 @@ class UNIXUserDisk(ServiceMonitor):
model = 'systemusers.SystemUser'
resource = ServiceMonitor.DISK
verbose_name = _('UNIX user disk')
delete_old_equal_values = True
def prepare(self):
super(UNIXUserDisk, self).prepare()
@ -211,6 +233,7 @@ class Exim4Traffic(ServiceMonitor):
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Exim4 traffic")
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('SYSTEMUSERS_MAIL_LOG_PATH',)
)
@ -296,6 +319,7 @@ class VsFTPdTraffic(ServiceMonitor):
resource = ServiceMonitor.TRAFFIC
verbose_name = _('VsFTPd traffic')
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('SYSTEMUSERS_FTP_LOG_PATH',)
)

View File

@ -8,6 +8,7 @@ class OpenVZTraffic(ServiceMonitor):
"""
model = 'vps.VPS'
resource = ServiceMonitor.TRAFFIC
monthly_sum_old_values = True
def process(self, line):
""" diff with last stored value """

View File

@ -420,9 +420,11 @@ class Apache2Traffic(ServiceMonitor):
model = 'websites.Website'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Apache 2 Traffic")
monthly_sum_old_values = True
doc_settings = (settings,
('WEBSITES_TRAFFIC_IGNORE_HOSTS',)
)
def prepare(self):
super(Apache2Traffic, self).prepare()
ignore_hosts = '\\|'.join(settings.WEBSITES_TRAFFIC_IGNORE_HOSTS)