pangea
/
django-orchestra
7
0
Fork 0
Code Issues 1 Pull Requests 2 Packages Projects Releases Wiki Activity

Implemented some monitors

This commit is contained in:
Marc 2014-07-11 14:48:46 +00:00
parent e38e70d539
commit c54c084dff
16 changed files with 168 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
TODO.md
View File

@ -49,3 +49,8 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* pip install pyinotify
* create custom field that returns backend python objects
* Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers)
* Resource metric: KB MB B?
* EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0)

4
orchestra/apps/domains/utils.py
View File

@ -1,8 +1,8 @@
import datetime
from django.utils import timezone
def generate_zone_serial():
today = datetime.date.today()
today = timezone.now()
return int("%.4d%.2d%.2d%.2d" % (today.year, today.month, today.day, 0))

45
orchestra/apps/lists/backends.py
View File

@ -1,9 +1,12 @@
from django.template import Template, Context
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from orchestra.apps.resources import ServiceMonitor
from . import settings
class MailmanBackend(ServiceController):
verbose_name = "Mailman"
@ -14,28 +17,22 @@ class MailmanTraffic(ServiceMonitor):
model = 'lists.List'
resource = ServiceMonitor.TRAFFIC
def process(self, output):
for line in output.readlines():
listname, value = line.strip().slpit()
def monitor(self, mail_list):
context = self.get_context(mail_list)
self.append(
"SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n"
"SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n"
" | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n"
" | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n"
" | tr '\\n' '+' \\\n"
" | xargs -i echo {}0 )\n"
"echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context)
def monitor(self, mailinglist):
self.append(
"LISTS=$(grep -v 'post to mailman' /var/log/mailman/post"
" | grep size | cut -d'<' -f2 | cut -d'>' -f1 | sort | uniq"
" | while read line; do \n"
" grep \"$line\" post | head -n1 | awk {'print $8\" \"$11'}"
" | sed 's/size=//' | sed 's/,//'\n"
"done)"
)
self.append(
'SUBS=""\n'
'while read LIST; do\n'
' NAME=$(echo "$LIST" | awk {\'print $1\'})\n'
' SIZE=$(echo "$LIST" | awk {\'print $2\'})\n'
' if [[ ! $(echo -e "$SUBS" | grep "$NAME") ]]; then\n'
' SUBS="${SUBS}${NAME} $(list_members "$NAME" | wc -l)\n"\n'
' fi\n'
' SUBSCRIBERS=$(echo -e "$SUBS" | grep "$NAME" | awk {\'print $2\'})\n'
' echo "$NAME $(($SUBSCRIBERS*$SIZE))"\n'
'done <<< "$LISTS"'
)
def get_context(self, mail_list):
return {
'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH,
'list_name': mail_list.name,
'object_id': mail_list.pk,
'last_date': timezone.localtime(self.get_last_date(mail_list)).strftime("%b %d %H:%M:%S"),
'current_date': timezone.localtime(self.get_current_date()).strftime("%b %d %H:%M:%S"),
}

4
orchestra/apps/lists/forms.py
View File

@ -26,7 +26,7 @@ class ListCreationForm(CleanAddressMixin, forms.ModelForm):
help_text=_("Enter the same password as above, for verification."))
def __init__(self, *args, **kwargs):
super(ListAdminForm, self).__init__(*args, **kwargs)
super(ListCreationForm, self).__init__(*args, **kwargs)
self.fields['password1'].validators.append(validate_password)
def clean_password2(self):
@ -38,7 +38,7 @@ class ListCreationForm(CleanAddressMixin, forms.ModelForm):
return password2
def save(self, commit=True):
obj = super(ListAdminForm, self).save(commit=commit)
obj = super(ListCreationForm, self).save(commit=commit)
obj.set_password(self.cleaned_data["password1"])
return obj

3
orchestra/apps/lists/settings.py
View File

@ -6,3 +6,6 @@ from django.conf import settings
LISTS_DOMAIN_MODEL = getattr(settings, 'LISTS_DOMAIN_MODEL', 'domains.Domain')
LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'grups.orchestra.lan')
LISTS_MAILMAN_POST_LOG_PATH = getattr(settings, 'LISTS_MAILMAN_POST_LOG_PATH',
'/var/log/mailman/post')

4
orchestra/apps/orchestration/backends.py
View File

@ -1,6 +1,6 @@
from datetime import datetime
from functools import partial
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.utils import plugins
@ -89,7 +89,7 @@ class ServiceBackend(object):
return sorted(choices, key=lambda e: e[0])
def get_banner(self):
time = datetime.now().strftime("%h %d, %Y %I:%M:%S")
time = timezone.now().strftime("%h %d, %Y %I:%M:%S")
return "Generated by Orchestra %s" % time
def execute(self, server):

6
orchestra/apps/orchestration/manager.py
View File

@ -10,9 +10,9 @@ from .helpers import send_report
def as_task(execute):
def wrapper(*args, **kwargs):
with db.transaction.commit_manually():
log = execute(*args, **kwargs)
db.transaction.commit()
# with db.transaction.commit_manually():
log = execute(*args, **kwargs)
# db.transaction.commit()
if log.state != log.SUCCESS:
send_report(execute, args, log)
return log

9
orchestra/apps/resources/backends.py
View File

@ -1,6 +1,7 @@
import datetime
from django.contrib.contenttypes.models import ContentType
from django.utils import timezone
from orchestra.apps.orchestration import ServiceBackend
from orchestra.utils.functional import cached
@ -17,14 +18,14 @@ class ServiceMonitor(ServiceBackend):
@classmethod
def get_backends(cls):
""" filter monitor classes """
return [plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__]
for backend in cls.plugins:
if backend != ServiceMonitor and ServiceMonitor in backend.__mro__:
yield backend
@cached
def get_last_date(self, obj):
from .models import MonitorData
try:
# TODO replace
#return MonitorData.objects.filter(content_object=obj).latest().date
ct = ContentType.objects.get_for_model(type(obj))
return MonitorData.objects.filter(content_type=ct, object_id=obj.pk).latest().date
except MonitorData.DoesNotExist:
@ -32,7 +33,7 @@ class ServiceMonitor(ServiceBackend):
@cached
def get_current_date(self):
return datetime.datetime.now()
return timezone.now()
def store(self, log):
""" object_id value """

32
orchestra/apps/resources/models.py
View File

@ -36,22 +36,27 @@ class Resource(models.Model):
verbose_name = models.CharField(_("verbose name"), max_length=256, unique=True)
content_type = models.ForeignKey(ContentType,
help_text=_("Model where this resource will be hooked"))
period = models.CharField(_("period"), max_length=16, choices=PERIODS, default=LAST,
help_text=_("Operation used for aggregating this resource monitored data."))
period = models.CharField(_("period"), max_length=16, choices=PERIODS,
default=LAST,
help_text=_("Operation used for aggregating this resource monitored"
"data."))
ondemand = models.BooleanField(_("on demand"), default=False,
help_text=_("If enabled the resource will not be pre-allocated, "
"but allocated under the application demand"))
default_allocation = models.PositiveIntegerField(_("default allocation"),
null=True, blank=True,
help_text=_("Default allocation value used when this is not an "
"on demand resource"),
null=True, blank=True)
"on demand resource"))
is_active = models.BooleanField(_("is active"), default=True)
disable_trigger = models.BooleanField(_("disable trigger"), default=False,
help_text=_("Disables monitor's resource exeeded and recovery triggers"))
help_text=_("Disables monitors exeeded and recovery triggers"))
crontab = models.ForeignKey(CrontabSchedule, verbose_name=_("crontab"),
help_text=_("Crontab for periodic execution"))
monitors = MultiSelectField(_("monitors"), max_length=256,
choices=ServiceMonitor.get_choices())
null=True, blank=True,
help_text=_("Crontab for periodic execution. "
"Leave it empty to disable periodic monitoring"))
monitors = MultiSelectField(_("monitors"), max_length=256, blank=True,
choices=ServiceMonitor.get_choices(),
help_text=_("Monitor backends used for monitoring this resource."))
def __unicode__(self):
return self.name
@ -83,7 +88,8 @@ class Resource(models.Model):
def group_by_content_type(cls):
prev = None
group = []
for resource in cls.objects.filter(is_active=True).order_by('content_type'):
resources = cls.objects.filter(is_active=True).order_by('content_type')
for resource in resources:
ct = resource.content_type
if prev != ct:
if group:
@ -121,7 +127,7 @@ class ResourceData(models.Model):
def get_used(self):
resource = self.resource
today = datetime.date.today()
today = timezone.now()
result = 0
has_result = False
for monitor in resource.monitors:
@ -133,7 +139,8 @@ class ResourceData(models.Model):
except MonitorData.DoesNotExist:
continue
has_result = True
epoch = datetime(year=today.year, month=today.month, day=1)
epoch = datetime(year=today.year, month=today.month, day=1,
tzinfo=timezone.utc)
total = (epoch-last.date).total_seconds()
dataset = dataset.filter(date__year=today.year,
date__month=today.month)
@ -154,7 +161,8 @@ class ResourceData(models.Model):
continue
has_result = True
else:
raise NotImplementedError("%s support not implemented" % self.period)
msg = "%s support not implemented" % self.period
raise NotImplementedError(msg)
return result if has_result else None

3
orchestra/apps/resources/serializers.py
View File

@ -24,6 +24,7 @@ class ResourceSerializer(serializers.ModelSerializer):
# Monkey-patching section
if not running_syncdb():
# TODO why this is even loaded during syncdb?
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
router.insert(model, 'resources', ResourceSerializer, required=False, many=True)
@ -54,7 +55,7 @@ if not running_syncdb():
old_metadata = viewset.metadata
def metadata(self, request, resources=resources):
""" Display resource configuration """
""" Provides available resources description """
ret = old_metadata(self, request)
ret['available_resources'] = [
{

17
orchestra/apps/resources/tasks.py
View File

@ -1,5 +1,5 @@
from celery import shared_task
from django.utils import timezone
from orchestra.apps.orchestration.models import BackendOperation as Operation
from .backends import ServiceMonitor
@ -26,13 +26,14 @@ def monitor(resource_id):
for obj in model.objects.all():
data = MonitorData.get_or_create(obj, resource)
current = data.get_used()
if data.used < data.allocated and current > data.allocated:
op = Operation.create(backend, data.content_object, Operation.EXCEED)
operations.append(op)
elif res.used > res.allocated and current < res.allocated:
op = Operation.create(backend, data.content_object, Operation.RECOVERY)
operation.append(op)
if not resource.disable_trigger:
if data.used < data.allocated and current > data.allocated:
op = Operation.create(backend, data.content_object, Operation.EXCEED)
operations.append(op)
elif res.used > res.allocated and current < res.allocated:
op = Operation.create(backend, data.content_object, Operation.RECOVERY)
operation.append(op)
data.used = current
data.las_update = datetime.now()
data.las_update = timezone.now()
data.save()
Operation.execute(operations)

68
orchestra/apps/users/backends.py
View File

@ -1,3 +1,4 @@
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
@ -49,13 +50,62 @@ class SystemUserDisk(ServiceMonitor):
def monitor(self, user):
context = self.get_context(user)
self.append("du -s %(home)s | {\n"
" read value\n"
" echo '%(username)s' $value\n"
"}" % context)
self.append("du -s %(home)s | xargs echo %(object_id)s" % context)
def get_context(self, user):
context = SystemUserBackend().get_context(user)
context['object_id'] = user.pk
return context
class FTPTraffic(ServiceMonitor):
model = 'users.User'
resource = ServiceMonitor.TRAFFIC
verbose_name = _('FTP traffic')
def monitor(self, user):
context = self.get_context(user)
self.append("""
grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk '
BEGIN {
ini = "%(last_date)s"
end = "%(current_date)s"
months["Jan"] = "01"
months["Feb"] = "02"
months["Mar"] = "03"
months["Apr"] = "04"
months["May"] = "05"
months["Jun"] = "06"
months["Jul"] = "07"
months["Aug"] = "08"
months["Sep"] = "09"
months["Oct"] = "10"
months["Nov"] = "11"
months["Dec"] = "12"
} {
# log: Fri Jul 11 13:23:17 2014
split($4, t, ":")
# line_date = year month day hour minute second
line_date = $5 months[$2] $3 t[1] t[2] t[3]
if ( line_date > ini && line_date < end)
split($0, l, "\\", ")
split(l[3], b, " ")
sum += b[1]
} END {
if ( sum )
print sum
else
print 0
}
' | xargs echo %(object_id)s """ % context)
def get_context(self, user):
return {
'log_file': settings.USERS_FTP_LOG_PATH,
'last_date': timezone.localtime(self.get_last_date(user)).strftime("%Y%m%d%H%M%S"),
'current_date': timezone.localtime(self.get_current_date()).strftime("%Y%m%d%H%M%S"),
'object_id': user.pk,
'username': user.username,
}
def process(self, output):
# TODO transaction
for line in output.readlines():
username, value = line.strip().slpit()
History.store(object_id=user_id, value=value)

22
orchestra/apps/users/roles/mail/backends.py
View File

@ -29,10 +29,10 @@ class MailSystemUserBackend(ServiceController):
self.append("chown %(username)s.%(group)s %(home)s" % context)
def generate_filter(self, mailbox, context):
datetime = timezone.now().strftime("%B %d, %Y, %H:%M")
now = timezone.now().strftime("%B %d, %Y, %H:%M")
context['filtering'] = (
"# Sieve Filter\n"
"# Generated by Orchestra %s\n\n" % datetime
"# Generated by Orchestra %s\n\n" % now
)
if mailbox.use_custom_filtering:
context['filtering'] += mailbox.custom_filtering
@ -139,7 +139,21 @@ class AutoresponseBackend(ServiceController):
model = 'mail.Autoresponse'
class MailDisk(ServiceMonitor):
class MaildirDisk(ServiceMonitor):
model = 'email.Mailbox'
resource = ServiceMonitor.DISK
verbose_name = _("Mail disk")
verbose_name = _("Maildir disk usage")
def monitor(self, mailbox):
context = self.get_context(mailbox)
self.append(
"SIZE=$(sed -n '2p' %(maildir_path)s | cut -d' ' -f1)\n"
"echo %(object_id)s ${SIZE:-0}" % context
)
def get_context(self, mailbox):
context = MailSystemUserBackend().get_context(site)
context['home'] = settings.EMAILS_HOME % context
context['maildir_path'] = os.path.join(context['home'], 'Maildir/maildirsize')
context['object_id'] = mailbox.pk
return context

2
orchestra/apps/users/settings.py
View File

@ -2,3 +2,5 @@ from django.conf import settings
USERS_SYSTEMUSER_HOME = getattr(settings, 'USERES_SYSTEMUSER_HOME', '/home/%(username)s')
USERS_FTP_LOG_PATH = getattr(settings, 'USERS_FTP_LOG_PATH', '/var/log/vsftpd.log')

28
orchestra/apps/websites/backends/apache.py
View File

@ -1,6 +1,7 @@
import os
from django.template import Template, Context
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
@ -185,8 +186,8 @@ class Apache2Traffic(ServiceMonitor):
context = self.get_context(site)
self.append("""
awk 'BEGIN {
ini = "%(last_date)s";
end = "%(current_date)s";
ini = "%(last_date)s"
end = "%(current_date)s"
months["Jan"] = "01";
months["Feb"] = "02";
@ -201,32 +202,31 @@ class Apache2Traffic(ServiceMonitor):
months["Nov"] = "11";
months["Dec"] = "12";
} {
# date = [11/Jul/2014:13:50:41
date = substr($4, 2)
year = substr(date, 8, 4)
month = months[substr(date, 4, 3)];
day = substr(date, 1, 2)
hour = substr(date, 13, 2)
minute = substr(date, 16, 2)
second = substr(date, 19, 2);
second = substr(date, 19, 2)
line_date = year month day hour minute second
if ( line_date > ini && line_date < end)
if ( $10 == "" )
sum += $9
else
sum += $10;
sum += $10
} END {
print sum;
}' %(log_file)s | {
read value
echo %(site_id)s $value
}
""" % context)
if ( sum )
print sum
else
print 0
}' %(log_file)s | xargs echo %(object_id)s """ % context)
def get_context(self, site):
# TODO log timezone!!
return {
'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
'last_date': self.get_last_date(site).strftime("%Y%m%d%H%M%S"),
'current_date': self.get_current_date().strftime("%Y%m%d%H%M%S"),
'site_id': site.pk,
'last_date': timezone.localtime(self.get_last_date(site)).strftime("%Y%m%d%H%M%S"),
'current_date': timezone.localtime(self.get_current_date()).strftime("%Y%m%d%H%M%S"),
'object_id': site.pk,
}

1
orchestra/utils/time.py
View File

@ -6,6 +6,7 @@ from django.utils.timesince import timesince as django_timesince
from django.utils.timezone import is_aware, utc
# TODO deprecate in favour of celery timesince
def timesince(d, now=None, reversed=False):
""" Hack to provide second precision under 2 minutes """
if not now: