pangea
/
django-orchestra
7
0
Fork 0
Code Issues 1 Pull Requests 2 Packages Projects Releases Wiki Activity

Improved monitoring backends

This commit is contained in:
Marc 2014-07-25 15:17:50 +00:00
parent 800ee58d2d
commit 72ef63ffdf
11 changed files with 139 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
TODO.md
View File

@ -51,7 +51,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* create custom field that returns backend python objects
* Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers)
* Resource metric: KB MB B? RESOURCE UNIT!! forms and serializers
* EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0)
@ -72,3 +71,4 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon
* backend logs with hal logo
* Use logs for storing monitored values

31
orchestra/apps/lists/backends.py
View File

@ -1,3 +1,5 @@
import textwrap
from django.template import Template, Context
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
@ -17,24 +19,35 @@ class MailmanTraffic(ServiceMonitor):
model = 'lists.List'
resource = ServiceMonitor.TRAFFIC
def prepare(self):
current_date = timezone.localtime(self.current_date)
current_date = current_date.strftime("%b %d %H:%M:%S")
self.append(textwrap.dedent("""
function monitor () {
OBJECT_ID=$1
LAST_DATE=$2
LIST_NAME="$3"
MAILMAN_LOG="$4"
SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l)
SIZE=$(grep ' post to ${LIST_NAME} ' "${MAILMAN_LOG}" \
| awk '\"$LAST_DATE\"<=$0 && $0<=\"%s\"' \
| sed 's/.*size=\([0-9]*\).*/\\1/' \
| tr '\\n' '+' \
| xargs -i echo {} )
echo ${OBJECT_ID} $(( ${SIZE}*${SUBSCRIBERS} ))
}""" % current_date))
def monitor(self, mail_list):
context = self.get_context(mail_list)
self.append(
"SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n"
"SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n"
" | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n"
" | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n"
" | tr '\\n' '+' \\\n"
" | xargs -i echo {}0 )\n"
"echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context)
'monitor %(object_id)i %(last_date)s "%(list_name)s" "%(log_file)s"' % context)
def get_context(self, mail_list):
last_date = timezone.localtime(self.get_last_date(mail_list.pk))
current_date = timezone.localtime(self.current_date)
return {
'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH,
'list_name': mail_list.name,
'object_id': mail_list.pk,
'last_date': last_date.strftime("%b %d %H:%M:%S"),
'current_date': current_date.strftime("%b %d %H:%M:%S"),
}

4
orchestra/apps/orchestration/admin.py
View File

@ -81,8 +81,8 @@ class BackendLogAdmin(admin.ModelAdmin):
date_hierarchy = 'last_update'
inlines = [BackendOperationInline]
fields = [
'backend', 'server', 'state', 'mono_script', 'mono_stdout', 'mono_stderr',
'mono_traceback', 'exit_code', 'task_id', 'display_created',
'backend', 'server_link', 'state', 'mono_script', 'mono_stdout',
'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created',
'display_last_update', 'execution_time'
]
readonly_fields = fields

4
orchestra/apps/orchestration/backends.py
View File

@ -97,6 +97,10 @@ class ServiceBackend(plugins.Plugin):
else:
self.cmds[-1][1].append(cmd)
def prepare(self):
""" hook for executing something at the beging """
pass
def commit(self):
"""
apply the configuration, usually reloading a service

2
orchestra/apps/orchestration/manager.py
View File

@ -38,11 +38,11 @@ def execute(operations):
cache = {}
for operation in operations:
servers = router.get_servers(operation, cache=cache)
print cache
for server in servers:
key = (server, operation.backend)
if key not in scripts:
scripts[key] = (operation.backend(), [operation])
scripts[key][0].prepare()
else:
scripts[key][1].append(operation)
method = getattr(scripts[key][0], operation.action)

2
orchestra/apps/orchestration/models.py
View File

@ -60,7 +60,7 @@ class BackendLog(models.Model):
traceback = models.TextField(_("traceback"))
exit_code = models.IntegerField(_("exit code"), null=True)
task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True,
help_text="Celery task ID")
help_text="Celery task ID when used as execution backend")
created = models.DateTimeField(_("created"), auto_now_add=True)
last_update = models.DateTimeField(_("last update"), auto_now=True)

5
orchestra/apps/resources/models.py
View File

@ -129,10 +129,10 @@ class ResourceData(models.Model):
ct = ContentType.objects.get_for_model(type(obj))
try:
return cls.objects.get(content_type=ct, object_id=obj.pk,
resource=resource)
resource=resource)
except cls.DoesNotExist:
return cls.objects.create(content_object=obj, resource=resource,
allocated=resource.default_allocation)
allocated=resource.default_allocation)
def get_used(self):
return helpers.compute_resource_usage(self)
@ -173,6 +173,7 @@ def create_resource_relation():
return data
def __get__(self, obj, cls):
""" proxy handled object """
self.obj = obj
return self

4
orchestra/apps/resources/serializers.py
View File

@ -25,8 +25,8 @@ class ResourceSerializer(serializers.ModelSerializer):
if not running_syncdb():
# TODO why this is even loaded during syncdb?
for resources in Resource.group_by_content_type():
model = resources[0].content_type.model_class()
for ct, resources in Resource.objects.group_by('content_type'):
model = ct.model_class()
try:
router.insert(model, 'resources', ResourceSerializer, required=False, many=True)
except KeyError:

102
orchestra/apps/users/backends.py
View File

@ -1,3 +1,5 @@
import textwrap
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
@ -15,16 +17,16 @@ class SystemUserBackend(ServiceController):
def save(self, user):
context = self.get_context(user)
if user.is_main:
self.append(
"if [[ $( id %(username)s ) ]]; then \n"
" usermod --password '%(password)s' %(username)s \n"
"else \n"
" useradd %(username)s --password '%(password)s' \\\n"
" --shell /bin/false \n"
"fi" % context
)
self.append("mkdir -p %(home)s" % context)
self.append("chown %(username)s.%(username)s %(home)s" % context)
self.append(textwrap.dedent("""
if [[ $( id %(username)s ) ]]; then
usermod --password '%(password)s' %(username)s
else
useradd %(username)s --password '%(password)s' \
--shell /bin/false
fi
mkdir -p %(home)s
chown %(username)s.%(username)s %(home)s""" % context
))
else:
self.delete(user)
@ -63,50 +65,58 @@ class FTPTraffic(ServiceMonitor):
resource = ServiceMonitor.TRAFFIC
verbose_name = _('FTP traffic')
def prepare(self):
current_date = timezone.localtime(self.current_date)
current_date = current_date.strftime("%Y%m%d%H%M%S")
self.append(textwrap.dedent("""
function monitor () {
OBJECT_ID=$1
INI_DATE=$2
USERNAME="$3"
LOG_FILE="$4"
grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \
| grep " \\[${USERNAME}\\] " \
| awk -v ini="${INI_DATE}" '
BEGIN {
end = "%s"
sum = 0
months["Jan"] = "01"
months["Feb"] = "02"
months["Mar"] = "03"
months["Apr"] = "04"
months["May"] = "05"
months["Jun"] = "06"
months["Jul"] = "07"
months["Aug"] = "08"
months["Sep"] = "09"
months["Oct"] = "10"
months["Nov"] = "11"
months["Dec"] = "12"
} {
# log: Fri Jul 11 13:23:17 2014
split($4, t, ":")
# line_date = year month day hour minute second
line_date = $5 months[$2] $3 t[1] t[2] t[3]
if ( line_date > ini && line_date < end)
split($0, l, "\\", ")
split(l[3], b, " ")
sum += b[1]
} END {
print sum
}
' | xargs echo ${OBJECT_ID}
}""" % current_date))
def monitor(self, user):
context = self.get_context(user)
self.append("""
grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk '
BEGIN {
ini = "%(last_date)s"
end = "%(current_date)s"
months["Jan"] = "01"
months["Feb"] = "02"
months["Mar"] = "03"
months["Apr"] = "04"
months["May"] = "05"
months["Jun"] = "06"
months["Jul"] = "07"
months["Aug"] = "08"
months["Sep"] = "09"
months["Oct"] = "10"
months["Nov"] = "11"
months["Dec"] = "12"
} {
# log: Fri Jul 11 13:23:17 2014
split($4, t, ":")
# line_date = year month day hour minute second
line_date = $5 months[$2] $3 t[1] t[2] t[3]
if ( line_date > ini && line_date < end)
split($0, l, "\\", ")
split(l[3], b, " ")
sum += b[1]
} END {
if ( sum )
print sum
else
print 0
}
' | xargs echo %(object_id)s """ % context)
self.append(
'monitor %(object_id)i %(last_date)s "%(username)s" "%(log_file)s"' % context)
def get_context(self, user):
last_date = timezone.localtime(self.get_last_date(user.pk))
current_date = timezone.localtime(self.current_date)
return {
'log_file': settings.USERS_FTP_LOG_PATH,
'last_date': last_date.strftime("%Y%m%d%H%M%S"),
'current_date': current_date.strftime("%Y%m%d%H%M%S"),
'object_id': user.pk,
'username': user.username,
}

80
orchestra/apps/websites/backends/apache.py
View File

@ -1,3 +1,4 @@
import textwrap
import os
from django.template import Template, Context
@ -182,48 +183,57 @@ class Apache2Traffic(ServiceMonitor):
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Apache 2 Traffic")
def prepare(self):
current_date = timezone.localtime(self.current_date)
current_date = current_date.strftime("%Y%m%d%H%M%S")
self.append(textwrap.dedent("""
function monitor () {
OBJECT_ID=$1
INI_DATE=$2
LOG_FILE="$3"
{
awk -v ini="${INI_DATE}" '
BEGIN {
end = "%s"
sum = 0
months["Jan"] = "01";
months["Feb"] = "02";
months["Mar"] = "03";
months["Apr"] = "04";
months["May"] = "05";
months["Jun"] = "06";
months["Jul"] = "07";
months["Aug"] = "08";
months["Sep"] = "09";
months["Oct"] = "10";
months["Nov"] = "11";
months["Dec"] = "12";
} {
# date = [11/Jul/2014:13:50:41
date = substr($4, 2)
year = substr(date, 8, 4)
month = months[substr(date, 4, 3)];
day = substr(date, 1, 2)
hour = substr(date, 13, 2)
minute = substr(date, 16, 2)
second = substr(date, 19, 2)
line_date = year month day hour minute second
if ( line_date > ini && line_date < end)
sum += $NF
} END {
print sum
}' "${LOG_FILE}" || echo 0
} | xargs echo ${OBJECT_ID}
}""" % current_date))
def monitor(self, site):
context = self.get_context(site)
self.append("""{
awk 'BEGIN {
ini = "%(last_date)s"
end = "%(current_date)s"
sum = 0
months["Jan"] = "01";
months["Feb"] = "02";
months["Mar"] = "03";
months["Apr"] = "04";
months["May"] = "05";
months["Jun"] = "06";
months["Jul"] = "07";
months["Aug"] = "08";
months["Sep"] = "09";
months["Oct"] = "10";
months["Nov"] = "11";
months["Dec"] = "12";
} {
# date = [11/Jul/2014:13:50:41
date = substr($4, 2)
year = substr(date, 8, 4)
month = months[substr(date, 4, 3)];
day = substr(date, 1, 2)
hour = substr(date, 13, 2)
minute = substr(date, 16, 2)
second = substr(date, 19, 2)
line_date = year month day hour minute second
if ( line_date > ini && line_date < end)
sum += $NF
} END {
print sum
}' %(log_file)s || echo 0; } | xargs echo %(object_id)s """ % context)
self.append('monitor %(object_id)i %(last_date)s "%(log_file)s"' % context)
def get_context(self, site):
last_date = timezone.localtime(self.get_last_date(site.pk))
current_date = timezone.localtime(self.current_date)
return {
'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
'last_date': last_date.strftime("%Y%m%d%H%M%S"),
'current_date': current_date.strftime("%Y%m%d%H%M%S"),
'object_id': site.pk,
}

4
orchestra/conf/base_settings.py
View File

@ -118,8 +118,8 @@ AUTHENTICATION_BACKENDS = [
]
# Email config
EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
#TODO Email config
#EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
#################################