Improved monitoring backends
This commit is contained in:
parent
800ee58d2d
commit
72ef63ffdf
2
TODO.md
2
TODO.md
|
@ -51,7 +51,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
|
|||
* create custom field that returns backend python objects
|
||||
|
||||
* Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers)
|
||||
* Resource metric: KB MB B? RESOURCE UNIT!! forms and serializers
|
||||
|
||||
* EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0)
|
||||
|
||||
|
@ -72,3 +71,4 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon
|
|||
|
||||
|
||||
* backend logs with hal logo
|
||||
* Use logs for storing monitored values
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
import textwrap
|
||||
|
||||
from django.template import Template, Context
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
@ -17,24 +19,35 @@ class MailmanTraffic(ServiceMonitor):
|
|||
model = 'lists.List'
|
||||
resource = ServiceMonitor.TRAFFIC
|
||||
|
||||
def prepare(self):
|
||||
current_date = timezone.localtime(self.current_date)
|
||||
current_date = current_date.strftime("%b %d %H:%M:%S")
|
||||
self.append(textwrap.dedent("""
|
||||
function monitor () {
|
||||
OBJECT_ID=$1
|
||||
LAST_DATE=$2
|
||||
LIST_NAME="$3"
|
||||
MAILMAN_LOG="$4"
|
||||
|
||||
SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l)
|
||||
SIZE=$(grep ' post to ${LIST_NAME} ' "${MAILMAN_LOG}" \
|
||||
| awk '\"$LAST_DATE\"<=$0 && $0<=\"%s\"' \
|
||||
| sed 's/.*size=\([0-9]*\).*/\\1/' \
|
||||
| tr '\\n' '+' \
|
||||
| xargs -i echo {} )
|
||||
echo ${OBJECT_ID} $(( ${SIZE}*${SUBSCRIBERS} ))
|
||||
}""" % current_date))
|
||||
|
||||
def monitor(self, mail_list):
|
||||
context = self.get_context(mail_list)
|
||||
self.append(
|
||||
"SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n"
|
||||
"SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n"
|
||||
" | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n"
|
||||
" | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n"
|
||||
" | tr '\\n' '+' \\\n"
|
||||
" | xargs -i echo {}0 )\n"
|
||||
"echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context)
|
||||
'monitor %(object_id)i %(last_date)s "%(list_name)s" "%(log_file)s"' % context)
|
||||
|
||||
def get_context(self, mail_list):
|
||||
last_date = timezone.localtime(self.get_last_date(mail_list.pk))
|
||||
current_date = timezone.localtime(self.current_date)
|
||||
return {
|
||||
'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH,
|
||||
'list_name': mail_list.name,
|
||||
'object_id': mail_list.pk,
|
||||
'last_date': last_date.strftime("%b %d %H:%M:%S"),
|
||||
'current_date': current_date.strftime("%b %d %H:%M:%S"),
|
||||
}
|
||||
|
|
|
@ -81,8 +81,8 @@ class BackendLogAdmin(admin.ModelAdmin):
|
|||
date_hierarchy = 'last_update'
|
||||
inlines = [BackendOperationInline]
|
||||
fields = [
|
||||
'backend', 'server', 'state', 'mono_script', 'mono_stdout', 'mono_stderr',
|
||||
'mono_traceback', 'exit_code', 'task_id', 'display_created',
|
||||
'backend', 'server_link', 'state', 'mono_script', 'mono_stdout',
|
||||
'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created',
|
||||
'display_last_update', 'execution_time'
|
||||
]
|
||||
readonly_fields = fields
|
||||
|
|
|
@ -97,6 +97,10 @@ class ServiceBackend(plugins.Plugin):
|
|||
else:
|
||||
self.cmds[-1][1].append(cmd)
|
||||
|
||||
def prepare(self):
|
||||
""" hook for executing something at the beging """
|
||||
pass
|
||||
|
||||
def commit(self):
|
||||
"""
|
||||
apply the configuration, usually reloading a service
|
||||
|
|
|
@ -38,11 +38,11 @@ def execute(operations):
|
|||
cache = {}
|
||||
for operation in operations:
|
||||
servers = router.get_servers(operation, cache=cache)
|
||||
print cache
|
||||
for server in servers:
|
||||
key = (server, operation.backend)
|
||||
if key not in scripts:
|
||||
scripts[key] = (operation.backend(), [operation])
|
||||
scripts[key][0].prepare()
|
||||
else:
|
||||
scripts[key][1].append(operation)
|
||||
method = getattr(scripts[key][0], operation.action)
|
||||
|
|
|
@ -60,7 +60,7 @@ class BackendLog(models.Model):
|
|||
traceback = models.TextField(_("traceback"))
|
||||
exit_code = models.IntegerField(_("exit code"), null=True)
|
||||
task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True,
|
||||
help_text="Celery task ID")
|
||||
help_text="Celery task ID when used as execution backend")
|
||||
created = models.DateTimeField(_("created"), auto_now_add=True)
|
||||
last_update = models.DateTimeField(_("last update"), auto_now=True)
|
||||
|
||||
|
|
|
@ -129,10 +129,10 @@ class ResourceData(models.Model):
|
|||
ct = ContentType.objects.get_for_model(type(obj))
|
||||
try:
|
||||
return cls.objects.get(content_type=ct, object_id=obj.pk,
|
||||
resource=resource)
|
||||
resource=resource)
|
||||
except cls.DoesNotExist:
|
||||
return cls.objects.create(content_object=obj, resource=resource,
|
||||
allocated=resource.default_allocation)
|
||||
allocated=resource.default_allocation)
|
||||
|
||||
def get_used(self):
|
||||
return helpers.compute_resource_usage(self)
|
||||
|
@ -173,6 +173,7 @@ def create_resource_relation():
|
|||
return data
|
||||
|
||||
def __get__(self, obj, cls):
|
||||
""" proxy handled object """
|
||||
self.obj = obj
|
||||
return self
|
||||
|
||||
|
|
|
@ -25,8 +25,8 @@ class ResourceSerializer(serializers.ModelSerializer):
|
|||
|
||||
if not running_syncdb():
|
||||
# TODO why this is even loaded during syncdb?
|
||||
for resources in Resource.group_by_content_type():
|
||||
model = resources[0].content_type.model_class()
|
||||
for ct, resources in Resource.objects.group_by('content_type'):
|
||||
model = ct.model_class()
|
||||
try:
|
||||
router.insert(model, 'resources', ResourceSerializer, required=False, many=True)
|
||||
except KeyError:
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
import textwrap
|
||||
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import ugettext_lazy as _
|
||||
|
||||
|
@ -15,16 +17,16 @@ class SystemUserBackend(ServiceController):
|
|||
def save(self, user):
|
||||
context = self.get_context(user)
|
||||
if user.is_main:
|
||||
self.append(
|
||||
"if [[ $( id %(username)s ) ]]; then \n"
|
||||
" usermod --password '%(password)s' %(username)s \n"
|
||||
"else \n"
|
||||
" useradd %(username)s --password '%(password)s' \\\n"
|
||||
" --shell /bin/false \n"
|
||||
"fi" % context
|
||||
)
|
||||
self.append("mkdir -p %(home)s" % context)
|
||||
self.append("chown %(username)s.%(username)s %(home)s" % context)
|
||||
self.append(textwrap.dedent("""
|
||||
if [[ $( id %(username)s ) ]]; then
|
||||
usermod --password '%(password)s' %(username)s
|
||||
else
|
||||
useradd %(username)s --password '%(password)s' \
|
||||
--shell /bin/false
|
||||
fi
|
||||
mkdir -p %(home)s
|
||||
chown %(username)s.%(username)s %(home)s""" % context
|
||||
))
|
||||
else:
|
||||
self.delete(user)
|
||||
|
||||
|
@ -63,50 +65,58 @@ class FTPTraffic(ServiceMonitor):
|
|||
resource = ServiceMonitor.TRAFFIC
|
||||
verbose_name = _('FTP traffic')
|
||||
|
||||
def prepare(self):
|
||||
current_date = timezone.localtime(self.current_date)
|
||||
current_date = current_date.strftime("%Y%m%d%H%M%S")
|
||||
self.append(textwrap.dedent("""
|
||||
function monitor () {
|
||||
OBJECT_ID=$1
|
||||
INI_DATE=$2
|
||||
USERNAME="$3"
|
||||
LOG_FILE="$4"
|
||||
grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \
|
||||
| grep " \\[${USERNAME}\\] " \
|
||||
| awk -v ini="${INI_DATE}" '
|
||||
BEGIN {
|
||||
end = "%s"
|
||||
sum = 0
|
||||
months["Jan"] = "01"
|
||||
months["Feb"] = "02"
|
||||
months["Mar"] = "03"
|
||||
months["Apr"] = "04"
|
||||
months["May"] = "05"
|
||||
months["Jun"] = "06"
|
||||
months["Jul"] = "07"
|
||||
months["Aug"] = "08"
|
||||
months["Sep"] = "09"
|
||||
months["Oct"] = "10"
|
||||
months["Nov"] = "11"
|
||||
months["Dec"] = "12"
|
||||
} {
|
||||
# log: Fri Jul 11 13:23:17 2014
|
||||
split($4, t, ":")
|
||||
# line_date = year month day hour minute second
|
||||
line_date = $5 months[$2] $3 t[1] t[2] t[3]
|
||||
if ( line_date > ini && line_date < end)
|
||||
split($0, l, "\\", ")
|
||||
split(l[3], b, " ")
|
||||
sum += b[1]
|
||||
} END {
|
||||
print sum
|
||||
}
|
||||
' | xargs echo ${OBJECT_ID}
|
||||
}""" % current_date))
|
||||
|
||||
def monitor(self, user):
|
||||
context = self.get_context(user)
|
||||
self.append("""
|
||||
grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk '
|
||||
BEGIN {
|
||||
ini = "%(last_date)s"
|
||||
end = "%(current_date)s"
|
||||
|
||||
months["Jan"] = "01"
|
||||
months["Feb"] = "02"
|
||||
months["Mar"] = "03"
|
||||
months["Apr"] = "04"
|
||||
months["May"] = "05"
|
||||
months["Jun"] = "06"
|
||||
months["Jul"] = "07"
|
||||
months["Aug"] = "08"
|
||||
months["Sep"] = "09"
|
||||
months["Oct"] = "10"
|
||||
months["Nov"] = "11"
|
||||
months["Dec"] = "12"
|
||||
} {
|
||||
# log: Fri Jul 11 13:23:17 2014
|
||||
split($4, t, ":")
|
||||
# line_date = year month day hour minute second
|
||||
line_date = $5 months[$2] $3 t[1] t[2] t[3]
|
||||
if ( line_date > ini && line_date < end)
|
||||
split($0, l, "\\", ")
|
||||
split(l[3], b, " ")
|
||||
sum += b[1]
|
||||
} END {
|
||||
if ( sum )
|
||||
print sum
|
||||
else
|
||||
print 0
|
||||
}
|
||||
' | xargs echo %(object_id)s """ % context)
|
||||
self.append(
|
||||
'monitor %(object_id)i %(last_date)s "%(username)s" "%(log_file)s"' % context)
|
||||
|
||||
def get_context(self, user):
|
||||
last_date = timezone.localtime(self.get_last_date(user.pk))
|
||||
current_date = timezone.localtime(self.current_date)
|
||||
return {
|
||||
'log_file': settings.USERS_FTP_LOG_PATH,
|
||||
'last_date': last_date.strftime("%Y%m%d%H%M%S"),
|
||||
'current_date': current_date.strftime("%Y%m%d%H%M%S"),
|
||||
'object_id': user.pk,
|
||||
'username': user.username,
|
||||
}
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import textwrap
|
||||
import os
|
||||
|
||||
from django.template import Template, Context
|
||||
|
@ -182,48 +183,57 @@ class Apache2Traffic(ServiceMonitor):
|
|||
resource = ServiceMonitor.TRAFFIC
|
||||
verbose_name = _("Apache 2 Traffic")
|
||||
|
||||
def prepare(self):
|
||||
current_date = timezone.localtime(self.current_date)
|
||||
current_date = current_date.strftime("%Y%m%d%H%M%S")
|
||||
self.append(textwrap.dedent("""
|
||||
function monitor () {
|
||||
OBJECT_ID=$1
|
||||
INI_DATE=$2
|
||||
LOG_FILE="$3"
|
||||
{
|
||||
awk -v ini="${INI_DATE}" '
|
||||
BEGIN {
|
||||
end = "%s"
|
||||
sum = 0
|
||||
months["Jan"] = "01";
|
||||
months["Feb"] = "02";
|
||||
months["Mar"] = "03";
|
||||
months["Apr"] = "04";
|
||||
months["May"] = "05";
|
||||
months["Jun"] = "06";
|
||||
months["Jul"] = "07";
|
||||
months["Aug"] = "08";
|
||||
months["Sep"] = "09";
|
||||
months["Oct"] = "10";
|
||||
months["Nov"] = "11";
|
||||
months["Dec"] = "12";
|
||||
} {
|
||||
# date = [11/Jul/2014:13:50:41
|
||||
date = substr($4, 2)
|
||||
year = substr(date, 8, 4)
|
||||
month = months[substr(date, 4, 3)];
|
||||
day = substr(date, 1, 2)
|
||||
hour = substr(date, 13, 2)
|
||||
minute = substr(date, 16, 2)
|
||||
second = substr(date, 19, 2)
|
||||
line_date = year month day hour minute second
|
||||
if ( line_date > ini && line_date < end)
|
||||
sum += $NF
|
||||
} END {
|
||||
print sum
|
||||
}' "${LOG_FILE}" || echo 0
|
||||
} | xargs echo ${OBJECT_ID}
|
||||
}""" % current_date))
|
||||
|
||||
def monitor(self, site):
|
||||
context = self.get_context(site)
|
||||
self.append("""{
|
||||
awk 'BEGIN {
|
||||
ini = "%(last_date)s"
|
||||
end = "%(current_date)s"
|
||||
sum = 0
|
||||
|
||||
months["Jan"] = "01";
|
||||
months["Feb"] = "02";
|
||||
months["Mar"] = "03";
|
||||
months["Apr"] = "04";
|
||||
months["May"] = "05";
|
||||
months["Jun"] = "06";
|
||||
months["Jul"] = "07";
|
||||
months["Aug"] = "08";
|
||||
months["Sep"] = "09";
|
||||
months["Oct"] = "10";
|
||||
months["Nov"] = "11";
|
||||
months["Dec"] = "12";
|
||||
} {
|
||||
# date = [11/Jul/2014:13:50:41
|
||||
date = substr($4, 2)
|
||||
year = substr(date, 8, 4)
|
||||
month = months[substr(date, 4, 3)];
|
||||
day = substr(date, 1, 2)
|
||||
hour = substr(date, 13, 2)
|
||||
minute = substr(date, 16, 2)
|
||||
second = substr(date, 19, 2)
|
||||
line_date = year month day hour minute second
|
||||
if ( line_date > ini && line_date < end)
|
||||
sum += $NF
|
||||
} END {
|
||||
print sum
|
||||
}' %(log_file)s || echo 0; } | xargs echo %(object_id)s """ % context)
|
||||
self.append('monitor %(object_id)i %(last_date)s "%(log_file)s"' % context)
|
||||
|
||||
def get_context(self, site):
|
||||
last_date = timezone.localtime(self.get_last_date(site.pk))
|
||||
current_date = timezone.localtime(self.current_date)
|
||||
return {
|
||||
'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
|
||||
'last_date': last_date.strftime("%Y%m%d%H%M%S"),
|
||||
'current_date': current_date.strftime("%Y%m%d%H%M%S"),
|
||||
'object_id': site.pk,
|
||||
}
|
||||
|
|
|
@ -118,8 +118,8 @@ AUTHENTICATION_BACKENDS = [
|
|||
]
|
||||
|
||||
|
||||
# Email config
|
||||
EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
|
||||
#TODO Email config
|
||||
#EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
|
||||
|
||||
|
||||
#################################
|
||||
|
|
Loading…
Reference in New Issue