Improved monitoring backends

TODO.md

@@ -51,7 +51,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 * create custom field that returns backend python objects
 
 * Timezone awareness on monitoring system (reading server-side logs with different TZ than orchestra) maybe a settings value? (use UTC internally, timezone.localtime() when interacting with servers)
-* Resource metric: KB MB B? RESOURCE UNIT!! forms and serializers
 
 * EMAIL backend operations which contain stderr messages (because under certain failures status code is still 0)
 
@@ -72,3 +71,4 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon
 
 
 * backend logs with hal logo
+* Use logs for storing monitored values

orchestra/apps/lists/backends.py

@@ -1,3 +1,5 @@
+import textwrap
+
 from django.template import Template, Context
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
@@ -17,24 +19,35 @@ class MailmanTraffic(ServiceMonitor):
     model = 'lists.List'
     resource = ServiceMonitor.TRAFFIC
     
+    def prepare(self):
+        current_date = timezone.localtime(self.current_date)
+        current_date = current_date.strftime("%b %d %H:%M:%S")
+        self.append(textwrap.dedent("""
+            function monitor () {
+                OBJECT_ID=$1
+                LAST_DATE=$2
+                LIST_NAME="$3"
+                MAILMAN_LOG="$4"
+                
+                SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l)
+                SIZE=$(grep ' post to ${LIST_NAME} ' "${MAILMAN_LOG}" \
+                       | awk '\"$LAST_DATE\"<=$0 && $0<=\"%s\"' \
+                       | sed 's/.*size=\([0-9]*\).*/\\1/' \
+                       | tr '\\n' '+' \
+                       | xargs -i echo {} )
+                echo ${OBJECT_ID} $(( ${SIZE}*${SUBSCRIBERS} ))
+            }""" % current_date))
+    
     def monitor(self, mail_list):
         context = self.get_context(mail_list)
         self.append(
-            "SUBSCRIBERS=$(list_members %(list_name)s | wc -l)\n"
+            'monitor %(object_id)i %(last_date)s "%(list_name)s" "%(log_file)s"' % context)
-            "SIZE=$(grep ' post to %(list_name)s ' %(mailman_log)s \\\n"
-            "       | awk '\"%(last_date)s\"<=$0 && $0<=\"%(current_date)s\"' \\\n"
-            "       | sed 's/.*size=\([0-9]*\).*/\\1/' \\\n"
-            "       | tr '\\n' '+' \\\n"
-            "       | xargs -i echo {}0 )\n"
-            "echo %(object_id)s $(( ${SIZE}*${SUBSCRIBERS} ))" % context)
     
     def get_context(self, mail_list):
         last_date = timezone.localtime(self.get_last_date(mail_list.pk))
-        current_date = timezone.localtime(self.current_date)
         return {
             'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH,
             'list_name': mail_list.name,
             'object_id': mail_list.pk,
             'last_date': last_date.strftime("%b %d %H:%M:%S"),
-            'current_date': current_date.strftime("%b %d %H:%M:%S"),
         }

orchestra/apps/orchestration/admin.py

@@ -81,8 +81,8 @@ class BackendLogAdmin(admin.ModelAdmin):
     date_hierarchy = 'last_update'
     inlines = [BackendOperationInline]
     fields = [
-        'backend', 'server', 'state', 'mono_script', 'mono_stdout', 'mono_stderr',
+        'backend', 'server_link', 'state', 'mono_script', 'mono_stdout',
-        'mono_traceback', 'exit_code', 'task_id', 'display_created',
+        'mono_stderr', 'mono_traceback', 'exit_code', 'task_id', 'display_created',
         'display_last_update', 'execution_time'
     ]
     readonly_fields = fields

orchestra/apps/orchestration/backends.py

@@ -97,6 +97,10 @@ class ServiceBackend(plugins.Plugin):
         else:
             self.cmds[-1][1].append(cmd)
     
+    def prepare(self):
+        """ hook for executing something at the beging """
+        pass
+    
     def commit(self):
         """
         apply the configuration, usually reloading a service 

orchestra/apps/orchestration/manager.py

@@ -38,11 +38,11 @@ def execute(operations):
     cache = {}
     for operation in operations:
         servers = router.get_servers(operation, cache=cache)
-        print cache
         for server in servers:
             key = (server, operation.backend)
             if key not in scripts:
                 scripts[key] = (operation.backend(), [operation])
+                scripts[key][0].prepare()
             else:
                 scripts[key][1].append(operation)
             method = getattr(scripts[key][0], operation.action)

orchestra/apps/orchestration/models.py

@@ -60,7 +60,7 @@ class BackendLog(models.Model):
     traceback = models.TextField(_("traceback"))
     exit_code = models.IntegerField(_("exit code"), null=True)
     task_id = models.CharField(_("task ID"), max_length=36, unique=True, null=True,
-            help_text="Celery task ID")
+            help_text="Celery task ID when used as execution backend")
     created = models.DateTimeField(_("created"), auto_now_add=True)
     last_update = models.DateTimeField(_("last update"), auto_now=True)
     

orchestra/apps/resources/models.py

@@ -129,10 +129,10 @@ class ResourceData(models.Model):
         ct = ContentType.objects.get_for_model(type(obj))
         try:
             return cls.objects.get(content_type=ct, object_id=obj.pk,
-                                   resource=resource)
+                    resource=resource)
         except cls.DoesNotExist:
             return cls.objects.create(content_object=obj, resource=resource,
-                                      allocated=resource.default_allocation)
+                    allocated=resource.default_allocation)
     
     def get_used(self):
         return helpers.compute_resource_usage(self)
@@ -173,6 +173,7 @@ def create_resource_relation():
             return data
         
         def __get__(self, obj, cls):
+            """ proxy handled object """
             self.obj = obj
             return self
     

orchestra/apps/resources/serializers.py

@@ -25,8 +25,8 @@ class ResourceSerializer(serializers.ModelSerializer):
 
 if not running_syncdb():
     # TODO why this is even loaded during syncdb?
-    for resources in Resource.group_by_content_type():
+    for ct, resources in Resource.objects.group_by('content_type'):
-        model = resources[0].content_type.model_class()
+        model = ct.model_class()
         try:
             router.insert(model, 'resources', ResourceSerializer, required=False, many=True)
         except KeyError:

orchestra/apps/users/backends.py

@@ -1,3 +1,5 @@
+import textwrap
+
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
@@ -15,16 +17,16 @@ class SystemUserBackend(ServiceController):
     def save(self, user):
         context = self.get_context(user)
         if user.is_main:
-            self.append(
+            self.append(textwrap.dedent("""
-                "if [[ $( id %(username)s ) ]]; then \n"
+                if [[ $( id %(username)s ) ]]; then
-                "   usermod --password '%(password)s' %(username)s \n"
+                   usermod --password '%(password)s' %(username)s
-                "else \n"
+                else
-                "   useradd %(username)s --password '%(password)s' \\\n"
+                   useradd %(username)s --password '%(password)s' \
-                "       --shell /bin/false \n"
+                         --shell /bin/false
-                "fi" % context
+                fi
-            )
+                mkdir -p %(home)s
-            self.append("mkdir -p %(home)s" % context)
+                chown %(username)s.%(username)s %(home)s""" % context
-            self.append("chown %(username)s.%(username)s %(home)s" % context)
+            ))
         else:
             self.delete(user)
     
@@ -63,50 +65,58 @@ class FTPTraffic(ServiceMonitor):
     resource = ServiceMonitor.TRAFFIC
     verbose_name = _('FTP traffic')
     
+    def prepare(self):
+        current_date = timezone.localtime(self.current_date)
+        current_date = current_date.strftime("%Y%m%d%H%M%S")
+        self.append(textwrap.dedent("""
+            function monitor () {
+                OBJECT_ID=$1
+                INI_DATE=$2
+                USERNAME="$3"
+                LOG_FILE="$4"
+                grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \
+                    | grep " \\[${USERNAME}\\] " \
+                    | awk -v ini="${INI_DATE}" '
+                        BEGIN {
+                            end = "%s"
+                            sum = 0
+                            months["Jan"] = "01"
+                            months["Feb"] = "02"
+                            months["Mar"] = "03"
+                            months["Apr"] = "04"
+                            months["May"] = "05"
+                            months["Jun"] = "06"
+                            months["Jul"] = "07"
+                            months["Aug"] = "08"
+                            months["Sep"] = "09"
+                            months["Oct"] = "10"
+                            months["Nov"] = "11"
+                            months["Dec"] = "12"
+                        } {
+                            # log: Fri Jul 11 13:23:17 2014
+                            split($4, t, ":")
+                            # line_date = year month day hour minute second
+                            line_date = $5 months[$2] $3 t[1] t[2] t[3]
+                            if ( line_date > ini && line_date < end)
+                                split($0, l, "\\", ")
+                                split(l[3], b, " ")
+                                sum += b[1]
+                        } END {
+                            print sum
+                        }
+                    ' | xargs echo ${OBJECT_ID}
+            }""" % current_date))
+    
     def monitor(self, user):
         context = self.get_context(user)
-        self.append("""
+        self.append(
-            grep "UPLOAD\|DOWNLOAD" %(log_file)s | grep " \\[%(username)s\\] " | awk '
+            'monitor %(object_id)i %(last_date)s "%(username)s" "%(log_file)s"' % context)
-                BEGIN {
-                    ini = "%(last_date)s"
-                    end = "%(current_date)s"
-                    
-                    months["Jan"] = "01"
-                    months["Feb"] = "02"
-                    months["Mar"] = "03"
-                    months["Apr"] = "04"
-                    months["May"] = "05"
-                    months["Jun"] = "06"
-                    months["Jul"] = "07"
-                    months["Aug"] = "08"
-                    months["Sep"] = "09"
-                    months["Oct"] = "10"
-                    months["Nov"] = "11"
-                    months["Dec"] = "12"
-                } {
-                    # log: Fri Jul 11 13:23:17 2014
-                    split($4, t, ":")
-                    # line_date = year month day hour minute second
-                    line_date = $5 months[$2] $3 t[1] t[2] t[3]
-                    if ( line_date > ini && line_date < end)
-                        split($0, l, "\\", ")
-                        split(l[3], b, " ")
-                        sum += b[1]
-                } END {
-                    if ( sum )
-                        print sum
-                    else
-                        print 0
-                }
-            ' | xargs echo %(object_id)s """ % context)
     
     def get_context(self, user):
         last_date = timezone.localtime(self.get_last_date(user.pk))
-        current_date = timezone.localtime(self.current_date)
         return {
             'log_file': settings.USERS_FTP_LOG_PATH,
             'last_date': last_date.strftime("%Y%m%d%H%M%S"),
-            'current_date': current_date.strftime("%Y%m%d%H%M%S"),
             'object_id': user.pk,
             'username': user.username,
         }

orchestra/apps/websites/backends/apache.py

@@ -1,3 +1,4 @@
+import textwrap
 import os
 
 from django.template import Template, Context
@@ -182,48 +183,57 @@ class Apache2Traffic(ServiceMonitor):
     resource = ServiceMonitor.TRAFFIC
     verbose_name = _("Apache 2 Traffic")
     
+    def prepare(self):
+        current_date = timezone.localtime(self.current_date)
+        current_date = current_date.strftime("%Y%m%d%H%M%S")
+        self.append(textwrap.dedent("""
+            function monitor () {
+                OBJECT_ID=$1
+                INI_DATE=$2
+                LOG_FILE="$3"
+                {
+                    awk -v ini="${INI_DATE}" '
+                    BEGIN {
+                        end = "%s"
+                        sum = 0
+                        months["Jan"] = "01";
+                        months["Feb"] = "02";
+                        months["Mar"] = "03";
+                        months["Apr"] = "04";
+                        months["May"] = "05";
+                        months["Jun"] = "06";
+                        months["Jul"] = "07";
+                        months["Aug"] = "08";
+                        months["Sep"] = "09";
+                        months["Oct"] = "10";
+                        months["Nov"] = "11";
+                        months["Dec"] = "12";
+                    } {
+                        # date = [11/Jul/2014:13:50:41
+                        date = substr($4, 2)
+                        year = substr(date, 8, 4)
+                        month = months[substr(date, 4, 3)];
+                        day = substr(date, 1, 2)
+                        hour = substr(date, 13, 2)
+                        minute = substr(date, 16, 2)
+                        second = substr(date, 19, 2)
+                        line_date = year month day hour minute second
+                        if ( line_date > ini && line_date < end)
+                            sum += $NF
+                    } END {
+                        print sum
+                    }' "${LOG_FILE}" || echo 0
+                } | xargs echo ${OBJECT_ID}
+            }""" % current_date))
+    
     def monitor(self, site):
         context = self.get_context(site)
-        self.append("""{
+        self.append('monitor %(object_id)i %(last_date)s "%(log_file)s"' % context)
-            awk 'BEGIN {
-                ini = "%(last_date)s"
-                end = "%(current_date)s"
-                sum = 0
-                
-                months["Jan"] = "01";
-                months["Feb"] = "02";
-                months["Mar"] = "03";
-                months["Apr"] = "04";
-                months["May"] = "05";
-                months["Jun"] = "06";
-                months["Jul"] = "07";
-                months["Aug"] = "08";
-                months["Sep"] = "09";
-                months["Oct"] = "10";
-                months["Nov"] = "11";
-                months["Dec"] = "12";
-            } {
-                # date = [11/Jul/2014:13:50:41
-                date = substr($4, 2)
-                year = substr(date, 8, 4)
-                month = months[substr(date, 4, 3)];
-                day = substr(date, 1, 2)
-                hour = substr(date, 13, 2)
-                minute = substr(date, 16, 2)
-                second = substr(date, 19, 2)
-                line_date = year month day hour minute second
-                if ( line_date > ini && line_date < end)
-                    sum += $NF
-            } END {
-                print sum
-            }' %(log_file)s || echo 0; } | xargs echo %(object_id)s """ % context)
     
     def get_context(self, site):
         last_date = timezone.localtime(self.get_last_date(site.pk))
-        current_date = timezone.localtime(self.current_date)
         return {
             'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name),
             'last_date': last_date.strftime("%Y%m%d%H%M%S"),
-            'current_date': current_date.strftime("%Y%m%d%H%M%S"),
             'object_id': site.pk,
         }

orchestra/conf/base_settings.py

@@ -118,8 +118,8 @@ AUTHENTICATION_BACKENDS = [
 ]
 
 
-# Email config
+#TODO  Email config
-EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
+#EMAIL_BACKEND = 'djcelery_email.backends.CeleryEmailBackend'
 
 
 #################################