Added phplist and wordpress SaaS traffic backends

TODO.md

@@ -393,3 +393,6 @@ Case
 # Don't enforce one contact per account? remove account.email in favour of contacts?
 
 # Mailer: mark as sent
+
+
+# Implement wordpressmu change password or remove password from the form

orchestra/contrib/lists/backends.py

@@ -302,7 +302,7 @@ class MailmanTraffic(ServiceMonitor):
                                             # anonymized post
                                             pass
                     except IOError as e:
-                        sys.stderr.write(e)
+                        sys.stderr.write(str(e))
                 
                 for list_name, opts in lists.items():
                     __, object_id, size = opts

orchestra/contrib/mailboxes/backends.py

@@ -549,7 +549,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
                                                     except KeyError:
                                                         counter[req_id] = 1
                     except IOError as e:
-                        sys.stderr.write(e)
+                        sys.stderr.write(str(e))
                     
                 for username, opts in users.iteritems():
                     size = 0

orchestra/contrib/resources/models.py

@@ -60,7 +60,7 @@ class Resource(models.Model):
     scale = models.CharField(_("scale"), max_length=32, validators=[validate_scale],
         help_text=_("Scale in which this resource monitoring resoults should "
                     "be prorcessed to match with unit. e.g. <tt>10**9</tt>"))
-    disable_trigger = models.BooleanField(_("disable trigger"), default=False,
+    disable_trigger = models.BooleanField(_("disable trigger"), default=True,
         help_text=_("Disables monitors exeeded and recovery triggers"))
     crontab = models.ForeignKey('djcelery.CrontabSchedule', verbose_name=_("crontab"),
         null=True, blank=True,

orchestra/contrib/saas/backends/phplist.py

@@ -7,6 +7,7 @@ import requests
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.contrib.orchestration import ServiceController
+from orchestra.contrib.resources import ServiceMonitor
 from orchestra.utils.sys import sshrun
 
 from .. import settings
@@ -121,3 +122,114 @@ class PhpListSaaSBackend(ServiceController):
             'db_name': context['db_name'] % context,
         })
         return context
+
+
+class PhpListTraffic(ServiceMonitor):
+    verbose_name = _("phpList SaaS Traffic")
+    model = 'saas.SaaS'
+    default_route_match = "saas.service == 'phplist'"
+    resource = ServiceMonitor.TRAFFIC
+    script_executable = '/usr/bin/python'
+    monthly_sum_old_values = True
+    doc_settings = (settings,
+        ('SAAS_PHPLIST_MAIL_LOG_PATH',)
+    )
+    
+    def prepare(self):
+        mail_log = settings.SAAS_PHPLIST_MAIL_LOG_PATH
+        context = {
+            'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'mail_logs': str((mail_log, mail_log+'.1')),
+        }
+        self.append(textwrap.dedent("""\
+            import sys
+            from datetime import datetime
+            from dateutil import tz
+            
+            def prepare(object_id, list_domain, ini_date):
+                global lists
+                ini_date = to_local_timezone(ini_date)
+                ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
+                lists[list_domain] = [ini_date, object_id, 0]
+            
+            def inside_period(month, day, time, ini_date):
+                global months
+                global end_datetime
+                # Mar  9 17:13:22
+                month = months[month]
+                year = end_datetime.year
+                if month == '12' and end_datetime.month == 1:
+                    year = year+1
+                if len(day) == 1:
+                    day = '0' + day
+                date = str(year) + month + day
+                date += time.replace(':', '')
+                return ini_date < int(date) < end_date
+            
+            def to_local_timezone(date, tzlocal=tz.tzlocal()):
+                # Converts orchestra's UTC dates to local timezone
+                date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
+                date = date.replace(tzinfo=tz.tzutc())
+                date = date.astimezone(tzlocal)
+                return date
+            
+            maillogs = {mail_logs}
+            end_datetime = to_local_timezone('{current_date}')
+            end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
+            months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
+            months = dict((m, '%02d' % n) for n, m in enumerate(months, 1))
+            
+            lists = {{}}
+            id_to_domain = {{}}
+            
+            def monitor(lists, id_to_domain, maillogs):
+                for maillog in maillogs:
+                    try:
+                        with open(maillog, 'r') as maillog:
+                            for line in maillog.readlines():
+                                if ': message-id=<' in line:
+                                    # Sep 15 09:36:51 web postfix/cleanup[8138]: C20FF244283: message-id=<fe94cc3afd20a9dc634cc9d9ed03fee0@u-romani.lists.pangea.org>
+                                    month, day, time, __, __, id, message_id = line.split()[:7]
+                                    list_domain = message_id.split('@')[1][:-1]
+                                    try:
+                                        opts = lists[list_domain]
+                                    except KeyError:
+                                        pass
+                                    else:
+                                        ini_date = opts[0]
+                                        if inside_period(month, day, time, ini_date):
+                                            id = id[:-1]
+                                            id_to_domain[id] = list_domain
+                                elif '>, size=' in line:
+                                    # Sep 15 09:36:51 web postfix/qmgr[2296]: C20FF244283: from=<u-romani@pangea.org>, size=12252, nrcpt=1 (queue active)
+                                    month, day, time, __, __, id, __, size = line.split()[:8]
+                                    id = id[:-1]
+                                    try:
+                                        list_domain = id_to_domain[id]
+                                    except KeyError:
+                                        pass
+                                    else:
+                                        opts = lists[list_domain]
+                                        size = int(size[5:-1])
+                                        opts[2] += size
+                    except IOError as e:
+                        sys.stderr.write(str(e))
+                for opts in lists.values():
+                    print opts[1], opts[2]
+            """).format(**context)
+        )
+
+    def commit(self):
+        self.append('monitor(lists, id_to_domain, maillogs)')
+    
+    def monitor(self, saas):
+        context = self.get_context(saas)
+        self.append("prepare(%(object_id)s, '%(list_domain)s', '%(last_date)s')" % context)
+    
+    def get_context(self, saas):
+        context = {
+            'list_domain': saas.get_site_domain(),
+            'object_id': saas.pk,
+            'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
+        }
+        return context

orchestra/contrib/saas/backends/wordpressmu.py

@@ -1,9 +1,11 @@
 import re
+import textwrap
 
 import requests
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.contrib.orchestration import ServiceController
+from orchestra.contrib.resources import ServiceMonitor
 
 from .. import settings
 
@@ -119,3 +121,105 @@ class WordpressMuBackend(ServiceController):
     
     def delete(self, saas):
         self.append(self.delete_blog, saas)
+
+
+class WordpressMuTraffic(ServiceMonitor):
+    """
+    Parses apache logs,
+    looking for the size of each request on the last word of the log line.
+    """
+    verbose_name = _("Wordpress MU Traffic")
+    model = 'saas.SaaS'
+    default_route_match = "saas.service == 'wordpress'"
+    script_executable = '/usr/bin/python'
+    monthly_sum_old_values = True
+    doc_settings = (settings,
+        ('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_WORDPRESS_LOG_PATH')
+    )
+    
+    def prepare(self):
+        access_log = settings.SAAS_WORDPRESS_LOG_PATH
+        context = {
+            'access_logs': str((access_log, access_log+'.1')),
+            'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS),
+        }
+        self.append(textwrap.dedent("""\
+            import sys
+            from datetime import datetime
+            from dateutil import tz
+            
+            def to_local_timezone(date, tzlocal=tz.tzlocal()):
+                date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
+                date = date.replace(tzinfo=tz.tzutc())
+                date = date.astimezone(tzlocal)
+                return date
+            
+            # Use local timezone
+            end_date = to_local_timezone('{current_date}')
+            end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
+            access_logs = {access_logs}
+            blogs = {{}}
+            months = {{
+                'Jan': '01',
+                'Feb': '02',
+                'Mar': '03',
+                'Apr': '04',
+                'May': '05',
+                'Jun': '06',
+                'Jul': '07',
+                'Aug': '08',
+                'Sep': '09',
+                'Oct': '10',
+                'Nov': '11',
+                'Dec': '12',
+            }}
+            
+            def prepare(object_id, site_domain, ini_date):
+                global blogs
+                ini_date = to_local_timezone(ini_date)
+                ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
+                blogs[site_domain] = [ini_date, object_id, 0]
+            
+            def monitor(blogs, end_date, months, access_logs):
+                for access_log in access_logs:
+                    try:
+                        with open(access_log, 'r') as handler:
+                            for line in handler.readlines():
+                                meta, request, response, hostname, __ = line.split('"')
+                                host, __, __, date, tz = meta.split()
+                                if host in {ignore_hosts}:
+                                    continue
+                                try:
+                                    blog = blogs[hostname]
+                                except KeyError:
+                                    continue
+                                else:
+                                    # [16/Sep/2015:11:40:38 +0200]
+                                    day, month, date = date[1:].split('/')
+                                    year, hour, min, sec = date.split(':')
+                                    date = year + months[month] + day + hour + min + sec
+                                    if blog[0] < int(date) < end_date:
+                                        status, size = response.split()
+                                        blog[2] += int(size)
+                    except IOError as e:
+                        sys.stderr.write(str(e))
+                for opts in blogs.values():
+                    ini_date, object_id, size = opts
+                    print object_id, size
+            """).format(**context)
+        )
+    
+    def monitor(self, saas):
+        context = self.get_context(saas)
+        self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context)
+    
+    def commit(self):
+        self.append('monitor(blogs, end_date, months, access_logs)')
+    
+    def get_context(self, saas):
+        return {
+            'site_domain': saas.get_site_domain(),
+            'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
+            'object_id': saas.pk,
+        }

orchestra/contrib/saas/services/phplist.py

@@ -58,7 +58,7 @@ class PHPListService(SoftwareService):
     
     def get_account(self):
         account_model = self.instance._meta.get_field_by_name('account')[0]
-        return account_model.objects.get_main()
+        return account_model.rel.to.objects.get_main()
     
     def validate(self):
         super(PHPListService, self).validate()

orchestra/contrib/saas/settings.py

@@ -23,6 +23,16 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES',
 )
 
 
+SAAS_TRAFFIC_IGNORE_HOSTS = Setting('SAAS_TRAFFIC_IGNORE_HOSTS',
+    (),
+)
+
+
+SAAS_WORDPRESS_LOG_PATH = Setting('SAAS_WORDPRESS_LOG_PATH',
+    '',
+)
+
+
 SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD',
     'secret'
 )
@@ -132,6 +142,11 @@ SAAS_PHPLIST_CRONTAB = Setting('SAAS_PHPLIST_CRONTAB',
                 "Left blank if you don't want crontab to be configured")
 )
 
+SAAS_PHPLIST_MAIL_LOG_PATH = Setting('SAAS_PHPLIST_MAIL_LOG_PATH',
+    '/var/log/mail.log',
+)
+
+
 SAAS_SEAFILE_DOMAIN = Setting('SAAS_SEAFILE_DOMAIN',
     'seafile.{}'.format(ORCHESTRA_BASE_DOMAIN),
     help_text="Uses <tt>ORCHESTRA_BASE_DOMAIN</tt> by default.",