pangea/django-orchestra
7
0
Fork 0
Code Issues 1 Pull requests 2 Packages Projects Releases Wiki Activity

Added phplist and wordpress SaaS traffic backends

Browse source
This commit is contained in:
Marc Aymerich 2015-09-16 12:15:05 +00:00
parent 2beed2677a
commit 5b5d62ef38
8 changed files with 238 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
TODO.md
View file

@ -393,3 +393,6 @@ Case
# Don't enforce one contact per account? remove account.email in favour of contacts?
# Mailer: mark as sent
# Implement wordpressmu change password or remove password from the form

2
orchestra/contrib/lists/backends.py
View file

@ -302,7 +302,7 @@ class MailmanTraffic(ServiceMonitor):
# anonymized post
pass
except IOError as e:
sys.stderr.write(e)
sys.stderr.write(str(e))
for list_name, opts in lists.items():
__, object_id, size = opts

2
orchestra/contrib/mailboxes/backends.py
View file

@ -549,7 +549,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
except KeyError:
counter[req_id] = 1
except IOError as e:
sys.stderr.write(e)
sys.stderr.write(str(e))
for username, opts in users.iteritems():
size = 0

2
orchestra/contrib/resources/models.py
View file

@ -60,7 +60,7 @@ class Resource(models.Model):
scale = models.CharField(_("scale"), max_length=32, validators=[validate_scale],
help_text=_("Scale in which this resource monitoring resoults should "
"be prorcessed to match with unit. e.g. <tt>10**9</tt>"))
disable_trigger = models.BooleanField(_("disable trigger"), default=False,
disable_trigger = models.BooleanField(_("disable trigger"), default=True,
help_text=_("Disables monitors exeeded and recovery triggers"))
crontab = models.ForeignKey('djcelery.CrontabSchedule', verbose_name=_("crontab"),
null=True, blank=True,

112
orchestra/contrib/saas/backends/phplist.py
View file

@ -7,6 +7,7 @@ import requests
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
from orchestra.utils.sys import sshrun
from .. import settings
@ -121,3 +122,114 @@ class PhpListSaaSBackend(ServiceController):
'db_name': context['db_name'] % context,
})
return context
class PhpListTraffic(ServiceMonitor):
verbose_name = _("phpList SaaS Traffic")
model = 'saas.SaaS'
default_route_match = "saas.service == 'phplist'"
resource = ServiceMonitor.TRAFFIC
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('SAAS_PHPLIST_MAIL_LOG_PATH',)
)
def prepare(self):
mail_log = settings.SAAS_PHPLIST_MAIL_LOG_PATH
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'mail_logs': str((mail_log, mail_log+'.1')),
}
self.append(textwrap.dedent("""\
import sys
from datetime import datetime
from dateutil import tz
def prepare(object_id, list_domain, ini_date):
global lists
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
lists[list_domain] = [ini_date, object_id, 0]
def inside_period(month, day, time, ini_date):
global months
global end_datetime
# Mar 9 17:13:22
month = months[month]
year = end_datetime.year
if month == '12' and end_datetime.month == 1:
year = year+1
if len(day) == 1:
day = '0' + day
date = str(year) + month + day
date += time.replace(':', '')
return ini_date < int(date) < end_date
def to_local_timezone(date, tzlocal=tz.tzlocal()):
# Converts orchestra's UTC dates to local timezone
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
maillogs = {mail_logs}
end_datetime = to_local_timezone('{current_date}')
end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
months = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec')
months = dict((m, '%02d' % n) for n, m in enumerate(months, 1))
lists = {{}}
id_to_domain = {{}}
def monitor(lists, id_to_domain, maillogs):
for maillog in maillogs:
try:
with open(maillog, 'r') as maillog:
for line in maillog.readlines():
if ': message-id=<' in line:
# Sep 15 09:36:51 web postfix/cleanup[8138]: C20FF244283: message-id=<fe94cc3afd20a9dc634cc9d9ed03fee0@u-romani.lists.pangea.org>
month, day, time, __, __, id, message_id = line.split()[:7]
list_domain = message_id.split('@')[1][:-1]
try:
opts = lists[list_domain]
except KeyError:
pass
else:
ini_date = opts[0]
if inside_period(month, day, time, ini_date):
id = id[:-1]
id_to_domain[id] = list_domain
elif '>, size=' in line:
# Sep 15 09:36:51 web postfix/qmgr[2296]: C20FF244283: from=<u-romani@pangea.org>, size=12252, nrcpt=1 (queue active)
month, day, time, __, __, id, __, size = line.split()[:8]
id = id[:-1]
try:
list_domain = id_to_domain[id]
except KeyError:
pass
else:
opts = lists[list_domain]
size = int(size[5:-1])
opts[2] += size
except IOError as e:
sys.stderr.write(str(e))
for opts in lists.values():
print opts[1], opts[2]
""").format(**context)
)
def commit(self):
self.append('monitor(lists, id_to_domain, maillogs)')
def monitor(self, saas):
context = self.get_context(saas)
self.append("prepare(%(object_id)s, '%(list_domain)s', '%(last_date)s')" % context)
def get_context(self, saas):
context = {
'list_domain': saas.get_site_domain(),
'object_id': saas.pk,
'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
return context

104
orchestra/contrib/saas/backends/wordpressmu.py
View file

@ -1,9 +1,11 @@
import re
import textwrap
import requests
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
from .. import settings
@ -119,3 +121,105 @@ class WordpressMuBackend(ServiceController):
def delete(self, saas):
self.append(self.delete_blog, saas)
class WordpressMuTraffic(ServiceMonitor):
"""
Parses apache logs,
looking for the size of each request on the last word of the log line.
"""
verbose_name = _("Wordpress MU Traffic")
model = 'saas.SaaS'
default_route_match = "saas.service == 'wordpress'"
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_WORDPRESS_LOG_PATH')
)
def prepare(self):
access_log = settings.SAAS_WORDPRESS_LOG_PATH
context = {
'access_logs': str((access_log, access_log+'.1')),
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS),
}
self.append(textwrap.dedent("""\
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
access_logs = {access_logs}
blogs = {{}}
months = {{
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}}
def prepare(object_id, site_domain, ini_date):
global blogs
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
blogs[site_domain] = [ini_date, object_id, 0]
def monitor(blogs, end_date, months, access_logs):
for access_log in access_logs:
try:
with open(access_log, 'r') as handler:
for line in handler.readlines():
meta, request, response, hostname, __ = line.split('"')
host, __, __, date, tz = meta.split()
if host in {ignore_hosts}:
continue
try:
blog = blogs[hostname]
except KeyError:
continue
else:
# [16/Sep/2015:11:40:38 +0200]
day, month, date = date[1:].split('/')
year, hour, min, sec = date.split(':')
date = year + months[month] + day + hour + min + sec
if blog[0] < int(date) < end_date:
status, size = response.split()
blog[2] += int(size)
except IOError as e:
sys.stderr.write(str(e))
for opts in blogs.values():
ini_date, object_id, size = opts
print object_id, size
""").format(**context)
)
def monitor(self, saas):
context = self.get_context(saas)
self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context)
def commit(self):
self.append('monitor(blogs, end_date, months, access_logs)')
def get_context(self, saas):
return {
'site_domain': saas.get_site_domain(),
'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': saas.pk,
}

2
orchestra/contrib/saas/services/phplist.py
View file

@ -58,7 +58,7 @@ class PHPListService(SoftwareService):
def get_account(self):
account_model = self.instance._meta.get_field_by_name('account')[0]
return account_model.objects.get_main()
return account_model.rel.to.objects.get_main()
def validate(self):
super(PHPListService, self).validate()

15
orchestra/contrib/saas/settings.py
View file

@ -23,6 +23,16 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES',
)
SAAS_TRAFFIC_IGNORE_HOSTS = Setting('SAAS_TRAFFIC_IGNORE_HOSTS',
(),
)
SAAS_WORDPRESS_LOG_PATH = Setting('SAAS_WORDPRESS_LOG_PATH',
'',
)
SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD',
'secret'
)
@ -132,6 +142,11 @@ SAAS_PHPLIST_CRONTAB = Setting('SAAS_PHPLIST_CRONTAB',
"Left blank if you don't want crontab to be configured")
)
SAAS_PHPLIST_MAIL_LOG_PATH = Setting('SAAS_PHPLIST_MAIL_LOG_PATH',
'/var/log/mail.log',
)
SAAS_SEAFILE_DOMAIN = Setting('SAAS_SEAFILE_DOMAIN',
'seafile.{}'.format(ORCHESTRA_BASE_DOMAIN),
help_text="Uses <tt>ORCHESTRA_BASE_DOMAIN</tt> by default.",