pangea
/
django-orchestra
7
0
Fork 0
Code Issues 1 Pull Requests 2 Packages Projects Releases Wiki Activity

Added dokuwiki traffic accountancy backend

This commit is contained in:
Marc Aymerich 2015-09-17 11:21:35 +00:00
parent 5b5d62ef38
commit 2491367d42
16 changed files with 197 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
orchestra/admin/menu.py
View File

@ -29,30 +29,33 @@ def api_link(context):
def process_registry(register):
def get_item(model, options, parent=False):
name = options.get('verbose_name_plural')
def get_item(model, options, name=None):
if name is None:
name = capfirst(options.get('verbose_name_plural'))
if isinstance(model, str):
url = reverse('admin:'+model)
else:
opts = model._meta
url = reverse('admin:{}_{}_changelist'.format(
opts.app_label, opts.model_name))
if parent:
name = opts.app_label
name = capfirst(name)
return items.MenuItem(name, url)
opts.app_label, opts.model_name)
)
item = items.MenuItem(name, url)
item.options = options
return item
childrens = {}
for model, options in register.get().items():
if options.get('menu', True):
parent = options.get('parent')
if parent:
name = capfirst(model._meta.app_label)
parent_item = childrens.get(parent)
if parent_item:
if not parent_item.children:
parent_item.children.append(deepcopy(parent_item))
parent_item.title = name
else:
parent_item = get_item(parent, register[parent], parent=True)
parent_item = get_item(parent, register[parent], name=name)
parent_item.children = []
parent_item.children.append(get_item(model, options))
childrens[parent] = parent_item

3
orchestra/contrib/bills/models.py
View File

@ -458,3 +458,6 @@ class BillSubline(models.Model):
description = models.CharField(_("description"), max_length=256)
total = models.DecimalField(max_digits=12, decimal_places=2)
type = models.CharField(_("type"), max_length=16, choices=TYPES, default=OTHER)
def __str__(self):
return "%s %i" % (self.description, self.total)

8
orchestra/contrib/bills/templates/bills/microspective.html
View File

@ -79,7 +79,7 @@
{% for line in lines %}
{% with sublines=line.sublines.all description=line.description|slice:"38:" %}
<span class="{% if not sublines and not description %}last {% endif %}column-id">{% if not line.order_id %}L{% endif %}{{ line.order_id|default:line.pk }}</span>
<span class="{% if not sublines and not description %}last {% endif %}column-description">{{ line.description|slice:":38" }}</span>
<span class="{% if not sublines and not description %}last {% endif %}column-description">{{ line.description|safe|slice:":38" }}</span>
<span class="{% if not sublines and not description %}last {% endif %}column-period">{{ line.get_verbose_period }}</span>
<span class="{% if not sublines and not description %}last {% endif %}column-quantity">{{ line.get_verbose_quantity|default:"&nbsp;"|safe }}</span>
<span class="{% if not sublines and not description %}last {% endif %}column-rate">{% if line.rate %}{{ line.rate }} &{{ currency.lower }};{% else %}&nbsp;{% endif %}</span>
@ -87,7 +87,7 @@
<br>
{% if description %}
<span class="{% if not sublines %}last {% endif %}subline column-id">&nbsp;</span>
<span class="{% if not sublines %}last {% endif %}subline column-description">{{ description|truncatechars:39 }}</span>
<span class="{% if not sublines %}last {% endif %}subline column-description">{{ description|safe|truncatechars:39 }}</span>
<span class="{% if not sublines %}last {% endif %}subline column-period">&nbsp;</span>
<span class="{% if not sublines %}last {% endif %}subline column-quantity">&nbsp;</span>
<span class="{% if not sublines %}last {% endif %}subline column-rate">&nbsp;</span>
@ -95,7 +95,7 @@
{% endif %}
{% for subline in sublines %}
<span class="{% if forloop.last %}last {% endif %}subline column-id">&nbsp;</span>
<span class="{% if forloop.last %}last {% endif %}subline column-description">{{ subline.description|truncatechars:39 }}</span>
<span class="{% if forloop.last %}last {% endif %}subline column-description">{{ subline.description|safe|truncatechars:39 }}</span>
<span class="{% if forloop.last %}last {% endif %}subline column-period">&nbsp;</span>
<span class="{% if forloop.last %}last {% endif %}subline column-quantity">&nbsp;</span>
<span class="{% if forloop.last %}last {% endif %}subline column-rate">&nbsp;</span>
@ -126,9 +126,11 @@
<div class="footer">
<div id="footer-column-1">
<div id="comments">
{% block comments %}
{% if bill.comments %}
<span class="title">{% trans "COMMENTS" %}</span> {{ bill.comments|linebreaksbr }}
{% endif %}
{% endblock %}
</div>
</div>
<div id="footer-column-2">

2
orchestra/contrib/lists/backends.py
View File

@ -302,7 +302,7 @@ class MailmanTraffic(ServiceMonitor):
# anonymized post
pass
except IOError as e:
sys.stderr.write(str(e))
sys.stderr.write(str(e)+'\\n')
for list_name, opts in lists.items():
__, object_id, size = opts

2
orchestra/contrib/mailboxes/backends.py
View File

@ -549,7 +549,7 @@ class PostfixMailscannerTraffic(ServiceMonitor):
except KeyError:
counter[req_id] = 1
except IOError as e:
sys.stderr.write(str(e))
sys.stderr.write(str(e)+'\\n')
for username, opts in users.iteritems():
size = 0

2
orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html
View File

@ -62,7 +62,7 @@ $(document).ready( function () {
{% for line in lines %}
<tr class="form-row {% if forloop.counter|divisibleby:2 %}row2{% else %}row1{% endif %}">
<td>
<a href="{{ line.order | admin_url }}">{{ line.order.description }}</a>
<a href="{{ line.order | admin_url }}">{{ line.order.description|safe }}</a>
{% for discount in line.discounts %}
<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Discount per {{ discount.type }}
{% endfor %}

1
orchestra/contrib/resources/apps.py
View File

@ -21,6 +21,7 @@ class ResourcesConfig(AppConfig):
administration.register(Resource, icon='gauge.png')
administration.register(ResourceData, parent=Resource, icon='monitor.png')
administration.register(MonitorData, parent=Resource, dashboard=False)
from . import signals
def reload_relations(self):
from .admin import insert_resource_inlines

7
orchestra/contrib/resources/models.py
View File

@ -116,17 +116,12 @@ class Resource(models.Model):
]})
def save(self, *args, **kwargs):
created = not self.pk
super(Resource, self).save(*args, **kwargs)
self.sync_periodic_task()
# This only works on tests (multiprocessing used on real deployments)
apps.get_app_config('resources').reload_relations()
def delete(self, *args, **kwargs):
super(Resource, self).delete(*args, **kwargs)
self.sync_periodic_task()
def sync_periodic_task(self):
""" sync periodic task on save/delete resource operations """
name = 'monitor.%s' % str(self)
if self.pk and self.crontab and self.is_active:
try:

12
orchestra/contrib/resources/signals.py Normal file
View File

@ -0,0 +1,12 @@
from django.db.models.signals import post_delete, post_save
from django.dispatch import receiver
from .models import Resource
@receiver(post_save, sender=Resource, dispatch_uid="resources.sync_periodic_task")
@receiver(post_delete, sender=Resource, dispatch_uid="resources.sync_periodic_task")
def sync_periodic_task(sender, **kwargs):
""" useing signals instead of Model.delete() override beucause of admin bulk delete() """
instance = kwargs['instance']
instance.sync_periodic_task()

107
orchestra/contrib/saas/backends/__init__.py
View File

@ -1,13 +1,110 @@
import pkgutil
import textwrap
from orchestra.contrib.resources import ServiceMonitor
from .. import settings
class SaaSServiceMixin(object):
model = 'saas.SaaS'
# TODO Match definition support on backends (mysql) and saas
class SaaSWebTraffic(ServiceMonitor):
"""
Parses apache logs,
looking for the size of each request on the last word of the log line.
def get_context(self, webapp):
# TODO
Compatible log format:
<tt>LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host</tt>
<tt>CustomLog /home/pangea/logs/apache/host_blog.pangea.org.log host</tt>
"""
model = 'saas.SaaS'
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
abstract = True
def prepare(self):
access_log = self.log_path
context = {
'access_logs': str((access_log, access_log+'.1')),
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS),
}
self.append(textwrap.dedent("""\
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
access_logs = {access_logs}
sites = {{}}
months = {{
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}}
def prepare(object_id, site_domain, ini_date):
global sites
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
sites[site_domain] = [ini_date, object_id, 0]
def monitor(sites, end_date, months, access_logs):
for access_log in access_logs:
try:
with open(access_log, 'r') as handler:
for line in handler.readlines():
meta, request, response, hostname, __ = line.split('"')
host, __, __, date, tz = meta.split()
if host in {ignore_hosts}:
continue
try:
site = sites[hostname]
except KeyError:
continue
else:
# [16/Sep/2015:11:40:38 +0200]
day, month, date = date[1:].split('/')
year, hour, min, sec = date.split(':')
date = year + months[month] + day + hour + min + sec
if site[0] < int(date) < end_date:
status, size = response.split()
site[2] += int(size)
except IOError as e:
sys.stderr.write(str(e)+'\\n')
for opts in sites.values():
ini_date, object_id, size = opts
print object_id, size
""").format(**context)
)
def monitor(self, saas):
context = self.get_context(saas)
self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context)
def commit(self):
self.append('monitor(sites, end_date, months, access_logs)')
def get_context(self, saas):
return {
'site_domain': saas.get_site_domain(),
'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': saas.pk,
}

11
orchestra/contrib/saas/backends/dokuwikimu.py
View File

@ -7,6 +7,7 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController
from orchestra.utils.python import random_ascii
from . import SaaSWebTraffic
from .. import settings
@ -64,3 +65,13 @@ class DokuWikiMuBackend(ServiceController):
'users_path': os.path.join(context['app_path'], 'conf/users.auth.php'),
})
return context
class DokuWikiMuTraffic(SaaSWebTraffic):
__doc__ = SaaSWebTraffic.__doc__
verbose_name = _("DokuWiki MU Traffic")
default_route_match = "saas.service == 'dokuwiki'"
doc_settings = (settings,
('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_DOKUWIKI_LOG_PATH')
)
log_path = settings.SAAS_DOKUWIKI_LOG_PATH

2
orchestra/contrib/saas/backends/phplist.py
View File

@ -213,7 +213,7 @@ class PhpListTraffic(ServiceMonitor):
size = int(size[5:-1])
opts[2] += size
except IOError as e:
sys.stderr.write(str(e))
sys.stderr.write(str(e)+'\\n')
for opts in lists.values():
print opts[1], opts[2]
""").format(**context)

101
orchestra/contrib/saas/backends/wordpressmu.py
View File

@ -1,12 +1,11 @@
import re
import textwrap
import requests
from django.utils.translation import ugettext_lazy as _
from orchestra.contrib.orchestration import ServiceController
from orchestra.contrib.resources import ServiceMonitor
from . import SaaSWebTraffic
from .. import settings
@ -123,103 +122,11 @@ class WordpressMuBackend(ServiceController):
self.append(self.delete_blog, saas)
class WordpressMuTraffic(ServiceMonitor):
"""
Parses apache logs,
looking for the size of each request on the last word of the log line.
"""
class WordpressMuTraffic(SaaSWebTraffic):
__doc__ = SaaSWebTraffic.__doc__
verbose_name = _("Wordpress MU Traffic")
model = 'saas.SaaS'
default_route_match = "saas.service == 'wordpress'"
script_executable = '/usr/bin/python'
monthly_sum_old_values = True
doc_settings = (settings,
('SAAS_TRAFFIC_IGNORE_HOSTS', 'SAAS_WORDPRESS_LOG_PATH')
)
def prepare(self):
access_log = settings.SAAS_WORDPRESS_LOG_PATH
context = {
'access_logs': str((access_log, access_log+'.1')),
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'ignore_hosts': str(settings.SAAS_TRAFFIC_IGNORE_HOSTS),
}
self.append(textwrap.dedent("""\
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
access_logs = {access_logs}
blogs = {{}}
months = {{
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}}
def prepare(object_id, site_domain, ini_date):
global blogs
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
blogs[site_domain] = [ini_date, object_id, 0]
def monitor(blogs, end_date, months, access_logs):
for access_log in access_logs:
try:
with open(access_log, 'r') as handler:
for line in handler.readlines():
meta, request, response, hostname, __ = line.split('"')
host, __, __, date, tz = meta.split()
if host in {ignore_hosts}:
continue
try:
blog = blogs[hostname]
except KeyError:
continue
else:
# [16/Sep/2015:11:40:38 +0200]
day, month, date = date[1:].split('/')
year, hour, min, sec = date.split(':')
date = year + months[month] + day + hour + min + sec
if blog[0] < int(date) < end_date:
status, size = response.split()
blog[2] += int(size)
except IOError as e:
sys.stderr.write(str(e))
for opts in blogs.values():
ini_date, object_id, size = opts
print object_id, size
""").format(**context)
)
def monitor(self, saas):
context = self.get_context(saas)
self.append("prepare(%(object_id)s, '%(site_domain)s', '%(last_date)s')" % context)
def commit(self):
self.append('monitor(blogs, end_date, months, access_logs)')
def get_context(self, saas):
return {
'site_domain': saas.get_site_domain(),
'last_date': self.get_last_date(saas.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': saas.pk,
}
log_path = settings.SAAS_WORDPRESS_LOG_PATH

8
orchestra/contrib/saas/settings.py
View File

@ -25,11 +25,14 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES',
SAAS_TRAFFIC_IGNORE_HOSTS = Setting('SAAS_TRAFFIC_IGNORE_HOSTS',
(),
help_text=_("IP addresses to ignore during traffic accountability."),
)
SAAS_WORDPRESS_LOG_PATH = Setting('SAAS_WORDPRESS_LOG_PATH',
'',
help_text=_('Filesystem path for the webserver access logs.<br>'
'<tt>LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Host}i\"" host</tt>'),
)
@ -83,6 +86,11 @@ SAAS_DOKUWIKI_GROUP = Setting('SAAS_DOKUWIKI_GROUP',
)
SAAS_DOKUWIKI_LOG_PATH = Setting('SAAS_DOKUWIKI_LOG_PATH',
'',
)
SAAS_DRUPAL_SITES_PATH = Setting('WEBSITES_DRUPAL_SITES_PATH',
'/home/httpd/htdocs/drupal-mu/sites/%(site_name)s',
)

34
orchestra/contrib/services/migrations/0003_auto_20150917_0942.py Normal file
View File

@ -0,0 +1,34 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('services', '0002_auto_20150509_1501'),
]
operations = [
migrations.AlterField(
model_name='service',
name='billing_point',
field=models.CharField(choices=[('ON_REGISTER', 'Registration date'), ('ON_FIXED_DATE', 'Every April')], help_text='Reference point for calculating the renewal date on recurring invoices', verbose_name='billing point', max_length=16, default='ON_FIXED_DATE'),
),
migrations.AlterField(
model_name='service',
name='is_fee',
field=models.BooleanField(help_text='Designates whether this service should be billed as membership fee or not', verbose_name='fee', default=False),
),
migrations.AlterField(
model_name='service',
name='order_description',
field=models.CharField(help_text="Python <a href='https://docs.python.org/2/library/functions.html#eval'>expression</a> used for generating the description for the bill lines of this services.<br>Defaults to <tt>'%s: %s' % (ugettext(handler.description), instance)</tt>", blank=True, max_length=256, verbose_name='Order description'),
),
migrations.AlterField(
model_name='service',
name='rate_algorithm',
field=models.CharField(choices=[('orchestra.contrib.plans.ratings.step_price', 'Step price'), ('orchestra.contrib.plans.ratings.best_price', 'Best price'), ('orchestra.contrib.plans.ratings.match_price', 'Match price')], help_text='Algorithm used to interprete the rating table.<br>&nbsp;&nbsp;Step price: All rates with a quantity lower or equal than the metric are applied. Nominal price will be used when initial block is missing.<br>&nbsp;&nbsp;Best price: Produces the best possible price given all active rating lines (those with quantity lower or equal to the metric).<br>&nbsp;&nbsp;Match price: Only <b>the rate</b> with a) inmediate inferior metric and b) lower price is applied. Nominal price will be used when initial block is missing.', verbose_name='rate algorithm', max_length=64, default='orchestra.contrib.plans.ratings.step_price'),
),
]

2
orchestra/contrib/services/models.py
View File

@ -97,7 +97,7 @@ class Service(models.Model):
default=FIXED_DATE)
is_fee = models.BooleanField(_("fee"), default=False,
help_text=_("Designates whether this service should be billed as membership fee or not"))
order_description = models.CharField(_("Order description"), max_length=128, blank=True,
order_description = models.CharField(_("Order description"), max_length=256, blank=True,
help_text=_(
"Python <a href='https://docs.python.org/2/library/functions.html#eval'>expression</a> "
"used for generating the description for the bill lines of this services.<br>"