Reimplemented traffic monitors, order of magnitude faster

This commit is contained in:
Marc Aymerich 2015-03-20 15:13:08 +00:00
parent 7711988a63
commit 2d3e925c36
16 changed files with 543 additions and 38 deletions

20
TODO.md
View File

@ -215,3 +215,23 @@ Php binaries should have this format: /usr/bin/php5.2-cgi
* show details data on webapp changelist * show details data on webapp changelist
* lock resource monitoring
* Optimize backends like mail backend (log files single read), single "/var/log/vsftpd.log{,.1}" on ftp traffic
* -EXecCGI in common CMS upload locations /wp-upload/upload/uploads
* cgi user / pervent shell access
* merge php wrapper configuration to optimize process classes
* prevent stderr when users exists on backend i.e. mysql user create
* disable anonymized list options (mailman)
* webapps directory protection and disable excecgi
* php-fpm disable execCGI
* SuexecUserGroup needs to be per app othewise wrapper/fpm user can't be correct

View File

@ -155,22 +155,25 @@ class MailmanBackend(ServiceController):
return context return context
class MailmanTraffic(ServiceMonitor): class MailmanTrafficBash(ServiceMonitor):
model = 'lists.List' model = 'lists.List'
resource = ServiceMonitor.TRAFFIC resource = ServiceMonitor.TRAFFIC
verbose_name = _("Mailman traffic") verbose_name = _("Mailman traffic (Bash)")
def prepare(self): def prepare(self):
super(MailmanTraffic, self).prepare() super(MailmanTraffic, self).prepare()
current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") context = {
'mailman_log': '%s{,.1}' % settings.LISTS_MAILMAN_POST_LOG_PATH,
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
}
self.append(textwrap.dedent("""\ self.append(textwrap.dedent("""\
function monitor () { function monitor () {
OBJECT_ID=$1 OBJECT_ID=$1
# Dates convertions are done server-side because of timezone discrepancies # Dates convertions are done server-side because of timezone discrepancies
INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2")
END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s') END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s')
LIST_NAME="$3" LIST_NAME="$3"
MAILMAN_LOG="$4" MAILMAN_LOG=%(mailman_log)s
SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l) SUBSCRIBERS=$(list_members ${LIST_NAME} | wc -l)
{ {
@ -203,17 +206,115 @@ class MailmanTraffic(ServiceMonitor):
print sum * subs print sum * subs
}' || [[ $? == 1 ]] && true }' || [[ $? == 1 ]] && true
} | xargs echo ${OBJECT_ID} } | xargs echo ${OBJECT_ID}
}""") % current_date) }""") % context)
def monitor(self, mail_list): def monitor(self, mail_list):
context = self.get_context(mail_list) context = self.get_context(mail_list)
self.append( self.append(
'monitor %(object_id)i "%(last_date)s" "%(list_name)s" %(mailman_log)s{,.1}' % context 'monitor %(object_id)i "%(last_date)s" "%(list_name)s"' % context
) )
def get_context(self, mail_list): def get_context(self, mail_list):
return { return {
'mailman_log': settings.LISTS_MAILMAN_POST_LOG_PATH, 'list_name': mail_list.name,
'object_id': mail_list.pk,
'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
class MailmanTraffic(ServiceMonitor):
model = 'lists.List'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Mailman traffic")
script_executable = '/usr/bin/python'
def prepare(self):
postlog = settings.LISTS_MAILMAN_POST_LOG_PATH
context = {
'postlogs': str((postlog, postlog+'.1')),
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
}
self.append(textwrap.dedent("""\
import re
import subprocess
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
postlogs = {postlogs}
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
lists = {{}}
months = {{
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}}
def prepare(object_id, list_name, ini_date):
global lists
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
lists[list_name] = [ini_date, object_id, 0]
def monitor(lists, end_date, months, postlogs):
for postlog in postlogs:
try:
with open(postlog, 'r') as postlog:
for line in postlog.readlines():
month, day, time, year, __, __, __, list_name, __, __, size = line.split()[:11]
try:
list = lists[list_name]
except KeyError:
continue
else:
date = year + months[month] + day + time.replace(':', '')
if list[0] < int(date) < end_date:
size = size[5:-1]
try:
list[2] += int(size)
except ValueError:
# anonymized post
pass
except IOError as e:
sys.stderr.write(e)
for list_name, opts in lists.iteritems():
__, object_id, size = opts
if size:
cmd = ' '.join(('list_members', list_name, '| wc -l'))
ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
subscribers = ps.communicate()[0].strip()
size *= int(subscribers)
print object_id, size
""").format(**context)
)
def monitor(self, user):
context = self.get_context(user)
self.append("prepare(%(object_id)s, '%(list_name)s', '%(last_date)s')" % context)
def commit(self):
self.append('monitor(lists, end_date, months, postlogs)')
def get_context(self, mail_list):
return {
'list_name': mail_list.name, 'list_name': mail_list.name,
'object_id': mail_list.pk, 'object_id': mail_list.pk,
'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),

View File

@ -9,7 +9,7 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController from orchestra.apps.orchestration import ServiceController
from orchestra.apps.systemusers.backends import SystemUserBackend from orchestra.apps.systemusers.backends import SystemUserBackend
from orchestra.apps.resources import ServiceMonitor from orchestra.apps.resources import ServiceMonitor
from orchestra.utils.humanize import unit_to_bytes #from orchestra.utils.humanize import unit_to_bytes
from . import settings from . import settings
from .models import Address from .models import Address
@ -42,7 +42,8 @@ class MailSystemUserBackend(ServiceController):
self.set_quota(mailbox, context) self.set_quota(mailbox, context)
def set_quota(self, mailbox, context): def set_quota(self, mailbox, context):
context['quota'] = mailbox.resources.disk.allocated * unit_to_bytes(mailbox.resources.disk.unit) context['quota'] = mailbox.resources.disk.allocated * mailbox.resources.disk.resource.get_scale()
#unit_to_bytes(mailbox.resources.disk.unit)
self.append(textwrap.dedent(""" self.append(textwrap.dedent("""
mkdir -p %(home)s/Maildir mkdir -p %(home)s/Maildir
chown %(user)s:%(group)s %(home)s/Maildir chown %(user)s:%(group)s %(home)s/Maildir
@ -294,3 +295,166 @@ class MaildirDisk(ServiceMonitor):
} }
context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context
return context return context
class PostfixTraffic(ServiceMonitor):
"""
A high-performance log parser
Reads the mail.log file only once, for all users
"""
model = 'mailboxes.Mailbox'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Postfix traffic usage")
script_executable = '/usr/bin/python'
def prepare(self):
mail_log = '/var/log/mail.log'
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'mail_logs': str((mail_log, mail_log+'.1')),
}
self.append(textwrap.dedent("""\
import re
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
# Converts orchestra's UTC dates to local timezone
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
maillogs = {mail_logs}
end_datetime = to_local_timezone('{current_date}')
end_date = int(end_datetime.strftime('%Y%m%d%H%M%S'))
months = {{
"Jan": "01",
"Feb": "02",
"Mar": "03",
"Apr": "04",
"May": "05",
"Jun": "06",
"Jul": "07",
"Aug": "08",
"Sep": "09",
"Oct": "10",
"Nov": "11",
"Dec": "12",
}}
def inside_period(month, day, time, ini_date):
global months
global end_datetime
# Mar 19 17:13:22
month = months[month]
year = end_datetime.year
if month == '12' and end_datetime.month == 1:
year = year+1
date = str(year) + month + day
date += time.replace(':', '')
return ini_date < int(date) < end_date
users = {{}}
delivers = {{}}
reverse = {{}}
def prepare(object_id, mailbox, ini_date):
global users
global delivers
global reverse
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
users[mailbox] = (ini_date, object_id)
delivers[mailbox] = set()
reverse[mailbox] = set()
def monitor(users, delivers, reverse, maillogs):
targets = {{}}
counter = {{}}
user_regex = re.compile(r'\(Authenticated sender: ([^ ]+)\)')
for maillog in maillogs:
try:
with open(maillog, 'r') as maillog:
for line in maillog.readlines():
# Only search for Authenticated sendings
if '(Authenticated sender: ' in line:
username = user_regex.search(line).groups()[0]
try:
sender = users[username]
except KeyError:
continue
else:
month, day, time, __, proc, id = line.split()[:6]
if inside_period(month, day, time, sender[0]):
# Add new email
delivers[id[:-1]] = username
# Look for a MailScanner requeue ID
elif ' Requeue: ' in line:
id, __, req_id = line.split()[6:9]
id = id.split('.')[0]
try:
username = delivers[id]
except KeyError:
pass
else:
targets[req_id] = (username, None)
reverse[username].add(req_id)
# Look for the mail size and count the number of recipients of each email
else:
try:
month, day, time, __, proc, req_id, __, msize = line.split()[:8]
except ValueError:
# not interested in this line
continue
if proc.startswith('postfix/'):
req_id = req_id[:-1]
if msize.startswith('size='):
try:
target = targets[req_id]
except KeyError:
pass
else:
targets[req_id] = (target[0], int(msize[5:-1]))
elif proc.startswith('postfix/smtp'):
try:
target = targets[req_id]
except KeyError:
pass
else:
if inside_period(month, day, time, users[target[0]][0]):
try:
counter[req_id] += 1
except KeyError:
counter[req_id] = 1
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
size = 0
for req_id in reverse[username]:
size += targets[req_id][1] * counter.get(req_id, 0)
print opts[1], size
""").format(**context)
)
def commit(self):
self.append('monitor(users, delivers, reverse, maillogs)')
def monitor(self, mailbox):
context = self.get_context(mailbox)
self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context)
def get_context(self, mailbox):
return {
# 'mainlog': settings.LISTS_MAILMAN_POST_LOG_PATH,
'mailbox': mailbox.name,
'object_id': mailbox.pk,
'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}

View File

@ -88,9 +88,15 @@ def SSH(backend, log, server, cmds, async=False):
# Non-blocking is the secret ingridient in the async sauce # Non-blocking is the secret ingridient in the async sauce
select.select([channel], [], []) select.select([channel], [], [])
if channel.recv_ready(): if channel.recv_ready():
log.stdout += channel.recv(1024) part = channel.recv(1024)
while part:
log.stdout += part
part = channel.recv(1024)
if channel.recv_stderr_ready(): if channel.recv_stderr_ready():
log.stderr += channel.recv_stderr(1024) part = channel.recv_stderr(1024)
while part:
log.stderr += part
part = channel.recv_stderr(1024)
log.save(update_fields=['stdout', 'stderr']) log.save(update_fields=['stdout', 'stderr'])
if channel.exit_status_ready(): if channel.exit_status_ready():
break break

View File

@ -16,7 +16,7 @@ def run_monitor(modeladmin, request, queryset):
if not async: if not async:
for result in results: for result in results:
if hasattr(result, 'log'): if hasattr(result, 'log'):
logs.add(result.log.pk) logs.add(str(result.log.pk))
modeladmin.log_change(request, resource, _("Run monitors")) modeladmin.log_change(request, resource, _("Run monitors"))
if async: if async:
num = len(queryset) num = len(queryset)
@ -28,8 +28,8 @@ def run_monitor(modeladmin, request, queryset):
else: else:
num = len(logs) num = len(logs)
if num == 1: if num == 1:
log = logs.pop() log_pk = int(logs.pop())
link = reverse('admin:orchestration_backendlog_change', args=(log,)) link = reverse('admin:orchestration_backendlog_change', args=(log_pk,))
msg = _("One related monitor has <a href='%s'>been executed</a>.") % link msg = _("One related monitor has <a href='%s'>been executed</a>.") % link
elif num >= 1: elif num >= 1:
link = reverse('admin:orchestration_backendlog_changelist') link = reverse('admin:orchestration_backendlog_changelist')

View File

@ -179,25 +179,37 @@ admin.site.register(MonitorData, MonitorDataAdmin)
def resource_inline_factory(resources): def resource_inline_factory(resources):
class ResourceInlineFormSet(generic.BaseGenericInlineFormSet): class ResourceInlineFormSet(generic.BaseGenericInlineFormSet):
def total_form_count(self, resources=resources): def total_form_count(self, resources=resources):
return len(resources) return len(resources)
def get_queryset(self):
queryset = super(ResourceInlineFormSet, self).get_queryset()
return queryset.order_by('-id').filter(resource__is_active=True)
@cached_property @cached_property
def forms(self, resources=resources): def forms(self, resources=resources):
forms = [] forms = []
resources_copy = list(resources) resources_copy = list(resources)
queryset = self.queryset # Remove queryset disabled objects
queryset = [data for data in self.queryset if data.resource in resources]
if self.instance.pk: if self.instance.pk:
# Create missing resource data # Create missing resource data
queryset = list(queryset)
queryset_resources = [data.resource for data in queryset] queryset_resources = [data.resource for data in queryset]
for resource in resources: for resource in resources:
if resource not in queryset_resources: if resource not in queryset_resources:
data = resource.dataset.create(content_object=self.instance) kwargs = {
'content_object': self.instance,
}
if resource.default_allocation:
kwargs['allocated'] = resource.default_allocation
data = resource.dataset.create(**kwargs)
queryset.append(data) queryset.append(data)
# Existing dataset # Existing dataset
for i, data in enumerate(queryset): for i, data in enumerate(queryset):
forms.append(self._construct_form(i, resource=data.resource)) forms.append(self._construct_form(i, resource=data.resource))
resources_copy.remove(data.resource) try:
resources_copy.remove(data.resource)
except ValueError:
pass
# Missing dataset # Missing dataset
for i, resource in enumerate(resources_copy, len(queryset)): for i, resource in enumerate(resources_copy, len(queryset)):
forms.append(self._construct_form(i, resource=resource)) forms.append(self._construct_form(i, resource=resource))
@ -246,8 +258,8 @@ def insert_resource_inlines():
for inline in getattr(modeladmin_class, 'inlines', []): for inline in getattr(modeladmin_class, 'inlines', []):
if inline.__name__ == 'ResourceInline': if inline.__name__ == 'ResourceInline':
modeladmin_class.inlines.remove(inline) modeladmin_class.inlines.remove(inline)
resources = Resource.objects.filter(is_active=True)
for ct, resources in Resource.objects.group_by('content_type').iteritems(): for ct, resources in resources.group_by('content_type').iteritems():
inline = resource_inline_factory(resources) inline = resource_inline_factory(resources)
model = ct.model_class() model = ct.model_class()
insertattr(model, 'inlines', inline) insertattr(model, 'inlines', inline)

View File

@ -64,7 +64,11 @@ class ServiceMonitor(ServiceBackend):
ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower()) ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower())
for line in log.stdout.splitlines(): for line in log.stdout.splitlines():
line = line.strip() line = line.strip()
object_id, value = self.process(line) try:
object_id, value = self.process(line)
except ValueError:
cls_name = self.__class__.__name__
raise ValueError("%s expected '<id> <value>' got '%s'" % (cls_name, line))
MonitorData.objects.create(monitor=name, object_id=object_id, MonitorData.objects.create(monitor=name, object_id=object_id,
content_type=ct, value=value, created_at=self.current_date) content_type=ct, value=value, created_at=self.current_date)

View File

@ -25,7 +25,6 @@ class ResourceForm(forms.ModelForm):
else: else:
self.fields['allocated'].required = True self.fields['allocated'].required = True
self.fields['allocated'].initial = self.resource.default_allocation self.fields['allocated'].initial = self.resource.default_allocation
# def has_changed(self): # def has_changed(self):
# """ Make sure resourcedata objects are created for all resources """ # """ Make sure resourcedata objects are created for all resources """
# if not self.instance.pk: # if not self.instance.pk:

View File

@ -86,6 +86,10 @@ class Resource(models.Model):
def clean(self): def clean(self):
self.verbose_name = self.verbose_name.strip() self.verbose_name = self.verbose_name.strip()
if self.on_demand and self.default_allocation:
raise validators.ValidationError({
'default_allocation': _("Default allocation can not be set for 'on demand' services")
})
# Validate that model path exists between ct and each monitor.model # Validate that model path exists between ct and each monitor.model
monitor_errors = [] monitor_errors = []
for monitor in self.monitors: for monitor in self.monitors:
@ -172,6 +176,9 @@ class ResourceData(models.Model):
unique_together = ('resource', 'content_type', 'object_id') unique_together = ('resource', 'content_type', 'object_id')
verbose_name_plural = _("resource data") verbose_name_plural = _("resource data")
def __unicode__(self):
return "%s: %s" % (str(self.resource), str(self.content_object))
@classmethod @classmethod
def get_or_create(cls, obj, resource): def get_or_create(cls, obj, resource):
ct = ContentType.objects.get_for_model(type(obj)) ct = ContentType.objects.get_for_model(type(obj))

View File

@ -99,21 +99,24 @@ class SystemUserDisk(ServiceMonitor):
} }
class FTPTraffic(ServiceMonitor): class FTPTrafficBash(ServiceMonitor):
model = 'systemusers.SystemUser' model = 'systemusers.SystemUser'
resource = ServiceMonitor.TRAFFIC resource = ServiceMonitor.TRAFFIC
verbose_name = _('Systemuser FTP traffic') verbose_name = _('Systemuser FTP traffic (Bash)')
def prepare(self): def prepare(self):
super(FTPTraffic, self).prepare() super(FTPTrafficBash, self).prepare()
current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") context = {
'log_file': '%s{,.1}' % settings.SYSTEMUSERS_FTP_LOG_PATH,
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
}
self.append(textwrap.dedent("""\ self.append(textwrap.dedent("""\
function monitor () { function monitor () {
OBJECT_ID=$1 OBJECT_ID=$1
INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2")
END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s') END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s')
USERNAME="$3" USERNAME="$3"
LOG_FILE="$4" LOG_FILE=%(log_file)s
{ {
grep " bytes, " ${LOG_FILE} \\ grep " bytes, " ${LOG_FILE} \\
| grep " \\[${USERNAME}\\] " \\ | grep " \\[${USERNAME}\\] " \\
@ -145,18 +148,191 @@ class FTPTraffic(ServiceMonitor):
print sum print sum
}' || [[ $? == 1 ]] && true }' || [[ $? == 1 ]] && true
} | xargs echo ${OBJECT_ID} } | xargs echo ${OBJECT_ID}
}""") % current_date) }""") % context)
def monitor(self, user): def monitor(self, user):
context = self.get_context(user) context = self.get_context(user)
self.append( self.append(
'monitor {object_id} "{last_date}" "{username}" {log_file}'.format(**context) 'monitor {object_id} "{last_date}" "{username}"'.format(**context)
) )
def get_context(self, user): def get_context(self, user):
return { return {
'log_file': '%s{,.1}' % settings.SYSTEMUSERS_FTP_LOG_PATH,
'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': user.pk, 'object_id': user.pk,
'username': user.username, 'username': user.username,
} }
class Exim4Traffic(ServiceMonitor):
model = 'systemusers.SystemUser'
resource = ServiceMonitor.TRAFFIC
verbose_name = _("Exim4 traffic usage")
script_executable = '/usr/bin/python'
def prepare(self):
mainlog = '/var/log/exim4/mainlog'
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'mainlogs': str((mainlog, mainlog+'.1')),
}
self.append(textwrap.dedent("""\
import re
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
mainlogs = {mainlogs}
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
users = {{}}
def prepare(object_id, username, ini_date):
global users
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
users[username] = [ini_date, object_id, 0]
def monitor(users, end_date, mainlogs):
user_regex = re.compile(r' U=([^ ]+) ')
for mainlog in mainlogs:
try:
with open(mainlog, 'r') as mainlog:
for line in mainlog.readlines():
if ' <= ' in line and 'P=local' in line:
username = user_regex.search(line).groups()[0]
try:
sender = users[username]
except KeyError:
continue
else:
date, time, id, __, __, user, protocol, size = line.split()[:8]
date = date.replace('-', '')
date += time.replace(':', '')
if sender[0] < int(date) < end_date:
sender[2] += int(size[2:])
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
__, object_id, size = opts
print object_id, size
""").format(**context)
)
def commit(self):
self.append('monitor(users, end_date, mainlogs)')
def monitor(self, user):
context = self.get_context(user)
self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context)
def get_context(self, user):
return {
# 'mainlog': settings.LISTS_MAILMAN_POST_LOG_PATH,
'username': user.username,
'object_id': user.pk,
'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
}
class FTPTraffic(ServiceMonitor):
model = 'systemusers.SystemUser'
resource = ServiceMonitor.TRAFFIC
verbose_name = _('Systemuser FTP traffic')
script_executable = '/usr/bin/python'
def prepare(self):
vsftplog = settings.SYSTEMUSERS_FTP_LOG_PATH
context = {
'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"),
'vsftplogs': str((vsftplog, vsftplog+'.1')),
}
self.append(textwrap.dedent("""\
import re
import sys
from datetime import datetime
from dateutil import tz
def to_local_timezone(date, tzlocal=tz.tzlocal()):
date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z')
date = date.replace(tzinfo=tz.tzutc())
date = date.astimezone(tzlocal)
return date
vsftplogs = {vsftplogs}
# Use local timezone
end_date = to_local_timezone('{current_date}')
end_date = int(end_date.strftime('%Y%m%d%H%M%S'))
users = {{}}
months = {{
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12',
}}
def prepare(object_id, username, ini_date):
global users
ini_date = to_local_timezone(ini_date)
ini_date = int(ini_date.strftime('%Y%m%d%H%M%S'))
users[username] = [ini_date, object_id, 0]
def monitor(users, end_date, months, vsftplogs):
user_regex = re.compile(r'\] \[([^ ]+)\] OK ')
bytes_regex = re.compile(r', ([0-9]+) bytes, ')
for vsftplog in vsftplogs:
try:
with open(vsftplog, 'r') as vsftplog:
for line in vsftplog.readlines():
if ' bytes, ' in line:
username = user_regex.search(line).groups()[0]
try:
user = users[username]
except KeyError:
continue
else:
__, month, day, time, year = line.split()[:5]
date = year + months[month] + day + time.replace(':', '')
if user[0] < int(date) < end_date:
bytes = bytes_regex.search(line).groups()[0]
user[2] += int(bytes)
except IOError as e:
sys.stderr.write(e)
for username, opts in users.iteritems():
__, object_id, size = opts
print object_id, size
""").format(**context)
)
def monitor(self, user):
context = self.get_context(user)
self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context)
def commit(self):
self.append('monitor(users, end_date, months, vsftplogs)')
def get_context(self, user):
return {
'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': user.pk,
'username': user.username,
}

View File

@ -46,7 +46,7 @@ class WebAppOptionInline(admin.TabularInline):
class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
list_display = ('name', 'type', 'display_websites', 'account_link') list_display = ('name', 'type', 'display_detail', 'display_websites', 'account_link')
list_filter = ('type',) list_filter = ('type',)
# add_fields = ('account', 'name', 'type') # add_fields = ('account', 'name', 'type')
# fields = ('account_link', 'name', 'type') # fields = ('account_link', 'name', 'type')
@ -80,6 +80,10 @@ class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin)
display_websites.short_description = _("web sites") display_websites.short_description = _("web sites")
display_websites.allow_tags = True display_websites.allow_tags = True
def display_detail(self, webapp):
return webapp.type_instance.get_detail()
display_detail.short_description = _("detail")
# def formfield_for_dbfield(self, db_field, **kwargs): # def formfield_for_dbfield(self, db_field, **kwargs):
# """ Make value input widget bigger """ # """ Make value input widget bigger """
# if db_field.name == 'type': # if db_field.name == 'type':

View File

@ -28,6 +28,12 @@ WEBAPPS_PHP_ERROR_LOG_PATH = getattr(settings, 'WEBAPPS_PHP_ERROR_LOG_PATH',
'') '')
WEBAPPS_MERGE_PHP_WEBAPPS = getattr(settings, 'WEBAPPS_MERGE_PHP_WEBAPPS',
# Combine all fcgid-wrappers/fpm-pools into one per account-php_version
# to better control num processes per account and save memory
False)
WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', ( WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', (
'orchestra.apps.webapps.types.php.PHPApp', 'orchestra.apps.webapps.types.php.PHPApp',
'orchestra.apps.webapps.types.misc.StaticApp', 'orchestra.apps.webapps.types.misc.StaticApp',

View File

@ -100,6 +100,9 @@ class AppType(plugins.Plugin):
else: else:
yield (group, [(op.name, op.verbose_name) for op in options]) yield (group, [(op.name, op.verbose_name) for op in options])
def get_detail(self):
return ''
def save(self): def save(self):
pass pass

View File

@ -9,7 +9,7 @@ from orchestra.plugins.forms import PluginDataForm
from ..options import AppOption from ..options import AppOption
from . import AppType from . import AppType
from .php import PHPApp from .php import PHPApp, PHPAppForm, PHPAppSerializer
class StaticApp(AppType): class StaticApp(AppType):
@ -39,12 +39,12 @@ class WebalizerApp(AppType):
return ('static', webalizer_path) return ('static', webalizer_path)
class SymbolicLinkForm(PluginDataForm): class SymbolicLinkForm(PHPAppForm):
path = forms.CharField(label=_("Path"), widget=forms.TextInput(attrs={'size':'100'}), path = forms.CharField(label=_("Path"), widget=forms.TextInput(attrs={'size':'100'}),
help_text=_("Path for the origin of the symbolic link.")) help_text=_("Path for the origin of the symbolic link."))
class SymbolicLinkSerializer(serializers.Serializer): class SymbolicLinkSerializer(PHPAppSerializer):
path = serializers.CharField(label=_("Path")) path = serializers.CharField(label=_("Path"))

View File

@ -54,6 +54,9 @@ class PHPApp(AppType):
def is_fcgid(self): def is_fcgid(self):
return self.get_php_version().endswith('-cgi') return self.get_php_version().endswith('-cgi')
def get_detail(self):
return self.instance.data.get('php_version', '')
def get_context(self): def get_context(self):
""" context used to format settings """ """ context used to format settings """
return { return {

View File

@ -343,7 +343,7 @@ class Apache2Traffic(ServiceMonitor):
def get_context(self, site): def get_context(self, site):
return { return {
'log_file': '%s{,.1}' % site.get_www_log_path(), 'log_file': '%s{,.1}' % site.get_www_access_log_path(),
'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"),
'object_id': site.pk, 'object_id': site.pk,
} }