From 917b0b9329e21f6b3812d1b49a4706672346bf9d Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Thu, 20 Nov 2014 15:34:59 +0000 Subject: [PATCH] Refactored SaaS application --- TODO.md | 23 ++++++ orchestra/apps/contacts/admin.py | 1 + orchestra/apps/lists/admin.py | 1 + orchestra/apps/mailboxes/admin.py | 2 +- orchestra/apps/resources/admin.py | 1 + orchestra/apps/saas/admin.py | 21 +++++- orchestra/apps/saas/models.py | 30 ++++---- orchestra/apps/saas/services/bscw.py | 41 +++-------- orchestra/apps/saas/services/dokuwiki.py | 14 ---- orchestra/apps/saas/services/drupal.py | 14 ---- orchestra/apps/saas/services/gitlab.py | 14 ---- orchestra/apps/saas/services/options.py | 84 ++++++++++++++++++---- orchestra/apps/saas/services/phplist.py | 9 +-- orchestra/apps/saas/services/wordpress.py | 29 +++----- orchestra/apps/systemusers/backends.py | 69 +++++++++--------- orchestra/apps/systemusers/models.py | 7 +- orchestra/apps/websites/backends/apache.py | 74 ++++++++++--------- orchestra/apps/websites/settings.py | 4 ++ orchestra/conf/base_settings.py | 4 +- orchestra/core/validators.py | 4 ++ 20 files changed, 243 insertions(+), 203 deletions(-) diff --git a/TODO.md b/TODO.md index 1e89fe86..cfb8ea22 100644 --- a/TODO.md +++ b/TODO.md @@ -181,3 +181,26 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * Move plugins back from apps to orchestra main app * BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when) + + +* rename admin prefetch_related to list_prefetch_related for consistency + + +* LAST resource monitor option -> SUM(last backend) + +* Resource.monitor(async=True) admin action + +* Validate a model path exists between resource.content_type and backend.model + +* Add support for whitelisted IPs on traffic monitoring ['127.0.0.1',] + + +* Periodic task for cleaning old monitoring data + +* Generate reports of Account contracted services + +* Create an admin service_view with icons (like SaaS app) + +* Fix ftp traffic + +* Resource graph for each related object diff --git a/orchestra/apps/contacts/admin.py b/orchestra/apps/contacts/admin.py index 04aa387f..eb1d2f9b 100644 --- a/orchestra/apps/contacts/admin.py +++ b/orchestra/apps/contacts/admin.py @@ -14,6 +14,7 @@ class ContactAdmin(AccountAdminMixin, admin.ModelAdmin): list_display = ( 'dispaly_name', 'email', 'phone', 'phone2', 'country', 'account_link' ) + # TODO email usage custom filter contains list_filter = ('email_usage',) search_fields = ( 'contact__account__name', 'short_name', 'full_name', 'phone', 'phone2', diff --git a/orchestra/apps/lists/admin.py b/orchestra/apps/lists/admin.py index a4778eb4..1f5594f4 100644 --- a/orchestra/apps/lists/admin.py +++ b/orchestra/apps/lists/admin.py @@ -46,6 +46,7 @@ class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModel change_readonly_fields = ('name',) form = ListChangeForm add_form = ListCreationForm + list_select_related = ('account', 'address_domain',) filter_by_account_fields = ['address_domain'] address_domain_link = admin_link('address_domain', order='address_domain__name') diff --git a/orchestra/apps/mailboxes/admin.py b/orchestra/apps/mailboxes/admin.py index 549a7851..dc65f631 100644 --- a/orchestra/apps/mailboxes/admin.py +++ b/orchestra/apps/mailboxes/admin.py @@ -96,7 +96,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): list_display = ( - 'email', 'domain_link', 'display_mailboxes', 'display_forward', 'account_link' + 'email', 'account_link', 'domain_link', 'display_mailboxes', 'display_forward', ) list_filter = (HasMailboxListFilter, HasForwardListFilter) fields = ('account_link', ('name', 'domain'), 'mailboxes', 'forward') diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 694de216..f778d51d 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -102,6 +102,7 @@ class ResourceDataAdmin(ExtendedModelAdmin): resource_link = admin_link('resource') content_object_link = admin_link('content_object') + content_object_link.admin_order_field = None display_updated = admin_date('updated_at', short_description=_("Updated")) def get_urls(self): diff --git a/orchestra/apps/saas/admin.py b/orchestra/apps/saas/admin.py index 293cd0db..4152671f 100644 --- a/orchestra/apps/saas/admin.py +++ b/orchestra/apps/saas/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin +from django.utils.translation import ugettext_lazy as _ from orchestra.apps.accounts.admin import AccountAdminMixin from orchestra.apps.plugins.admin import SelectPluginAdminMixin @@ -8,10 +9,26 @@ from .services import SoftwareService class SaaSAdmin(SelectPluginAdminMixin, AccountAdminMixin, admin.ModelAdmin): - list_display = ('description', 'service', 'account_link') + list_display = ('username', 'service', 'display_site_name', 'account_link') list_filter = ('service',) plugin = SoftwareService plugin_field = 'service' - + + def display_site_name(self, saas): + site_name = saas.get_site_name() + return '%s' % (site_name, site_name) + display_site_name.short_description = _("Site name") + display_site_name.allow_tags = True + display_site_name.admin_order_field = 'site_name' + + def get_fields(self, request, obj=None): + fields = super(SaaSAdmin, self).get_fields(request, obj) + fields = list(fields) + # TODO do it in AccountAdminMixin? + if obj is not None: + fields.remove('account') + else: + fields.remove('account_link') + return fields admin.site.register(SaaS, SaaSAdmin) diff --git a/orchestra/apps/saas/models.py b/orchestra/apps/saas/models.py index f6f8e5a7..00c5fa8f 100644 --- a/orchestra/apps/saas/models.py +++ b/orchestra/apps/saas/models.py @@ -3,7 +3,8 @@ from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ from jsonfield import JSONField -from orchestra.core import services +from orchestra.core import services, validators +from orchestra.models.fields import NullableCharField from .services import SoftwareService @@ -11,33 +12,36 @@ from .services import SoftwareService class SaaS(models.Model): service = models.CharField(_("service"), max_length=32, choices=SoftwareService.get_plugin_choices()) - # TODO use model username password instead of data -# username = models.CharField(_("username"), max_length=64, unique=True, -# help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."), -# validators=[validators.RegexValidator(r'^[\w.-]+$', -# _("Enter a valid username."), 'invalid')]) -# password = models.CharField(_("password"), max_length=128) + username = models.CharField(_("username"), max_length=64, + help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."), + validators=[validators.validate_username]) + site_name = NullableCharField(_("site name"), max_length=32, null=True) account = models.ForeignKey('accounts.Account', verbose_name=_("account"), related_name='saas') - data = JSONField(_("data")) + data = JSONField(_("data"), help_text=_("Extra information dependent of each service.")) class Meta: verbose_name = "SaaS" verbose_name_plural = "SaaS" + unique_together = ( + ('username', 'service'), + ('site_name', 'service'), + ) def __unicode__(self): - return "%s (%s)" % (self.description, self.service_class.verbose_name) + return "%s@%s" % (self.username, self.service) @cached_property def service_class(self): return SoftwareService.get_plugin(self.service) - @cached_property - def description(self): - return self.service_class().get_description(self.data) + def get_site_name(self): + return self.service_class().get_site_name(self) def clean(self): self.data = self.service_class().clean_data(self) - + + def set_password(self, password): + self.password = password services.register(SaaS) diff --git a/orchestra/apps/saas/services/bscw.py b/orchestra/apps/saas/services/bscw.py index 4b02ead1..722c22bf 100644 --- a/orchestra/apps/saas/services/bscw.py +++ b/orchestra/apps/saas/services/bscw.py @@ -3,51 +3,26 @@ from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers -from orchestra.apps.plugins.forms import PluginDataForm -from orchestra.core import validators - -from .options import SoftwareService +from .options import SoftwareService, SoftwareServiceForm # TODO monitor quota since out of sync? -class BSCWForm(PluginDataForm): - username = forms.CharField(label=_("Username"), max_length=64) - password = forms.CharField(label=_("Password"), max_length=256, required=False) - email = forms.EmailField(label=_("Email")) +class BSCWForm(SoftwareServiceForm): + email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'})) quota = forms.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB.")) -class SEPADirectDebitSerializer(serializers.Serializer): - username = serializers.CharField(label=_("Username"), max_length=64, - validators=[validators.validate_name]) - password = serializers.CharField(label=_("Password"), max_length=256, required=False, - write_only=True) +class BSCWDataSerializer(serializers.Serializer): email = serializers.EmailField(label=_("Email")) quota = serializers.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB.")) - - def validate(self, data): - data['username'] = data['username'].strip() - return data class BSCWService(SoftwareService): verbose_name = "BSCW" form = BSCWForm - serializer = SEPADirectDebitSerializer - description_field = 'username' + serializer = BSCWDataSerializer icon = 'saas/icons/BSCW.png' - - @classmethod - def clean_data(cls, saas): - try: - data = super(BSCWService, cls).clean_data(saas) - except ValidationError, error: - if not saas.pk and 'password' not in saas.data: - error.error_dict['password'] = [_("Password is required.")] - raise error - if not saas.pk and 'password' not in saas.data: - raise ValidationError({ - 'password': _("Password is required.") - }) - return data + # TODO override from settings + site_name = 'bascw.orchestra.lan' + change_readonly_fileds = ('email',) diff --git a/orchestra/apps/saas/services/dokuwiki.py b/orchestra/apps/saas/services/dokuwiki.py index cdf3f981..2ff53b91 100644 --- a/orchestra/apps/saas/services/dokuwiki.py +++ b/orchestra/apps/saas/services/dokuwiki.py @@ -1,20 +1,6 @@ -from django import forms -from django.utils.translation import ugettext_lazy as _ - -from orchestra.apps.plugins.forms import PluginDataForm - from .options import SoftwareService -class DowkuwikiForm(PluginDataForm): - username = forms.CharField(label=_("Username"), max_length=64) - password = forms.CharField(label=_("Password"), max_length=64) - site_name = forms.CharField(label=_("Site name"), max_length=64) - email = forms.EmailField(label=_("Email")) - - class DokuwikiService(SoftwareService): verbose_name = "Dowkuwiki" - form = DowkuwikiForm - description_field = 'site_name' icon = 'saas/icons/Dokuwiki.png' diff --git a/orchestra/apps/saas/services/drupal.py b/orchestra/apps/saas/services/drupal.py index a42cce87..65ae9e5e 100644 --- a/orchestra/apps/saas/services/drupal.py +++ b/orchestra/apps/saas/services/drupal.py @@ -1,20 +1,6 @@ -from django import forms -from django.utils.translation import ugettext_lazy as _ - -from orchestra.apps.plugins.forms import PluginDataForm - from .options import SoftwareService -class DrupalForm(PluginDataForm): - username = forms.CharField(label=_("Username"), max_length=64) - password = forms.CharField(label=_("Password"), max_length=64) - site_name = forms.CharField(label=_("Site name"), max_length=64) - email = forms.EmailField(label=_("Email")) - - class DrupalService(SoftwareService): verbose_name = "Drupal" - form = DrupalForm - description_field = 'site_name' icon = 'saas/icons/Drupal.png' diff --git a/orchestra/apps/saas/services/gitlab.py b/orchestra/apps/saas/services/gitlab.py index 0668214a..d9d2f581 100644 --- a/orchestra/apps/saas/services/gitlab.py +++ b/orchestra/apps/saas/services/gitlab.py @@ -1,20 +1,6 @@ -from django import forms -from django.utils.translation import ugettext_lazy as _ - -from orchestra.apps.plugins.forms import PluginDataForm - from .options import SoftwareService -class GitLabForm(PluginDataForm): - username = forms.CharField(label=_("Username"), max_length=64) - password = forms.CharField(label=_("Password"), max_length=64) - project_name = forms.CharField(label=_("Project name"), max_length=64) - email = forms.EmailField(label=_("Email")) - - class GitLabService(SoftwareService): verbose_name = "GitLab" - form = GitLabForm - description_field = 'project_name' icon = 'saas/icons/gitlab.png' diff --git a/orchestra/apps/saas/services/options.py b/orchestra/apps/saas/services/options.py index 88b727de..631dbd35 100644 --- a/orchestra/apps/saas/services/options.py +++ b/orchestra/apps/saas/services/options.py @@ -1,21 +1,84 @@ from django import forms from django.core.exceptions import ValidationError +from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ from orchestra.apps import plugins +from orchestra.apps.plugins.forms import PluginDataForm +from orchestra.core import validators +from orchestra.forms import widgets from orchestra.utils.functional import cached from orchestra.utils.python import import_class from .. import settings -# TODO if unique_description: make description_field create only +class SoftwareServiceForm(PluginDataForm): + password = forms.CharField(label=_("Password"), required=False, + widget=widgets.ReadOnlyWidget('Unknown password'), + help_text=_("Servide passwords are not stored, so there is no way to see this " + "service's password, but you can change the password using " + "this form.")) + password1 = forms.CharField(label=_("Password"), validators=[validators.validate_password], + widget=forms.PasswordInput) + password2 = forms.CharField(label=_("Password confirmation"), + widget=forms.PasswordInput, + help_text=_("Enter the same password as above, for verification.")) + + def __init__(self, *args, **kwargs): + super(SoftwareServiceForm, self).__init__(*args, **kwargs) + self.is_change = bool(self.instance and self.instance.pk) + if self.is_change: + for field in self.plugin.change_readonly_fileds + ('username',): + value = getattr(self.instance, field, None) or self.instance.data[field] + self.fields[field].required = False + self.fields[field].widget = widgets.ReadOnlyWidget(value) + self.fields[field].help_text = None + + site_name = self.instance.get_site_name() + self.fields['password1'].required = False + self.fields['password1'].widget = forms.HiddenInput() + self.fields['password2'].required = False + self.fields['password2'].widget = forms.HiddenInput() + else: + self.fields['password'].widget = forms.HiddenInput() + site_name = self.plugin.site_name + if site_name: + link = '%s' % (site_name, site_name) + self.fields['site_name'].widget = widgets.ReadOnlyWidget(site_name, mark_safe(link)) + self.fields['site_name'].required = False + else: + base_name = self.plugin.site_name_base_domain + help_text = _("The final URL would be <site_name>.%s") % base_name + self.fields['site_name'].help_text = help_text + + def clean_password2(self): + if not self.is_change: + password1 = self.cleaned_data.get("password1") + password2 = self.cleaned_data.get("password2") + if password1 and password2 and password1 != password2: + msg = _("The two password fields didn't match.") + raise forms.ValidationError(msg) + return password2 + + def clean_site_name(self): + if self.plugin.site_name: + return None + return self.cleaned_data['site_name'] + + def save(self, commit=True): + obj = super(SoftwareServiceForm, self).save(commit=commit) + if not self.is_change: + obj.set_password(self.cleaned_data["password1"]) + return obj + class SoftwareService(plugins.Plugin): - description_field = '' - unique_description = True - form = None + form = SoftwareServiceForm serializer = None + site_name = None + site_name_base_domain = 'orchestra.lan' icon = 'orchestra/icons/apps.png' + change_readonly_fileds = () class_verbose_name = _("Software as a Service") @classmethod @@ -29,18 +92,14 @@ class SoftwareService(plugins.Plugin): @classmethod def clean_data(cls, saas): """ model clean, uses cls.serizlier by default """ - if cls.unique_description and not saas.pk: - from ..models import SaaS - field = cls.description_field - if SaaS.objects.filter(data__contains='"%s":"%s"' % (field, saas.data[field])).exists(): - raise ValidationError({ - field: _("SaaS service with this %(field)s already exists.") - }, params={'field': field}) serializer = cls.serializer(data=saas.data) if not serializer.is_valid(): raise ValidationError(serializer.errors) return serializer.data + def get_site_name(self, saas): + return self.site_name or '.'.join((saas.site_name, self.site_name_base_domain)) + def get_form(self): self.form.plugin = self self.form.plugin_field = 'service' @@ -49,6 +108,3 @@ class SoftwareService(plugins.Plugin): def get_serializer(self): self.serializer.plugin = self return self.serializer - - def get_description(self, data): - return data[self.description_field] diff --git a/orchestra/apps/saas/services/phplist.py b/orchestra/apps/saas/services/phplist.py index 341fd50c..3cb4a90f 100644 --- a/orchestra/apps/saas/services/phplist.py +++ b/orchestra/apps/saas/services/phplist.py @@ -1,17 +1,14 @@ from django import forms from django.utils.translation import ugettext_lazy as _ -from orchestra.apps.plugins.forms import PluginDataForm - -from .options import SoftwareService +from .options import SoftwareService, SoftwareServiceForm -class PHPListForm(PluginDataForm): - email = forms.EmailField(label=_("Email")) +class PHPListForm(SoftwareServiceForm): + email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'})) class PHPListService(SoftwareService): verbose_name = "phpList" form = PHPListForm - description_field = 'email' icon = 'saas/icons/Phplist.png' diff --git a/orchestra/apps/saas/services/wordpress.py b/orchestra/apps/saas/services/wordpress.py index bd4f62dc..978e2de5 100644 --- a/orchestra/apps/saas/services/wordpress.py +++ b/orchestra/apps/saas/services/wordpress.py @@ -1,30 +1,23 @@ from django import forms from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers -from orchestra.apps.plugins.forms import PluginDataForm - -from .options import SoftwareService +from .options import SoftwareService, SoftwareServiceForm -class WordPressForm(PluginDataForm): - username = forms.CharField(label=_("Username"), max_length=64) - password = forms.CharField(label=_("Password"), max_length=64) - site_name = forms.CharField(label=_("Site name"), max_length=64, - help_text=_("URL will be <site_name>.blogs.orchestra.lan")) - email = forms.EmailField(label=_("Email")) - - def __init__(self, *args, **kwargs): - super(WordPressForm, self).__init__(*args, **kwargs) - instance = kwargs.get('instance') - if instance: - url = 'http://%s.%s' % (instance.data['site_name'], 'blogs.orchestra.lan') - url = '%s' % (url, url) - self.fields['site_name'].help_text = mark_safe(url) +class WordPressForm(SoftwareServiceForm): + email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'})) + + +class WordPressDataSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) class WordPressService(SoftwareService): verbose_name = "WordPress" form = WordPressForm - description_field = 'site_name' + serializer = WordPressDataSerializer icon = 'saas/icons/WordPress.png' + site_name_base_domain = 'blogs.orchestra.lan' + change_readonly_fileds = ('email',) diff --git a/orchestra/apps/systemusers/backends.py b/orchestra/apps/systemusers/backends.py index 527a579b..92bd8d79 100644 --- a/orchestra/apps/systemusers/backends.py +++ b/orchestra/apps/systemusers/backends.py @@ -99,52 +99,55 @@ class SystemUserDisk(ServiceMonitor): class FTPTraffic(ServiceMonitor): model = 'systemusers.SystemUser' resource = ServiceMonitor.TRAFFIC - verbose_name = _('Main FTP traffic') + verbose_name = _('Systemuser FTP traffic') def prepare(self): current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") self.append(textwrap.dedent("""\ function monitor () { OBJECT_ID=$1 - INI_DATE=$2 + INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") + END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s') USERNAME="$3" LOG_FILE="$4" - grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \\ - | grep " \\[${USERNAME}\\] " \\ - | awk -v ini="${INI_DATE}" end="$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s')" ' - BEGIN { - sum = 0 - months["Jan"] = "01" - months["Feb"] = "02" - months["Mar"] = "03" - months["Apr"] = "04" - months["May"] = "05" - months["Jun"] = "06" - months["Jul"] = "07" - months["Aug"] = "08" - months["Sep"] = "09" - months["Oct"] = "10" - months["Nov"] = "11" - months["Dec"] = "12" - } { - # log: Fri Jul 11 13:23:17 2014 - split($4, t, ":") - # line_date = year month day hour minute second - line_date = $5 months[$2] $3 t[1] t[2] t[3] - if ( line_date > ini && line_date < end) - split($0, l, "\\", ") - split(l[3], b, " ") - sum += b[1] - } END { - print sum - } - ' | xargs echo ${OBJECT_ID} + { + grep "UPLOAD\|DOWNLOAD" "${LOG_FILE}" \\ + | grep " \\[${USERNAME}\\] " \\ + | awk -v ini="${INI_DATE}" -v end="${END_DATE}" ' + BEGIN { + sum = 0 + months["Jan"] = "01" + months["Feb"] = "02" + months["Mar"] = "03" + months["Apr"] = "04" + months["May"] = "05" + months["Jun"] = "06" + months["Jul"] = "07" + months["Aug"] = "08" + months["Sep"] = "09" + months["Oct"] = "10" + months["Nov"] = "11" + months["Dec"] = "12" + } { + # log: Fri Jul 11 13:23:17 2014 + split($4, t, ":") + # line_date = year month day hour minute second + line_date = $5 months[$2] $3 t[1] t[2] t[3] + if ( line_date > ini && line_date < end) { + split($0, l, "\\", ") + split(l[3], b, " ") + sum += b[1] + } + } END { + print sum + }' || [[ $? == 1 ]] && true + } | xargs echo ${OBJECT_ID} }""" % current_date)) def monitor(self, user): context = self.get_context(user) self.append( - 'monitor %{object_id} $(date "+%Y%m%d%H%M%S" -d "{last_date}") "{username}" "{log_file}"'.format(**context) + 'monitor {object_id} "{last_date}" "{username}" {log_file}'.format(**context) ) def get_context(self, user): diff --git a/orchestra/apps/systemusers/models.py b/orchestra/apps/systemusers/models.py index dd0e8de7..cf53230c 100644 --- a/orchestra/apps/systemusers/models.py +++ b/orchestra/apps/systemusers/models.py @@ -1,14 +1,13 @@ import os from django.contrib.auth.hashers import make_password -from django.core import validators from django.core.exceptions import ValidationError from django.core.mail import send_mail from django.db import models from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ -from orchestra.core import services +from orchestra.core import services, validators from . import settings @@ -25,9 +24,7 @@ class SystemUser(models.Model): """ System users """ username = models.CharField(_("username"), max_length=64, unique=True, help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."), - validators=[ - validators.RegexValidator(r'^[\w.-]+$', _("Enter a valid username.")) - ]) + validators=[validators.validate_username]) password = models.CharField(_("password"), max_length=128) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), related_name='systemusers') diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 037ee559..25eac682 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -218,49 +218,55 @@ class Apache2Traffic(ServiceMonitor): verbose_name = _("Apache 2 Traffic") def prepare(self): - current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") + ignore_hosts = '\\|'.join(settings.WEBSITES_TRAFFIC_IGNORE_HOSTS) + context = { + 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), + 'ignore_hosts': '-v "%s"' % ignore_hosts if ignore_hosts else '', + } self.append(textwrap.dedent("""\ function monitor () { OBJECT_ID=$1 - INI_DATE=$2 + INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") + END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s') LOG_FILE="$3" { - awk -v ini="${INI_DATE}" -v end="$(date '+%%Y%%m%%d%%H%%M%%S' -d '%s')" ' - BEGIN { - sum = 0 - months["Jan"] = "01"; - months["Feb"] = "02"; - months["Mar"] = "03"; - months["Apr"] = "04"; - months["May"] = "05"; - months["Jun"] = "06"; - months["Jul"] = "07"; - months["Aug"] = "08"; - months["Sep"] = "09"; - months["Oct"] = "10"; - months["Nov"] = "11"; - months["Dec"] = "12"; - } { - # date = [11/Jul/2014:13:50:41 - date = substr($4, 2) - year = substr(date, 8, 4) - month = months[substr(date, 4, 3)]; - day = substr(date, 1, 2) - hour = substr(date, 13, 2) - minute = substr(date, 16, 2) - second = substr(date, 19, 2) - line_date = year month day hour minute second - if ( line_date > ini && line_date < end) - sum += $NF - } END { - print sum - }' "${LOG_FILE}" || echo 0 + { grep "%(ignore_hosts)s" "${LOG_FILE}" || echo '\\n'; } \\ + | awk -v ini="${INI_DATE}" -v end="${END_DATE}" ' + BEGIN { + sum = 0 + months["Jan"] = "01"; + months["Feb"] = "02"; + months["Mar"] = "03"; + months["Apr"] = "04"; + months["May"] = "05"; + months["Jun"] = "06"; + months["Jul"] = "07"; + months["Aug"] = "08"; + months["Sep"] = "09"; + months["Oct"] = "10"; + months["Nov"] = "11"; + months["Dec"] = "12"; + } { + # date = [11/Jul/2014:13:50:41 + date = substr($4, 2) + year = substr(date, 8, 4) + month = months[substr(date, 4, 3)]; + day = substr(date, 1, 2) + hour = substr(date, 13, 2) + minute = substr(date, 16, 2) + second = substr(date, 19, 2) + line_date = year month day hour minute second + if ( line_date > ini && line_date < end) + sum += $NF + } END { + print sum + }' || [[ $? == 1 ]] && true } | xargs echo ${OBJECT_ID} - }""" % current_date)) + }""" % context)) def monitor(self, site): context = self.get_context(site) - self.append('monitor {object_id} $(date "+%Y%m%d%H%M%S" -d "{last_date}") {log_file}'.format(**context)) + self.append('monitor {object_id} "{last_date}" {log_file}'.format(**context)) def get_context(self, site): return { diff --git a/orchestra/apps/websites/settings.py b/orchestra/apps/websites/settings.py index 477c56f4..46767432 100644 --- a/orchestra/apps/websites/settings.py +++ b/orchestra/apps/websites/settings.py @@ -84,3 +84,7 @@ WEBSITES_WEBALIZER_PATH = getattr(settings, 'WEBSITES_WEBALIZER_PATH', WEBSITES_WEBSITE_WWW_LOG_PATH = getattr(settings, 'WEBSITES_WEBSITE_WWW_LOG_PATH', # %(user_home)s %(name)s %(unique_name)s %(username)s '/var/log/apache2/virtual/%(unique_name)s') + + +WEBSITES_TRAFFIC_IGNORE_HOSTS = getattr(settings, 'WEBSITES_TRAFFIC_IGNORE_HOSTS', + []) diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index 9a115637..67512c89 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -162,7 +162,7 @@ FLUENT_DASHBOARD_APP_GROUPS = ( 'orchestra.apps.orchestration.models.BackendLog', 'orchestra.apps.orchestration.models.Server', 'orchestra.apps.resources.models.Resource', - 'orchestra.apps.resources.models.Monitor', + 'orchestra.apps.resources.models.ResourceData', 'orchestra.apps.services.models.Service', 'orchestra.apps.plans.models.Plan', 'orchestra.apps.miscellaneous.models.MiscService', @@ -206,7 +206,7 @@ FLUENT_DASHBOARD_APP_ICONS = { 'orchestration/route': 'hal.png', 'orchestration/backendlog': 'scriptlog.png', 'resources/resource': "gauge.png", - 'resources/monitor': "Utilities-system-monitor.png", + 'resources/resourcedata': "monitor.png", 'plans/plan': 'Pack.png', } diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index b11a722c..052a3a85 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -84,6 +84,10 @@ def validate_hostname(hostname): raise ValidationError(_("Not a valid hostname (%s).") % name) +def validate_username(value): + validators.RegexValidator(r'^[\w.-]+$', _("Enter a valid username."))(value) + + def validate_password(value): try: crack.VeryFascistCheck(value)