Improvements on resources

TODO.md

@@ -155,9 +155,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 * Subdomain saving should not trigger bind slave
 
 * multiple files monitoring
-* prevent adding local email addresses on account.contacts account.email
-
-* Resource monitoring without ROUTE alert or explicit error
 
 * Domain validation has to be done with injected records and subdomains
 
@@ -202,6 +199,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 
 * webapp backend option compatibility check?
 
-* Miscellaneous service construct form for specific data, fields, validation, uniquenes.. etc (domain usecase)
-
 * miscellaneous.indentifier.endswith(('.org', '.es', '.cat'))
+
+* miscservic icon miscellaneous icon + scissors

orchestra/apps/mailboxes/models.py

@@ -1,5 +1,5 @@
 from django.contrib.auth.hashers import make_password
-from django.core.validators import RegexValidator
+from django.core.validators import RegexValidator, ValidationError
 from django.db import models
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
@@ -109,6 +109,17 @@ class Address(models.Model):
 #            destinations.append(self.forward)
 #        return ' '.join(destinations)
     
+    def clean(self):
+        if self.account:
+            errors = []
+            for mailbox in self.get_forward_mailboxes():
+                if mailbox.account == self.account:
+                    errors.append(ValidationError(
+                        _("Please use mailboxes field for '%s' mailbox.") % mailbox
+                    ))
+            if errors:
+                raise ValidationError({'forward': errors})
+    
     def get_forward_mailboxes(self):
         for forward in self.forward.split():
             if '@' not in forward:

orchestra/apps/mailboxes/validators.py

@@ -13,7 +13,7 @@ from . import settings
 
 
 def validate_emailname(value):
-    msg = _("'%s' is not a correct email name" % value)
+    msg = _("'%s' is not a correct email name." % value)
     if '@' in value:
         raise ValidationError(msg)
     value += '@localhost'
@@ -26,20 +26,27 @@ def validate_emailname(value):
 def validate_forward(value):
     """ space separated mailboxes or emails """
     from .models import Mailbox
+    errors = []
     destinations = []
     for destination in value.split():
         if destination in destinations:
-            raise ValidationError(_("'%s' is already present.") % destination)
+            errors.append(ValidationError(
+                _("'%s' is already present.") % destination
+            ))
         destinations.append(destination)
-        msg = _("'%s' is not an existent mailbox" % destination)
         if '@' in destination:
-            if not destination[-1].isalpha():
+            try:
-                raise ValidationError(msg)
                 EmailValidator()(destination)
-        else:
+            except ValidationError:
-            if not Mailbox.objects.filter(user__username=destination).exists():
+                errors.append(ValidationError(
-                raise ValidationError(msg)
+                    _("'%s' is not a valid email address.") % destination
-            validate_emailname(destination)
+                ))
+        elif not Mailbox.objects.filter(name=destination).exists():
+            errors.append(ValidationError(
+                _("'%s' is not an existent mailbox.") % destination
+            ))
+    if errors:
+        raise ValidationError(errors)
 
 
 def validate_sieve(value):

orchestra/apps/miscellaneous/admin.py

@@ -1,3 +1,4 @@
+from django import forms
 from django.contrib import admin
 from django.core.urlresolvers import reverse
 from django.db import models
@@ -5,24 +6,27 @@ from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.admin import ExtendedModelAdmin
+from orchestra.admin.utils import admin_link
 from orchestra.apps.accounts.admin import AccountAdminMixin
+from orchestra.apps.plugins import PluginModelAdapter
+from orchestra.apps.plugins.admin import SelectPluginAdminMixin
 
+from . import settings
 from .models import MiscService, Miscellaneous
 
 
-from orchestra.apps.plugins.admin import SelectPluginAdminMixin, PluginAdapter
+class MiscServicePlugin(PluginModelAdapter):
-
-
-class MiscServicePlugin(PluginAdapter):
     model = MiscService
     name_field = 'name'
 
 
 class MiscServiceAdmin(ExtendedModelAdmin):
-    list_display = ('name', 'verbose_name', 'num_instances', 'has_amount', 'is_active')
+    list_display = (
-    list_editable = ('has_amount', 'is_active')
+        'name', 'verbose_name', 'num_instances', 'has_identifier', 'has_amount', 'is_active'
-    list_filter = ('has_amount', 'is_active')
+    )
-    fields = ('verbose_name', 'name', 'description', 'has_amount', 'is_active')
+    list_editable = ('is_active',)
+    list_filter = ('has_identifier', 'has_amount', 'is_active')
+    fields = ('verbose_name', 'name', 'description', 'has_identifier', 'has_amount', 'is_active')
     prepopulated_fields = {'name': ('verbose_name',)}
     change_readonly_fields = ('name',)
     
@@ -39,12 +43,28 @@ class MiscServiceAdmin(ExtendedModelAdmin):
         qs = super(MiscServiceAdmin, self).queryset(request)
         return qs.annotate(models.Count('instances', distinct=True))
     
+    def formfield_for_dbfield(self, db_field, **kwargs):
+        """ Make value input widget bigger """
+        if db_field.name == 'description':
+            kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 2})
+        return super(MiscServiceAdmin, self).formfield_for_dbfield(db_field, **kwargs)
+
 
 class MiscellaneousAdmin(AccountAdminMixin, SelectPluginAdminMixin, admin.ModelAdmin):
-    list_display = ('service', 'amount', 'active', 'account_link')
+    list_display = ('__unicode__', 'service_link', 'amount', 'dispaly_active', 'account_link')
+    list_filter = ('service__name', 'is_active')
+    list_select_related = ('service', 'account')
     plugin_field = 'service'
     plugin = MiscServicePlugin
     
+    service_link = admin_link('service')
+    
+    def dispaly_active(self, instance):
+        return instance.active
+    dispaly_active.short_description = _("Active")
+    dispaly_active.boolean = True
+    dispaly_active.admin_order_field = 'is_active'
+    
     def get_service(self, obj):
         if obj is None:
             return self.plugin.get_plugin(self.plugin_value)().instance
@@ -58,21 +78,29 @@ class MiscellaneousAdmin(AccountAdminMixin, SelectPluginAdminMixin, admin.ModelA
         service = self.get_service(obj)
         if service.has_amount:
             fields.insert(-1, 'amount')
-#        if service.has_identifier:
+        if service.has_identifier:
-#            fields.insert(1, 'identifier')
+            fields.insert(1, 'identifier')
         return fields
     
-    
     def get_form(self, request, obj=None, **kwargs):
         form = super(SelectPluginAdminMixin, self).get_form(request, obj=obj, **kwargs)
         service = self.get_service(obj)
         def clean_identifier(self, service=service):
+            identifier = self.cleaned_data['identifier']
             validator = settings.MISCELLANEOUS_IDENTIFIER_VALIDATORS.get(service.name, None)
             if validator:
-                validator(self.cleaned_data['identifier'])
+                validator(identifier)
+            return identifier
+        
         form.clean_identifier = clean_identifier
         return form
     
+    def formfield_for_dbfield(self, db_field, **kwargs):
+        """ Make value input widget bigger """
+        if db_field.name == 'description':
+            kwargs['widget'] = forms.Textarea(attrs={'cols': 70, 'rows': 4})
+        return super(MiscellaneousAdmin, self).formfield_for_dbfield(db_field, **kwargs)
+
 
 admin.site.register(MiscService, MiscServiceAdmin)
 admin.site.register(Miscellaneous, MiscellaneousAdmin)

orchestra/apps/miscellaneous/models.py

@@ -14,9 +14,9 @@ class MiscService(models.Model):
             help_text=_("Human readable name"))
     description = models.TextField(_("description"), blank=True,
             help_text=_("Optional description"))
-#    has_identifier = models.BooleanField(_("has identifier"), default=True,
+    has_identifier = models.BooleanField(_("has identifier"), default=True,
-#            help_text=_("Designates if this service has a <b>unique text</b> field that "
+            help_text=_("Designates if this service has a <b>unique text</b> field that "
-#                        "identifies it or not."))
+                        "identifies it or not."))
     has_amount = models.BooleanField(_("has amount"), default=False,
             help_text=_("Designates whether this service has <tt>amount</tt> "
                         "property or not."))
@@ -39,8 +39,8 @@ class Miscellaneous(models.Model):
             related_name='instances')
     account = models.ForeignKey('accounts.Account', verbose_name=_("account"),
             related_name='miscellaneous')
-#    identifier = NullableCharField(_("identifier"), max_length=256, null=True, unique=True,
+    identifier = NullableCharField(_("identifier"), max_length=256, null=True, unique=True,
-#            help_text=_("A unique identifier for this service."))
+            help_text=_("A unique identifier for this service."))
     description = models.TextField(_("description"), blank=True)
     amount = models.PositiveIntegerField(_("amount"), default=1)
     is_active = models.BooleanField(_("active"), default=True,
@@ -51,8 +51,7 @@ class Miscellaneous(models.Model):
         verbose_name_plural = _("miscellaneous")
     
     def __unicode__(self):
-#        return self.identifier or str(self.service)
+        return self.identifier or str(self.service)
-        return "{0}-{1}".format(str(self.service), str(self.account))
     
     @cached_property
     def active(self):
@@ -62,8 +61,8 @@ class Miscellaneous(models.Model):
             return self.is_active
     
     def clean(self):
-#        if self.identifier:
+        if self.identifier:
-#            self.identifier = self.identifier.strip()
+            self.identifier = self.identifier.strip()
         self.description = self.description.strip()
 
 

orchestra/apps/miscellaneous/settings.py

@@ -1,5 +1,5 @@
 from django.conf import settings
 
 
-MISCELLANEOUS_IDENTIFIER_VALIDATORS = getattr(settings, MISCELLANEOUS_IDENTIFIER_VALIDATORS, {})
+MISCELLANEOUS_IDENTIFIER_VALIDATORS = getattr(settings, 'MISCELLANEOUS_IDENTIFIER_VALIDATORS', {})
 # MISCELLANEOUS_IDENTIFIER_VALIDATORS = { miscservice__name: validator_function }

orchestra/apps/plugins/admin.py

@@ -74,35 +74,3 @@ class SelectPluginAdminMixin(object):
         if not change:
             setattr(obj, self.plugin_field, self.plugin_value)
         obj.save()
-
-
-class PluginAdapter(object):
-    """ Adapter class for using model classes as plugins """
-    
-    model = None
-    name_field = None
-    
-    def __init__(self, instance):
-        self.instance = instance
-    
-    @classmethod
-    @cached
-    def get_plugins(cls):
-        plugins = []
-        for instance in cls.model.objects.filter(is_active=True):
-            plugins.append(cls(instance))
-        return plugins
-    
-    @classmethod
-    def get_plugin(cls, name):
-        return cls(cls.model.objects.get(**{cls.name_field:name}))
-    
-    @property
-    def verbose_name(self):
-        return self.instance.verbose_name or str(getattr(self.instance, self.name_field))
-    
-    def get_name(self):
-        return getattr(self.instance, self.name_field)
-    
-    def __call__(self):
-        return self

orchestra/apps/plugins/options.py

@@ -41,6 +41,27 @@ class Plugin(object):
         return sorted(choices, key=lambda e: e[1])
 
 
+class PluginModelAdapter(Plugin):
+    """ Adapter class for using model classes as plugins """
+    model = None
+    name_field = None
+    
+    @classmethod
+    def get_plugins(cls):
+        plugins = []
+        for instance in cls.model.objects.filter(is_active=True):
+            attributes = {
+                'instance': instance,
+                'verbose_name': instance.verbose_name
+            }
+            plugins.append(type('PluginAdapter', (cls,), attributes))
+        return plugins
+    
+    @classmethod
+    def get_name(cls):
+        return getattr(cls.instance, cls.name_field)
+
+
 class PluginMount(type):
     def __init__(cls, name, bases, attrs):
         if not attrs.get('abstract', False):

orchestra/apps/plugins/templates/admin/plugins/select_plugin.html

@@ -20,7 +20,7 @@
                     {% for plugin in plugins %}
                     <li><a class="fluent-dashboard-icon" href="../?{{ field }}={{ plugin.get_name }}&{{ request.META.QUERY_STRING }}">
                         <img src="{% static plugin.icon %}" width="48" height="48" alt="{{ plugin.get_name }}"></a>
-                        <a class="fluent-dashboard-icon-caption" href="../?{{ field }}={{ plugin.get_name }}&{{ request.META.QUERY_STRING }}">{{ plugin.verbose_name }}</a></li>
+                        <a class="fluent-dashboard-icon-caption" href="../?{{ field }}={{ plugin.get_name }}&{{ request.META.QUERY_STRING }}">{{ plugin.get_verbose_name }}</a></li>
                     {% endfor %}
                 </ul>
             </div>
@@ -28,7 +28,7 @@
     {% else %}
         <ul>
         {% for plugin in plugins %}
-            <li><a style="font-size:small;" href="../?{{ field }}={{ plugin.get_name }}&{{ request.META.QUERY_STRING }}">{{ plugin.verbose_name }}</<a></li>
+            <li><a style="font-size:small;" href="../?{{ field }}={{ plugin.get_name }}&{{ request.META.QUERY_STRING }}">{{ plugin.get_verbose_name }}</<a></li>
         {% endfor %}
         </ul>
     {% endif %}

orchestra/apps/resources/admin.py

@@ -1,13 +1,15 @@
 from django.contrib import admin, messages
+from django.contrib.admin.utils import unquote
 from django.contrib.contenttypes import generic
 from django.core.urlresolvers import reverse
 from django.utils.functional import cached_property
 from django.utils.safestring import mark_safe
-from django.utils.translation import ugettext, ugettext_lazy as _
+from django.utils.translation import ungettext, ugettext, ugettext_lazy as _
 
 from orchestra.admin import ExtendedModelAdmin
 from orchestra.admin.filters import UsedContentTypeFilter
 from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
+from orchestra.apps.orchestration.models import Route
 from orchestra.core import services
 from orchestra.utils import database_ready
 
@@ -36,7 +38,7 @@ class ResourceAdmin(ExtendedModelAdmin):
             'fields': ('monitors', 'crontab'),
         }),
     )
-    change_readonly_fields = ('name', 'content_type', 'period')
+    change_readonly_fields = ('name', 'content_type')
     prepopulated_fields = {'name': ('verbose_name',)}
     
     def add_view(self, request, **kwargs):
@@ -48,6 +50,25 @@ class ResourceAdmin(ExtendedModelAdmin):
             )))
         return super(ResourceAdmin, self).add_view(request, **kwargs)
     
+    def change_view(self, request, object_id, form_url='', extra_context=None):
+        """ Remaind user when monitor routes are not configured """
+        if request.method == 'GET':
+            resource = self.get_object(request, unquote(object_id))
+            backends = Route.objects.values_list('backend', flat=True)
+            not_routed = []
+            for monitor in resource.monitors:
+                if monitor not in backends:
+                    not_routed.append(monitor)
+            if not_routed:
+                messages.warning(request, ungettext(
+                    _("%(not_routed)s monitor doesn't have any configured route."),
+                    _("%(not_routed)s monitors don't have any configured route."),
+                    len(not_routed),
+                ) % {
+                    'not_routed': ', '.join(not_routed)
+                })
+        return super(ResourceAdmin, self).changeform_view(request, object_id, form_url, extra_context)
+    
     def save_model(self, request, obj, form, change):
         super(ResourceAdmin, self).save_model(request, obj, form, change)
         model = obj.content_type.model_class()
@@ -70,23 +91,19 @@ class ResourceAdmin(ExtendedModelAdmin):
 
 class ResourceDataAdmin(ExtendedModelAdmin):
     list_display = (
-        'id', 'resource_link', 'content_object_link', 'used', 'allocated', 'display_unit',
+        'id', 'resource_link', 'content_object_link', 'display_used', 'allocated', 'display_unit',
         'display_updated'
     )
     list_filter = ('resource',)
-    add_fields = ('resource', 'content_type', 'object_id', 'used', 'updated_at', 'allocated')
     fields = (
-        'resource_link', 'content_type', 'content_object_link', 'used', 'display_updated',
+        'resource_link', 'content_type', 'content_object_link', 'display_used', 'display_updated',
         'allocated', 'display_unit'
     )
-    readonly_fields = ('display_unit',)
+    readonly_fields = fields
-    change_readonly_fields = (
-        'resource_link', 'content_type', 'content_object_link', 'used', 'display_updated',
-        'display_unit'
-    )
     actions = (run_monitor,)
     change_view_actions = actions
     ordering = ('-updated_at',)
+    list_select_related = ('resource',)
     prefetch_related = ('content_object',)
     
     resource_link = admin_link('resource')
@@ -98,6 +115,24 @@ class ResourceDataAdmin(ExtendedModelAdmin):
     display_unit.short_description = _("Unit")
     display_unit.admin_order_field = 'resource__unit'
     
+    def display_used(self, data):
+        if not data.used:
+            return ''
+        ids = []
+        for dataset in data.get_monitor_datasets():
+            if isinstance(dataset, MonitorData):
+                ids.append(dataset.id)
+            else:
+                ids += dataset.values_list('id', flat=True)
+        url = reverse('admin:resources_monitordata_changelist')
+        url += '?id__in=%s' % ','.join(map(str, ids))
+        return '<a href="%s">%s</a>' % (url, data.used)
+    display_used.short_description = _("Used")
+    display_used.allow_tags = True
+    
+    def has_add_permission(self, *args, **kwargs):
+        return False
+
 
 class MonitorDataAdmin(ExtendedModelAdmin):
     list_display = ('id', 'monitor', 'display_created', 'value', 'content_object_link')

orchestra/apps/resources/forms.py

@@ -5,8 +5,8 @@ from orchestra.forms.widgets import ShowTextWidget, ReadOnlyWidget
 
 
 class ResourceForm(forms.ModelForm):
-    verbose_name = forms.CharField(label=_("Name"), widget=ShowTextWidget(bold=True),
+    verbose_name = forms.CharField(label=_("Name"), required=False,
-            required=False)
+            widget=ShowTextWidget(bold=True))
     allocated = forms.IntegerField(label=_("Allocated"))
     unit = forms.CharField(label=_("Unit"), widget=ShowTextWidget(), required=False)
     

orchestra/apps/resources/helpers.py

@@ -1,52 +1,27 @@
 import datetime
 
-from django.contrib.contenttypes.models import ContentType
-from django.db.models.loading import get_model
-from django.utils import timezone
-
-from orchestra.models.utils import get_model_field_path
-
-from .backends import ServiceMonitor
-
 
 def compute_resource_usage(data):
     """ Computes MonitorData.used based on related monitors """
-    from .models import MonitorData
     resource = data.resource
-    today = timezone.now()
     result = 0
     has_result = False
-    for monitor in resource.monitors:
+    for dataset in data.get_monitor_datasets():
-        # Get related dataset
-        resource_model = data.content_type.model_class()
-        monitor_model = get_model(ServiceMonitor.get_backend(monitor).model)
-        if resource_model == monitor_model:
-            dataset = MonitorData.objects.filter(monitor=monitor,
-                    content_type=data.content_type_id, object_id=data.object_id)
-        else:
-            path = get_model_field_path(monitor_model, resource_model)
-            fields = '__'.join(path)
-            objects = monitor_model.objects.filter(**{fields: data.object_id})
-            pks = objects.values_list('id', flat=True)
-            ct = ContentType.objects.get_for_model(monitor_model)
-            dataset = MonitorData.objects.filter(monitor=monitor, content_type=ct, object_id__in=pks)
-        # Process dataset according to resource.period
         if resource.period == resource.MONTHLY_AVG:
-            try:
             last = dataset.latest()
-            except MonitorData.DoesNotExist:
+            epoch = datetime(
-                continue
+                year=today.year,
-            has_result = True
+                month=today.month,
-            epoch = datetime(year=today.year, month=today.month, day=1, tzinfo=timezone.utc)
+                day=1,
+                tzinfo=timezone.utc
+            )
             total = (last.created_at-epoch).total_seconds()
-            dataset = dataset.filter(created_at__year=today.year, created_at__month=today.month)
             ini = epoch
             for data in dataset:
                 slot = (data.created_at-ini).total_seconds()
                 result += data.value * slot/total
                 ini = data.created_at
         elif resource.period == resource.MONTHLY_SUM:
-            dataset = dataset.filter(created_at__year=today.year, created_at__month=today.month)
             # FIXME Aggregation of 0s returns None! django bug?
             # value = dataset.aggregate(models.Sum('value'))['value__sum']
             values = dataset.values_list('value', flat=True)
@@ -54,10 +29,7 @@ def compute_resource_usage(data):
                 has_result = True
                 result += sum(values)
         elif resource.period == resource.LAST:
-            try:
+            dataset.value
-                result += dataset.latest().value
-            except MonitorData.DoesNotExist:
-                continue
             has_result = True
         else:
             raise NotImplementedError("%s support not implemented" % data.period)

orchestra/apps/resources/models.py

@@ -2,6 +2,7 @@ from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelatio
 from django.contrib.contenttypes.models import ContentType
 from django.apps import apps
 from django.db import models
+from django.db.models.loading import get_model
 from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
@@ -9,6 +10,7 @@ from djcelery.models import PeriodicTask, CrontabSchedule
 
 from orchestra.core import validators
 from orchestra.models import queryset, fields
+from orchestra.models.utils import get_model_field_path
 from orchestra.utils.paths import get_project_root
 from orchestra.utils.system import run
 
@@ -88,8 +90,18 @@ class Resource(models.Model):
     def save(self, *args, **kwargs):
         created = not self.pk
         super(Resource, self).save(*args, **kwargs)
-        # Create Celery periodic task
+        self.sync_periodic_task()
+        # This only work on tests (multiprocessing used on real deployments)
+        apps.get_app_config('resources').reload_relations()
+        run('sleep 2 && touch %s/wsgi.py' % get_project_root(), async=True, display=True)
+    
+    def delete(self, *args, **kwargs):
+        super(Resource, self).delete(*args, **kwargs)
         name = 'monitor.%s' % str(self)
+    
+    def sync_periodic_task(self):
+        name = 'monitor.%s' % str(self)
+        if self.pk and self.crontab:
             try:
                 task = PeriodicTask.objects.get(name=name)
             except PeriodicTask.DoesNotExist:
@@ -101,18 +113,10 @@ class Resource(models.Model):
                         crontab=self.crontab
                     )
             else:
-            if not self.is_active:
+                if task.crontab != self.crontab:
-                task.delete()
-            elif task.crontab != self.crontab:
                     task.crontab = self.crontab
                     task.save(update_fields=['crontab'])
-        # This only work on tests (multiprocessing used on real deployments)
+        else:
-        apps.get_app_config('resources').reload_relations()
-        run('touch %s/wsgi.py' % get_project_root())
-    
-    def delete(self, *args, **kwargs):
-        super(Resource, self).delete(*args, **kwargs)
-        name = 'monitor.%s' % str(self)
             PeriodicTask.objects.filter(
                 name=name,
                 task='resources.Monitor',
@@ -146,10 +150,17 @@ class ResourceData(models.Model):
     def get_or_create(cls, obj, resource):
         ct = ContentType.objects.get_for_model(type(obj))
         try:
-            return cls.objects.get(content_type=ct, object_id=obj.pk, resource=resource)
+            return cls.objects.get(
+                content_type=ct,
+                object_id=obj.pk,
+                resource=resource
+            )
         except cls.DoesNotExist:
-            return cls.objects.create(content_object=obj, resource=resource,
+            return cls.objects.create(
-                    allocated=resource.default_allocation)
+                content_object=obj,
+                resource=resource,
+                allocated=resource.default_allocation
+            )
     
     @property
     def unit(self):
@@ -168,6 +179,47 @@ class ResourceData(models.Model):
     def monitor(self):
         tasks.monitor(self.resource_id, ids=(self.object_id,))
     
+    def get_monitor_datasets(self):
+        resource = self.resource
+        today = timezone.now()
+        datasets = []
+        for monitor in resource.monitors:
+            resource_model = self.content_type.model_class()
+            model_path = ServiceMonitor.get_backend(monitor).model
+            monitor_model = get_model(model_path)
+            if resource_model == monitor_model:
+                dataset = MonitorData.objects.filter(
+                    monitor=monitor,
+                    content_type=self.content_type_id,
+                    object_id=self.object_id
+                )
+            else:
+                path = get_model_field_path(monitor_model, resource_model)
+                fields = '__'.join(path)
+                objects = monitor_model.objects.filter(**{fields: self.object_id})
+                pks = objects.values_list('id', flat=True)
+                ct = ContentType.objects.get_for_model(monitor_model)
+                dataset = MonitorData.objects.filter(
+                    monitor=monitor,
+                    content_type=ct,
+                    object_id__in=pks
+                )
+            if resource.period in (resource.MONTHLY_AVG, resource.MONTHLY_SUM):
+                datasets.append(
+                    dataset.filter(
+                        created_at__year=today.year,
+                        created_at__month=today.month
+                    )
+                )
+            elif resource.period == resource.LAST:
+                try:
+                    datasets.append(dataset.latest())
+                except MonitorData.DoesNotExist:
+                    continue
+            else:
+                raise NotImplementedError("%s support not implemented" % self.period)
+        return datasets
+
 
 class MonitorData(models.Model):
     """ Stores monitored data """
@@ -207,10 +259,16 @@ def create_resource_relation():
                 data = self.obj.resource_set.get(resource__name=attr)
             except ResourceData.DoesNotExist:
                 model = self.obj._meta.model_name
-                resource = Resource.objects.get(content_type__model=model, name=attr,
+                resource = Resource.objects.get(
-                        is_active=True)
+                    content_type__model=model,
-                data = ResourceData(content_object=self.obj, resource=resource,
+                    name=attr,
-                        allocated=resource.default_allocation)
+                    is_active=True
+                )
+                data = ResourceData(
+                    content_object=self.obj,
+                    resource=resource,
+                    allocated=resource.default_allocation
+                )
             self.obj.__resource_cache[attr] = data
             return data
         

orchestra/apps/websites/backends/apache.py

@@ -161,10 +161,12 @@ class Apache2Backend(ServiceController):
         context = self.get_context(site)
         self.append("ls -l %(sites_enabled)s > /dev/null; DISABLED=$?" % context)
         if site.is_active:
-            self.append("if [[ $DISABLED ]]; then a2ensite %(site_unique_name)s.conf;\n"
+            self.append(
+                "if [[ $DISABLED ]]; then a2ensite %(site_unique_name)s.conf;\n"
                 "else UPDATED=0; fi" % context)
         else:
-            self.append("if [[ ! $DISABLED ]]; then a2dissite %(site_unique_name)s.conf;\n"
+            self.append(
+                "if [[ ! $DISABLED ]]; then a2dissite %(site_unique_name)s.conf;\n"
                 "else UPDATED=0; fi" % context)
     
     def get_username(self, site):
@@ -258,7 +260,7 @@ class Apache2Traffic(ServiceMonitor):
     
     def monitor(self, site):
         context = self.get_context(site)
-        self.append('monitor {object_id} $(date "+%Y%m%d%H%M%S" -d "{last_date}") "{log_file}"'.format(**context))
+        self.append('monitor {object_id} $(date "+%Y%m%d%H%M%S" -d "{last_date}") {log_file}'.format(**context))
     
     def get_context(self, site):
         return {

orchestra/apps/websites/models.py

@@ -52,6 +52,9 @@ class Website(models.Model):
     
     def get_www_log_path(self):
         context = {
+            'user_home': self.account.main_systemuser.get_home(),
+            'username': self.account.username,
+            'name': self.name,
             'unique_name': self.unique_name
         }
         return settings.WEBSITES_WEBSITE_WWW_LOG_PATH % context

orchestra/apps/websites/settings.py

@@ -82,4 +82,5 @@ WEBSITES_WEBALIZER_PATH = getattr(settings, 'WEBSITES_WEBALIZER_PATH',
 
 
 WEBSITES_WEBSITE_WWW_LOG_PATH = getattr(settings, 'WEBSITES_WEBSITE_WWW_LOG_PATH',
+    # %(user_home)s %(name)s %(unique_name)s %(username)s
     '/var/log/apache2/virtual/%(unique_name)s')

orchestra/conf/base_settings.py

@@ -168,6 +168,7 @@ FLUENT_DASHBOARD_APP_GROUPS = (
             'orchestra.apps.resources.models.Monitor',
             'orchestra.apps.services.models.Service',
             'orchestra.apps.services.models.Plan',
+            'orchestra.apps.miscellaneous.models.MiscService',
         ),
         'collapsible': True,
     }),
@@ -201,6 +202,7 @@ FLUENT_DASHBOARD_APP_ICONS = {
     'payments/transaction': 'transaction.png',
     'payments/transactionprocess': 'transactionprocess.png',
     'issues/ticket': 'Ticket_star.png',
+    'miscellaneous/miscservice': 'Misc-Misc-Box-icon.png',
     # Administration
     'djcelery/taskstate': 'taskstate.png',
     'orchestration/server': 'vps.png',

orchestra/models/fields.py

@@ -28,19 +28,21 @@ class MultiSelectField(models.CharField):
             return ','.join(value)
     
     def to_python(self, value):
-        if value is not None:
+        if value:
             if isinstance(value, list) and value[0].startswith('('):
                 # Workaround unknown bug on default model values
                 # [u"('SUPPORT'", u" 'ADMIN'", u" 'BILLING'", u" 'TECH'", u" 'ADDS'", u" 'EMERGENCY')"]
                 value = list(eval(', '.join(value)))
-            return value if isinstance(value, list) else value.split(',')
+            if isinstance(value, list):
-        return ''
+                return value
+            return value.split(',')
+        return []
     
     def contribute_to_class(self, cls, name):
         super(MultiSelectField, self).contribute_to_class(cls, name)
         if self.choices:
             def func(self, field=name, choices=dict(self.choices)):
-                return ','.join([ choices.get(value, value) for value in getattr(self, field) ])
+                return ','.join([choices.get(value, value) for value in getattr(self, field)])
             setattr(cls, 'get_%s_display' % self.name, func)
     
     def validate(self, value, model_instance):

orchestra/utils/system.py

@@ -46,8 +46,8 @@ def read_async(fd):
             return u''
 
 
-def run(command, display=False, error_codes=[0], silent=False, stdin=''):
+def runiterator(command, display=False, error_codes=[0], silent=False, stdin=''):
-    """ Subprocess wrapper for running commands """
+    """ Subprocess wrapper for running commands concurrently """
     if display:
         sys.stderr.write("\n\033[1m $ %s\033[0m\n" % command)
     
@@ -56,6 +56,7 @@ def run(command, display=False, error_codes=[0], silent=False, stdin=''):
     
     p.stdin.write(stdin)
     p.stdin.close()
+    yield
     
     make_async(p.stdout)
     make_async(p.stderr)
@@ -77,22 +78,39 @@ def run(command, display=False, error_codes=[0], silent=False, stdin=''):
         if display and stderrPiece:
             sys.stderr.write(stderrPiece)
         
-        stdout += stdoutPiece.decode("utf8")
+        return_code = p.poll()
-        stderr += stderrPiece.decode("utf8")
+        state = _AttributeUnicode(stdoutPiece.decode("utf8"))
-        returnCode = p.poll()
+        state.stderr = stderrPiece.decode("utf8")
+        state.return_code = return_code
+        yield state
         
-        if returnCode != None:
+        if return_code != None:
-            break
-    
-    out = _AttributeUnicode(stdout.strip())
-    err = _AttributeUnicode(stderr.strip())
             p.stdout.close()
             p.stderr.close()
+            raise StopIteration
+
+
+def run(command, display=False, error_codes=[0], silent=False, stdin='', async=False):
+    iterator = runiterator(command, display, error_codes, silent, stdin)
+    iterator.next()
+    if async:
+        return iterator
+    
+    stdout = ''
+    stderr = ''
+    for state in iterator:
+        stdout += state.stdout
+        stderr += state.stderr
+    
+    return_code = state.return_code
+    
+    out = _AttributeUnicode(stdout.strip())
+    err = stderr.strip()
     
     out.failed = False
-    out.return_code = returnCode
+    out.return_code = return_code
     out.stderr = err
-    if p.returncode not in error_codes:
+    if return_code not in error_codes:
         out.failed = True
         msg = "\nrun() encountered an error (return code %s) while executing '%s'\n"
         msg = msg % (p.returncode, command)