Improved orders filtering and added mailbox filtering

TODO.md

@@ -293,3 +293,6 @@ https://code.djangoproject.com/ticket/24576
 # insert settings on dashboard dynamically
 
 # convert all complex settings to string
+# @ something database names
+# password validation cracklib on change password form=?????
+# reset setting buton 

orchestra/contrib/issues/admin.py

@@ -190,7 +190,7 @@ class TicketAdmin(ChangeListDefaultFilter, ExtendedModelAdmin):
     display_creator = admin_link('creator')
     display_queue = admin_link('queue')
     display_owner = admin_link('owner')
-    updated = admin_date('updated')
+    updated = admin_date('updated_at')
     display_state = admin_colored('state', colors=STATE_COLORS, bold=False)
     display_priority = admin_colored('priority', colors=PRIORITY_COLORS, bold=False)
     
@@ -270,8 +270,8 @@ class TicketAdmin(ChangeListDefaultFilter, ExtendedModelAdmin):
         ticket.mark_as_read_by(request.user)
         context = {'title': "Issue #%i - %s" % (ticket.id, ticket.subject)}
         context.update(extra_context or {})
-        return super(TicketAdmin, self).change_view(
+        return super(TicketAdmin, self).change_view(request, object_id, form_url=form_url,
-            request, object_id, form_url, extra_context=context)
+            extra_context=context)
     
     def changelist_view(self, request, extra_context=None):
         # Hook user for bold_subject

orchestra/contrib/issues/models.py

@@ -86,7 +86,7 @@ class Ticket(models.Model):
         emails.append(self.creator.email)
         if self.owner:
             emails.append(self.owner.email)
-        for contact in self.creator.account.contacts.all():
+        for contact in self.creator.contacts.all():
             if self.queue and set(contact.email_usage).union(set(self.queue.notify)):
                 emails.append(contact.email)
         for message in self.messages.distinct('author'):

orchestra/contrib/issues/serializers.py

@@ -19,14 +19,18 @@ class MessageSerializer(serializers.HyperlinkedModelSerializer):
     def get_identity(self, data):
         return data.get('id')
     
-    def save_object(self, obj, **kwargs):
+    def create(self, validated_data):
-        obj.author = self.context['request'].user
+        validated_data['account'] = self.account
-        super(MessageSerializer, self).save_object(obj, **kwargs)
+        return super(AccountSerializerMixin, self).create(validated_data)
+    
+    def create(self, validated_data):
+        validated_data['author'] = self.context['request'].user
+        super(MessageSerializer, self).create(validated_data)
 
 
 class TicketSerializer(serializers.HyperlinkedModelSerializer):
     """ Validates if this zone generates a correct zone file """
-    messages = MessageSerializer(required=False, many=True)
+    messages = MessageSerializer(required=False, many=True, read_only=True)
     is_read = serializers.SerializerMethodField()
     
     class Meta:
@@ -40,6 +44,6 @@ class TicketSerializer(serializers.HyperlinkedModelSerializer):
     def get_is_read(self, obj):
         return obj.is_read_by(self.context['request'].user)
     
-    def save_object(self, obj, **kwargs):
+    def create(self, validated_data):
-        obj.creator = self.context['request'].user
+        validated_data['creator'] = self.context['request'].user
-        super(TicketSerializer, self).save_object(obj, **kwargs)
+        return super(TicketSerializer, self).create(validated_data)

orchestra/contrib/issues/settings.py

@@ -1,8 +1,12 @@
-from orchestra.settings import Setting
+from django.core.validators import validate_email
+
+from orchestra.settings import Setting, ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL
 
 
 ISSUES_SUPPORT_EMAILS = Setting('ISSUES_SUPPORT_EMAILS',
-    ()
+    (ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL,),
+    validators=[lambda emails: [validate_email(e) for e in emails]],
+    help_text="Includes <tt>ORCHESTRA_DEFAULT_SUPPORT_FROM_EMAIL</tt> by default",
 )
 
 

orchestra/contrib/mailboxes/backends.py

@@ -19,7 +19,21 @@ from .models import Address
 logger = logging.getLogger(__name__)
 
 
-class UNIXUserMaildirBackend(ServiceController):
+class FilteringMixin(object):
+    def generate_filter(self, mailbox, context):
+        name, content = mailbox.get_filtering()
+        if name == 'REDIRECT':
+            self.append("doveadm mailbox create -u %(user)s Spam" % context)
+        context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context
+        if content:
+            context['filtering'] = ('# %(banner)s\n' + filtering) % context
+            self.append("mkdir -p $(dirname '%(filtering_path)s')" % context)
+            self.append("echo '%(filtering)s' > %(filtering_path)s" % context)
+        else:
+            self.append("echo '' > %(filtering_path)s" % context)
+
+
+class UNIXUserMaildirBackend(FilteringMixin, ServiceController):
     """
     Assumes that all system users on this servers all mail accounts.
     If you want to have system users AND mailboxes on the same server you should consider using virtual mailboxes
@@ -41,6 +55,7 @@ class UNIXUserMaildirBackend(ServiceController):
         )
         if hasattr(mailbox, 'resources') and hasattr(mailbox.resources, 'disk'):
             self.set_quota(mailbox, context)
+        self.generate_filter(mailbox, context)
     
     def set_quota(self, mailbox, context):
         context['quota'] = mailbox.resources.disk.allocated * mailbox.resources.disk.resource.get_scale()
@@ -70,23 +85,25 @@ class UNIXUserMaildirBackend(ServiceController):
         context = {
             'user': mailbox.name,
             'group': mailbox.name,
+            'name': mailbox.name,
             'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
             'home': mailbox.get_home(),
             'initial_shell': '/dev/null',
+            'banner': self.get_banner(),
         }
         return replace(context, "'", '"')
 
 
-class DovecotPostfixPasswdVirtualUserBackend(ServiceController):
+class DovecotPostfixPasswdVirtualUserBackend(FilteringMixin, ServiceController):
     """
     WARNING: This backends is not fully implemented
     """
+    DEFAULT_GROUP = 'postfix'
+    
     verbose_name = _("Dovecot-Postfix virtualuser")
     model = 'mailboxes.Mailbox'
     # TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data
     
-    DEFAULT_GROUP = 'postfix'
-    
     def set_user(self, context):
         self.append(textwrap.dedent("""
             if [[ $( grep "^%(user)s:" %(passwd_path)s ) ]]; then
@@ -106,17 +123,6 @@ class DovecotPostfixPasswdVirtualUserBackend(ServiceController):
             fi""") % context
         )
     
-    def generate_filter(self, mailbox, context):
-        self.append("doveadm mailbox create -u %(user)s Spam" % context)
-        context['filtering_path'] = settings.MAILBOXES_SIEVE_PATH % context
-        filtering = mailbox.get_filtering()
-        if filtering:
-            context['filtering'] = '# %(banner)s\n' + filtering
-            self.append("mkdir -p $(dirname '%(filtering_path)s')" % context)
-            self.append("echo '%(filtering)s' > %(filtering_path)s" % context)
-        else:
-            self.append("rm -f %(filtering_path)s" % context)
-    
     def save(self, mailbox):
         context = self.get_context(mailbox)
         self.set_user(context)

orchestra/contrib/mailboxes/models.py

@@ -59,10 +59,10 @@ class Mailbox(models.Model):
             self.custom_filtering = ''
     
     def get_filtering(self):
-        __, filtering = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering]
+        name, content = settings.MAILBOXES_MAILBOX_FILTERINGS[self.filtering]
-        if isinstance(filtering, str):
+        if callable(content):
-            return filtering
+            return content(self)
-        return filtering(self)
+        return (name, content)
     
     def delete(self, *args, **kwargs):
         super(Mailbox, self).delete(*args, **kwargs)

orchestra/contrib/mailboxes/settings.py

@@ -7,8 +7,8 @@ from orchestra.core.validators import validate_name
 from orchestra.settings import ORCHESTRA_BASE_DOMAIN, Setting
 
 
-_names = ('name', 'username')
+_names = ('name', 'username',)
-_backend_names = _names + ('group', 'home')
+_backend_names = _names + ('user', 'group', 'home')
 
 
 MAILBOXES_DOMAIN_MODEL = Setting('MAILBOXES_DOMAIN_MODEL', 'domains.Domain',
@@ -24,7 +24,9 @@ MAILBOXES_HOME = Setting('MAILBOXES_HOME',
 
 
 MAILBOXES_SIEVE_PATH = Setting('MAILBOXES_SIEVE_PATH',
-    os.path.join(MAILBOXES_HOME, 'Maildir/sieve/orchestra.sieve')
+    os.path.join(MAILBOXES_HOME, 'Maildir/sieve/orchestra.sieve'),
+    help_text="Available fromat names: <tt>%s</tt>" % ', '.join(_names),
+    validators=[Setting.string_format_validator(_backend_names)],
 )
 
 

orchestra/contrib/mailboxes/validators.py

@@ -50,19 +50,18 @@ def validate_forward(value):
 
 
 def validate_sieve(value):
-    sieve_name = '%s.sieve' % hashlib.md5(value).hexdigest()
+    sieve_name = '%s.sieve' % hashlib.md5(value.encode('utf8')).hexdigest()
     path = os.path.join(settings.MAILBOXES_SIEVETEST_PATH, sieve_name)
-    with open(path, 'wb') as f:
+    with open(path, 'w') as f:
         f.write(value)
     context = {
         'orchestra_root': paths.get_orchestra_dir()
     }
     sievetest = settings.MAILBOXES_SIEVETEST_BIN_PATH % context
-    try:
+    test = run(' '.join([sievetest, path, '/dev/null']), silent=True)
-        test = run(' '.join([sievetest, path, '/dev/null']), display=False)
+    if test.return_code:
-    except CommandError:
         errors = []
-        for line in test.stderr.splitlines():
+        for line in test.stderr.decode('utf8').splitlines():
             error = re.match(r'^.*(line\s+[0-9]+:.*)', line)
             if error:
                 errors += error.groups()

orchestra/contrib/orchestration/backends.py

@@ -170,6 +170,9 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount):
         else:
             self.cmd_section[-1][1].append(cmd)
     
+    def get_context(self, obj):
+        return {}
+    
     def prepare(self):
         """
         hook for executing something at the beging

orchestra/contrib/orchestration/tasks.py

@@ -8,7 +8,7 @@ from .models import BackendLog
 
 
 @periodic_task(run_every=crontab(hour=7, minute=30, day_of_week=1))
-def backend_logs_cleanup(run_every=run_every):
+def backend_logs_cleanup():
     days = settings.ORCHESTRATION_BACKEND_CLEANUP_DAYS
     epoch = timezone.now()-timedelta(days=days)
     BackendLog.objects.filter(created_at__lt=epoch).delete()

orchestra/contrib/orders/admin.py

@@ -51,7 +51,7 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin):
         'id', 'service_link', 'account_link', 'content_object_link',
         'display_registered_on', 'display_billed_until', 'display_cancelled_on', 'display_metric'
     )
-    list_filter = (ActiveOrderListFilter, BilledOrderListFilter, IgnoreOrderListFilter, 'service',)
+    list_filter = (ActiveOrderListFilter, IgnoreOrderListFilter, 'service', BilledOrderListFilter)
     default_changelist_filters = (
         ('ignore', '0'),
     )
@@ -93,6 +93,22 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin):
         return metric.value
     display_metric.short_description = _("Metric")
 
+#    def get_changelist(self, request, **kwargs):
+#        ChangeList = super(OrderAdmin, self).get_changelist(request, **kwargs)
+#        class OrderFilterChangeList(ChangeList):
+#            def get_filters(self, request):
+#                filters = super(OrderFilterChangeList, self).get_filters(request)
+#                tail = []
+#                filters_copy = []
+#                for list_filter in filters[0]:
+#                    if getattr(list_filter, 'apply_last', False):
+#                        tail.append(list_filter)
+#                    else:
+#                        filters_copy.append(list_filter)
+#                filters = ((filters_copy+tail),) + filters[1:]
+#                return filters
+#        return OrderFilterChangeList
+
 
 class MetricStorageAdmin(admin.ModelAdmin):
     list_display = ('order', 'value', 'created_on', 'updated_on')

orchestra/contrib/orders/filters.py

@@ -1,9 +1,13 @@
+from datetime import timedelta, datetime
+
 from django.contrib.admin import SimpleListFilter
-from django.db.models import Q
+from django.db.models import Q, Prefetch, F
 from django.utils import timezone
 from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 
+from .models import MetricStorage, Order
+
 
 class ActiveOrderListFilter(SimpleListFilter):
     """ Filter tickets by created_by according to request.user """
@@ -27,7 +31,8 @@ class ActiveOrderListFilter(SimpleListFilter):
 class BilledOrderListFilter(SimpleListFilter):
     """ Filter tickets by created_by according to request.user """
     title = _("billed")
-    parameter_name = 'pending'
+    parameter_name = 'billed'
+#    apply_last = True
     
     def lookups(self, request, model_admin):
         return (
@@ -37,12 +42,33 @@ class BilledOrderListFilter(SimpleListFilter):
     
     def queryset(self, request, queryset):
         if self.value() == 'yes':
-            return queryset.filter(billed_until__isnull=False,
+            return queryset.filter(billed_until__isnull=False, billed_until__gte=timezone.now())
-                    billed_until__gte=timezone.now())
         elif self.value() == 'no':
+            mindelta = timedelta(days=2) # TODO
+            metric_pks = []
+            prefetch_valid_metrics = Prefetch('metrics', to_attr='valid_metrics',
+                queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'),
+                    created_on__lte=(F('updated_on')-mindelta))
+            )
+            prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric',
+                queryset=MetricStorage.objects.filter(order__billed_on__isnull=False,
+                    created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on'))
+            )
+            metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True)
+            for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric):
+                if len(order.billed_metric) != 1:
+                    raise ValueError("Data inconsistency.")
+                billed_metric = order.billed_metric[0].value
+                for metric in order.valid_metrics:
+                    if metric.created_on <= order.billed_on:
+                        raise ValueError("This value should already be filtered on the prefetch query.")
+                    if metric.value > billed_metric:
+                        metric_pks.append(order.pk)
+                        break
             return queryset.filter(
-                Q(billed_until__isnull=True) |
+                Q(pk__in=metric_pks) | Q(
-                Q(billed_until__lt=timezone.now())
+                    Q(billed_until__isnull=True) | Q(billed_until__lt=timezone.now())
+                )
             )
         return queryset
 

orchestra/contrib/webapps/types/php.py

@@ -80,29 +80,36 @@ class PHPApp(AppType):
             for webapp in webapps:
                 if webapp.type_instance.get_php_version() == php_version:
                     options += list(webapp.options.all())
-        php_options = [option.name for option in self.get_php_options()]
+        init_vars = OrderedDict((opt.name, opt.value) for opt in options)
-        enabled_functions = set()
+        # Enabled functions
-        for opt in options:
+        enabled_functions = init_vars.pop('enabled_functions', None)
-            if opt.name in php_options:
-                if opt.name == 'enable_functions':
-                    enabled_functions = enabled_functions.union(set(opt.value.split(',')))
-                else:
-                    init_vars[opt.name] = opt.value
         if enabled_functions:
+            enabled_functions = set(enabled_functions.split(','))
             disabled_functions = []
             for function in self.PHP_DISABLED_FUNCTIONS:
                 if function not in enabled_functions:
                     disabled_functions.append(function)
             init_vars['disable_functions'] = ','.join(disabled_functions)
+        # process timeout
         timeout = self.instance.options.filter(name='timeout').first()
         if timeout:
             # Give a little slack here
             timeout = str(int(timeout.value)-2)
             init_vars['max_execution_time'] = timeout
+        # Custom error log
         if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars:
             context = self.get_directive_context()
             error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context)
             init_vars['error_log'] = error_log_path
+        # auto update max_post_size
+        if 'upload_max_filesize' in init_vars:
+            upload_max_filesize = init_vars['upload_max_filesize']
+            post_max_size = init_vars.get('post_max_size', '0')
+            upload_max_filesize_value = eval(upload_max_filesize.replace('M', '*1024'))
+            post_max_size_value = eval(post_max_size.replace('M', '*1024'))
+            init_vars['post_max_size'] = post_max_size
+            if upload_max_filesize_value > post_max_size_value:
+                init_vars['post_max_size'] = upload_max_filesize
         return init_vars
     
     def get_directive_context(self):

orchestra/utils/options.py

@@ -22,7 +22,7 @@ def send_email_template(template, context, to, email_from=None, html=None, attac
     
     if not 'site' in context:
         from orchestra import settings
-        url = urlparse.urlparse(settings.ORCHESTRA_SITE_URL)
+        url = urlparse(settings.ORCHESTRA_SITE_URL)
         context['site'] = {
             'name': settings.ORCHESTRA_SITE_NAME,
             'scheme': url.scheme,