Custom mailbox filters and other random inprovements0

TODO.md

@@ -60,7 +60,7 @@
 
 * print open invoices as proforma?
 
-* env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python3 manage.py test orchestra.contrib.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture
+* env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python3 manage.py test orchestra.contrib.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture --keepdb
 
 * ForeignKey.swappable
 * Field.editable
@@ -386,4 +386,26 @@ http://wiki2.dovecot.org/Pigeonhole/Sieve/Examples
 Bash/Python/PHPBackend
 
 
-# Gandi sync domains cancelled
+# bill action view on a separate process. check memory consumption without debug (236m)
+
+# services.handler as generator in order to save memory? not swell like a balloon
+# mailboxes group username instead of mainuser
+
+import uwsgi
+from uwsgidecorators import timer
+from django.utils import autoreload
+
+@timer(3)
+def change_code_gracefull_reload(sig):
+    if autoreload.code_changed():
+        uwsgi.reload()
+# using kill to send the signal
+kill -HUP `cat /tmp/project-master.pid`
+# or the convenience option --reload
+uwsgi --reload /tmp/project-master.pid
+# or if uwsgi was started with touch-reload=/tmp/somefile
+touch /tmp/somefile
+
+
+
+# Pending vs bill(): get_billing_point() returns the next billing point, no matter if nbp > now(). pending filter filters by billed_until < now()

orchestra/contrib/mailboxes/api.py

@@ -10,6 +10,7 @@ from .serializers import AddressSerializer, MailboxSerializer
 class AddressViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet):
     queryset = Address.objects.select_related('domain').prefetch_related('mailboxes').all()
     serializer_class = AddressSerializer
+    filter_fields = ('domain', 'mailboxes__name')
 
 
 class MailboxViewSet(LogApiMixin, SetPasswordApiMixin, AccountApiMixin, viewsets.ModelViewSet):

orchestra/contrib/mailboxes/backends.py

@@ -60,9 +60,6 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
     
     verbose_name = _("UNIX maildir user")
     model = 'mailboxes.Mailbox'
-    doc_settings = (settings,
-        ('MAILBOXES_USE_ACCOUNT_AS_GROUP',)
-    )
     
     def save(self, mailbox):
         context = self.get_context(mailbox)
@@ -123,10 +120,9 @@ class UNIXUserMaildirBackend(SieveFilteringMixin, ServiceController):
         super(UNIXUserMaildirBackend, self).commit()
     
     def get_context(self, mailbox):
-        account_as_group = settings.MAILBOXES_USE_ACCOUNT_AS_GROUP
         context = {
             'user': mailbox.name,
-            'group': mailbox.account.username if account_as_group else mailbox.name,
+            'group': mailbox.name,
             'name': mailbox.name,
             'password': mailbox.password if mailbox.active else '*%s' % mailbox.password,
             'home': mailbox.get_home(),

orchestra/contrib/mailboxes/settings.py

@@ -48,12 +48,6 @@ MAILBOXES_SIEVETEST_BIN_PATH = Setting('MAILBOXES_SIEVETEST_BIN_PATH',
 )
 
 
-MAILBOXES_USE_ACCOUNT_AS_GROUP = Setting('MAILBOXES_USE_ACCOUNT_AS_GROUP',
-    False,
-    help_text="Group used for system user based mailboxes. If <tt>False</tt> mailbox.name will be used as group."
-)
-
-
 MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH = Setting('MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH',
     '/etc/postfix/virtual_mailboxes'
 )
@@ -81,34 +75,99 @@ MAILBOXES_PASSWD_PATH = Setting('MAILBOXES_PASSWD_PATH',
 )
 
 
+MAILBOXES_SPAM_SCORE_HEADER = Setting('MAILBOXES_SPAM_SCORE_HEADER',
+    'X-Spam-Score'
+)
+
+
+MAILBOXES_SPAM_SCORE_SYMBOL = Setting('MAILBOXES_SPAM_SCORE_SYMBOL',
+    '',
+    help_text="Blank for numeric spam score.",
+)
+
+
 MAILBOXES_MAILBOX_FILTERINGS = Setting('MAILBOXES_MAILBOX_FILTERINGS',
     {
         # value: (verbose_name, filter)
         'DISABLE': (_("Disable"), ''),
-        'REJECT': (mark_safe_lazy(_("Reject spam (Score&ge;9)")), textwrap.dedent("""\
+        'REJECT': (mark_safe_lazy(_("Reject spam (Score&ge;8)")), (
-            require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
+            textwrap.dedent("""\
-            if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "9" {
+                if header :contains "%(score_header)s" "%(score_value)s" {
-                discard;
+                    discard;
-                stop;
+                    stop;
-            }""")),
+                }""") if MAILBOXES_SPAM_SCORE_SYMBOL else
-        'REJECT5': (mark_safe_lazy(_("Reject spam (Score&ge;5)")), textwrap.dedent("""\
+            textwrap.dedent("""\
-            require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
+                require ["relational","comparator-i;ascii-numeric"];
-            if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {
+                if allof (
-                discard;
+                   not header :matches "%(score_header)s" "-*",
-                stop;
+                   header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "8" )
-            }""")),
+                {
-        'REDIRECT': (mark_safe_lazy(_("Archive spam (Score&ge;9)")), textwrap.dedent("""\
+                    discard;
-            require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
+                    stop;
-            if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "9" {
+                }""")) % {
-                fileinto "Spam";
+                    'score_header': MAILBOXES_SPAM_SCORE_HEADER,
-                stop;
+                    'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*8
-            }""")),
+                }
-        'REDIRECT5': (mark_safe_lazy(_("Archive spam (Score&ge;5)")), textwrap.dedent("""\
+            ),
-            require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
+        'REJECT5': (mark_safe_lazy(_("Reject spam (Score&ge;5)")), (
-            if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {
+            textwrap.dedent("""\
-                fileinto "Spam";
+                if header :contains "%(score_header)s" "%(score_value)s" {
-                stop;
+                    discard;
-            }""")),
+                    stop;
+                }""") if MAILBOXES_SPAM_SCORE_SYMBOL else
+            textwrap.dedent("""\
+                require ["relational","comparator-i;ascii-numeric"];
+                if allof (
+                   not header :matches "%(score_header)s" "-*",
+                   header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "5" )
+                {
+                    discard;
+                    stop;
+                }""")) % {
+                    'score_header': MAILBOXES_SPAM_SCORE_HEADER,
+                    'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*5
+                }
+            ),
+        'REDIRECT': (mark_safe_lazy(_("Archive spam (Score&ge;8)")), (
+            textwrap.dedent("""\
+                require "fileinto";
+                if header :contains "%(score_header)s" "%(score_value)s" {
+                    fileinto "Spam";
+                    stop;
+                }""") if MAILBOXES_SPAM_SCORE_SYMBOL else
+            textwrap.dedent("""\
+                require ["fileinto","relational","comparator-i;ascii-numeric"];
+                if allof (
+                   not header :matches "%(score_header)s" "-*",
+                   header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "8" )
+                {
+                    fileinto "Spam";
+                    stop;
+                }""")) % {
+                    'score_header': MAILBOXES_SPAM_SCORE_HEADER,
+                    'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*8
+                }
+            ),
+        'REDIRECT5': (mark_safe_lazy(_("Archive spam (Score&ge;5)")), (
+            textwrap.dedent("""\
+                require "fileinto";
+                if header :contains "%(score_header)s" "%(score_value)s" {
+                    fileinto "Spam";
+                    stop;
+                }""") if MAILBOXES_SPAM_SCORE_SYMBOL else
+            textwrap.dedent("""\
+                require ["fileinto","relational","comparator-i;ascii-numeric"];
+                if allof (
+                   not header :matches "%(score_header)s" "-*",
+                   header :value "ge" :comparator "i;ascii-numeric" "%(score_header)s" "5" )
+                {
+                    fileinto "Spam";
+                    stop;
+                }""")) % {
+                    'score_header': MAILBOXES_SPAM_SCORE_HEADER,
+                    'score_value': MAILBOXES_SPAM_SCORE_SYMBOL*5
+                }
+            ),
         'CUSTOM': (_("Custom filtering"), lambda mailbox: mailbox.custom_filtering),
     }
 )

orchestra/contrib/orders/actions.py

@@ -29,7 +29,10 @@ class BillSelectedOrders(object):
             'queryset': queryset,
             'action_checkbox_name': admin.helpers.ACTION_CHECKBOX_NAME,
         }
-        return self.set_options(request)
+        ret = self.set_options(request)
+        del(self.queryset)
+        del(self.context)
+        return ret
     
     def set_options(self, request):
         form = BillSelectedOptionsForm()
@@ -42,7 +45,6 @@ class BillSelectedOrders(object):
                     proforma=form.cleaned_data['proforma'],
                     new_open=form.cleaned_data['new_open'],
                 )
-                print(self.options)
                 if int(request.POST.get('step')) != 3:
                     return self.select_related(request)
                 else:
@@ -142,4 +144,3 @@ def mark_as_not_ignored(modeladmin, request, queryset):
         _("%i selected orders have been marked as not ignored.") % num,
         num)
     modeladmin.message_user(request, msg)
-

orchestra/contrib/orders/forms.py

@@ -44,6 +44,7 @@ class BillSelectRelatedForm(AdminFormMixin, forms.Form):
 #            initial=False, required=False, help_text=_("The price may vary "
 #                "depending on the billed orders. This options designates whether "
 #                "all existing orders will be used for price computation or not."))
+    select_all = forms.BooleanField(label=_("Select all"), required=False)
     selected_related = forms.ModelMultipleChoiceField(label=_("Related orders"),
         queryset=Order.objects.none(), widget=forms.CheckboxSelectMultiple,
         required=False)

orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html

@@ -12,6 +12,20 @@
 </style>
 {% endblock %}
 
+{% block extrahead %}
+{{ block.super }}
+<script src="{% static "admin/js/jquery.min.js" %}" type="text/javascript"></script>
+<script src="{% static "admin/js/jquery.init.js" %}" type="text/javascript"></script>
+<script>
+var $ = django.jQuery;
+
+$(document).ready( function () {
+  $('#id_select_all').click( function() {
+    $(":checkbox").attr('checked', $(this).is(':checked'));
+  });
+});
+</script>
+{% endblock %}
 
 {% block breadcrumbs %}
 <div class="breadcrumbs">

orchestra/contrib/plans/ratings.py

@@ -24,7 +24,7 @@ def _compute_steps(rates, metric):
             quantity = metric - accumulated
             next_barrier = quantity
         else:
-            quantity = rates[ix+1].quantity - rates[ix].quantity
+            quantity = rates[ix+1].quantity - max(rates[ix].quantity, 1)
             next_barrier = quantity
             if rates[ix+1].price > rates[ix].price:
                 quantity *= fold
@@ -52,13 +52,13 @@ def _standardize(rates):
     std_rates = []
     minimal = rates[0].quantity
     for rate in rates:
-        if rate.quantity == 0:
+        #if rate.quantity == 0:
-            rate.quantity = 1
+        #    rate.quantity = 1
-        elif rate.quantity == minimal and rate.quantity > 1:
+        if rate.quantity == minimal and rate.quantity > 0:
             service = rate.service
             rate_class = type(rate)
             std_rates.append(
-                rate_class(service=service, plan=rate.plan, quantity=1, price=service.nominal_price)
+                rate_class(service=service, plan=rate.plan, quantity=0, price=service.nominal_price)
             )
         std_rates.append(rate)
     return std_rates

orchestra/contrib/resources/admin.py

@@ -1,7 +1,8 @@
 from django.conf.urls import url
 from django.contrib import admin, messages
+from django.contrib.contenttypes.admin import GenericTabularInline
+from django.contrib.contenttypes.forms import BaseGenericInlineFormSet
 from django.contrib.admin.utils import unquote
-from django.contrib import contenttypes 
 from django.core.urlresolvers import reverse
 from django.shortcuts import redirect
 from django.utils.functional import cached_property
@@ -185,7 +186,7 @@ admin.site.register(MonitorData, MonitorDataAdmin)
 # Mokey-patching
 
 def resource_inline_factory(resources):
-    class ResourceInlineFormSet(contenttypes.forms.BaseGenericInlineFormSet):
+    class ResourceInlineFormSet(BaseGenericInlineFormSet):
         def total_form_count(self, resources=resources):
             return len(resources)
         
@@ -225,7 +226,7 @@ def resource_inline_factory(resources):
                 forms.append(self._construct_form(i, resource=resource))
             return forms
     
-    class ResourceInline(contenttypes.admin.GenericTabularInline):
+    class ResourceInline(GenericTabularInline):
         model = ResourceData
         verbose_name_plural = _("resources")
         form = ResourceForm

orchestra/contrib/services/handlers.py

@@ -136,8 +136,8 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
             return eval(self.order_description, safe_locals)
     
     def get_billing_point(self, order, bp=None, **options):
-        not_cachable = self.billing_point == self.FIXED_DATE and options.get('fixed_point')
+        cachable = bool(self.billing_point == self.FIXED_DATE and not options.get('fixed_point'))
-        if not_cachable or bp is None:
+        if not cachable or bp is None:
             bp = options.get('billing_point') or timezone.now().date()
             if not options.get('fixed_point'):
                 msg = ("Support for '%s' period and '%s' point is not implemented"

orchestra/contrib/services/tests/functional_tests/test_domain.py

@@ -25,7 +25,7 @@ class DomainBillingTest(BaseTestCase):
             nominal_price=10
         )
         plan = Plan.objects.create(is_default=True, name='Default')
-        service.rates.create(plan=plan, quantity=1, price=0)
+        service.rates.create(plan=plan, quantity=0, price=0)
         service.rates.create(plan=plan, quantity=2, price=10)
         service.rates.create(plan=plan, quantity=4, price=9)
         service.rates.create(plan=plan, quantity=6, price=6)

orchestra/contrib/services/tests/functional_tests/test_traffic.py

@@ -19,6 +19,7 @@ class FTPTrafficMonitor(ServiceMonitor):
 
 class BaseTrafficBillingTest(BaseTestCase):
     TRAFFIC_METRIC = 'account.resources.traffic.used'
+    DEPENDENCIES = ('orchestra.contrib.resources',)
     
     def create_traffic_service(self):
         service = Service.objects.create(

orchestra/contrib/settings/admin.py

@@ -67,11 +67,10 @@ class SettingView(generic.edit.FormView):
                 context = self.get_context_data(form=form)
                 context['diff'] = diff
                 return self.render_to_response(context)
-            
+            n = len(changes)
             # Save changes
             parser.save(changes)
             sys.touch_wsgi()
-            n = len(changes)
             context = {
                 'message': ngettext(
                     _("One change successfully applied, orchestra is being restarted."),

orchestra/contrib/settings/parser.py

@@ -1,4 +1,5 @@
 import ast
+import copy
 import os
 import re
 
@@ -98,6 +99,7 @@ def apply(changes, settings_file=get_settings_file()):
     """ returns settings_file content with applied changes """
     updates = _find_updates(changes, settings_file)
     content = []
+    _changes = copy.copy(changes)
     inside = False
     lineno = None
     if updates:
@@ -107,7 +109,7 @@ def apply(changes, settings_file=get_settings_file()):
         for num, line in enumerate(handler.readlines(), 1):
             line = line.rstrip()
             if num == lineno:
-                value = changes.pop(name)
+                value = _changes.pop(name)
                 line = _format_setting(name, value)
                 if line:
                     content.append(line)
@@ -134,7 +136,7 @@ def apply(changes, settings_file=get_settings_file()):
                     inside = False
     
     # insert new variables at the end of file
-    for name, value in changes.items():
+    for name, value in _changes.items():
         content.append(_format_setting(name, value))
     return '\n'.join(content)
 

orchestra/contrib/settings/templates/admin/settings/reload.html

@@ -17,7 +17,7 @@
                 right: 0;
                 margin: auto;
                 margin-top: 100px;
-                        }
+            }
             .alert-box span {
                 font-weight:bold;
                 text-transform:uppercase;
@@ -41,14 +41,14 @@
                 count--;
               }
             }
-            var count = 4;
+            var count = 3;
             var timer = setInterval(function() { handleTimer(count); }, 1000);
         </script>
     </head>
     <body>
-        <meta http-equiv="refresh" content="6">
+        <meta http-equiv="refresh" content="3">
         <div>
-        <div class="alert-box warning"><span>notice: </span>{{ message }}<br> Refreshing in <span id="count_num">5</span></span>.</div>
+        <div class="alert-box warning"><span>notice: </span>{{ message }}<br> Refreshing in <span id="count_num">2</span></span>.</div>
         </div>
     </body>
 </html>

orchestra/contrib/webapps/backends/php.py

@@ -142,7 +142,11 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
             }
             if [[ $is_last -eq 1 ]]; then
                 if [[ $UPDATED_APACHE -eq 1 || "$state" =~ .*RESTART$ ]]; then
-                    service apache2 status && service apache2 reload || service apache2 start
+                    if [[ $(service apache2 status) ]]; then
+                        service apache2 reload
+                    else
+                        service apache2 start
+                    fi
                 fi
                 rm /dev/shm/restart.apache2.locked
             else

orchestra/contrib/websites/api.py

@@ -11,7 +11,7 @@ from .serializers import WebsiteSerializer
 class WebsiteViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet):
     queryset = Website.objects.prefetch_related('domains', 'content_set__webapp', 'directives').all()
     serializer_class = WebsiteSerializer
-    filter_fields = ('name',)
+    filter_fields = ('name', 'domains__name')
     
     def options(self, request):
         metadata = super(WebsiteViewSet, self).options(request)

orchestra/contrib/websites/backends/apache.py

@@ -67,7 +67,8 @@ class Apache2Backend(ServiceController):
                 SuexecUserGroup {{ user }} {{ group }}\
             {% for line in extra_conf.splitlines %}
                 {{ line | safe }}{% endfor %}
-            </VirtualHost>""")
+            </VirtualHost>
+            """)
         ).render(Context(context))
     
     def render_redirect_https(self, context):
@@ -84,7 +85,8 @@ class Apache2Backend(ServiceController):
                 RewriteEngine On
                 RewriteCond %{HTTPS} off
                 RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI}
-            </VirtualHost>""")
+            </VirtualHost>
+            """)
         ).render(Context(context))
     
     def save(self, site):
@@ -111,7 +113,7 @@ class Apache2Backend(ServiceController):
                 }""") % context
             )
         if context['server_name'] and site.active:
-            self.append(textwrap.dedent("""\
+            self.append(textwrap.dedent("""
                 # Enable site %(site_name)s
                 if [[ ! -f %(sites_enabled)s ]]; then
                     a2ensite %(site_unique_name)s.conf
@@ -119,7 +121,7 @@ class Apache2Backend(ServiceController):
                 fi""") % context
             )
         else:
-            self.append(textwrap.dedent("""\
+            self.append(textwrap.dedent("""
                 # Disable site %(site_name)s
                 if [[ -f %(sites_enabled)s ]]; then
                     a2dissite %(site_unique_name)s.conf;
@@ -160,7 +162,11 @@ class Apache2Backend(ServiceController):
             }
             if [[ $is_last -eq 1 ]]; then
                 if [[ $UPDATED_APACHE -eq 1 || "$state" =~ .*RESTART$ ]]; then
-                    service apache2 status && service apache2 reload || service apache2 start
+                    if [[ $(service apache2 status) ]]; then
+                        service apache2 reload
+                    else
+                        service apache2 start
+                    fi
                 fi
                 rm /dev/shm/restart.apache2.locked
             else

orchestra/utils/sys.py

@@ -212,6 +212,6 @@ class LockFile(object):
             self.release()
 
 
-def touch_wsgi(delay=5):
+def touch_wsgi(delay=0):
     from . import paths
     run('{ sleep %i && touch %s/wsgi.py; } &' % (delay, paths.get_project_dir()), async=True)