Initial lists tests

TODO.md

@@ -166,3 +166,7 @@ APPS app?
 
 
 * disable account triggers save on cascade to execute backends save(update_field=[])
+
+
+* validate database user names
+* multiple domains creation; line separated domains

orchestra/admin/forms.py

@@ -115,7 +115,6 @@ class AdminPasswordChangeForm(forms.Form):
         for ix, rel in enumerate(self.related):
             password = self.cleaned_data['password1_%s' % ix]
             if password:
-                print password
                 set_password = getattr(rel, 'set_password')
                 set_password(password)
                 if commit:

orchestra/admin/utils.py

@@ -1,4 +1,5 @@
 import datetime
+import inspect
 from functools import wraps
 
 from django.conf import settings
@@ -31,30 +32,26 @@ def get_modeladmin(model, import_module=True):
         return get_modeladmin(model, import_module=False)
 
 
-def insertattr(model, name, value, weight=0):
+def insertattr(model, name, value):
     """ Inserts attribute to a modeladmin """
-    modeladmin_class = model
+    modeladmin = None
     if models.Model in model.__mro__:
-        modeladmin_class = type(get_modeladmin(model))
+        modeladmin = get_modeladmin(model)
+        modeladmin_class = type(modeladmin)
+    elif not inspect.isclass(model):
+        modeladmin = model
+        modeladmin_class = type(modeladmin)
+    else:
+        modeladmin_class = model
     # Avoid inlines defined on parent class be shared between subclasses
     # Seems that if we use tuples they are lost in some conditions like changing
     # the tuple in modeladmin.__init__
     if not getattr(modeladmin_class, name):
         setattr(modeladmin_class, name, [])
-    
+    setattr(modeladmin_class, name, list(getattr(modeladmin_class, name))+[value])
-    inserted_attrs = getattr(modeladmin_class, '__inserted_attrs__', {})
+    if modeladmin:
-    if not name in inserted_attrs:
+        # make sure class and object share the same attribute, to avoid wierd bugs
-        weights = {}
+        setattr(modeladmin, name, getattr(modeladmin_class, name))
-        if hasattr(modeladmin_class, 'weights') and name in modeladmin_class.weights:
-            weights = modeladmin_class.weights.get(name)
-        inserted_attrs[name] = [
-            (attr, weights.get(attr, 0)) for attr in getattr(modeladmin_class, name)
-        ]
-    
-    inserted_attrs[name].append((value, weight))
-    inserted_attrs[name].sort(key=lambda a: a[1])
-    setattr(modeladmin_class, name, [ attr[0] for attr in inserted_attrs[name] ])
-    setattr(modeladmin_class, '__inserted_attrs__', inserted_attrs)
 
 
 def wrap_admin_view(modeladmin, view):
@@ -84,7 +81,7 @@ def action_to_view(action, modeladmin):
         response = action(modeladmin, request, queryset)
         if not response:
             opts = modeladmin.model._meta
-            url = 'admin:%s_%s_change' % (opts.app_label, opts.module_name)
+            url = 'admin:%s_%s_change' % (opts.app_label, opts.model_name)
             return redirect(url, object_id)
         return response
     return action_view

orchestra/apps/databases/admin.py

@@ -89,7 +89,7 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
         if not change:
             user = form.cleaned_data['user']
             if not user:
-                user = DatabaseUser.objects.create(
+                user = DatabaseUser(
                     username=form.cleaned_data['username'],
                     type=obj.type,
                     account_id = obj.account.pk,

orchestra/apps/databases/backends.py

@@ -41,28 +41,28 @@ class MySQLUserBackend(ServiceController):
     verbose_name = "MySQL user"
     model = 'databases.DatabaseUser'
     
-    def save(self, database):
+    def save(self, user):
-        if database.type == database.MYSQL:
+        if user.type == user.MYSQL:
-            context = self.get_context(database)
+            context = self.get_context(user)
             self.append(
-                "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";'" % context
+                "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context
             )
             self.append(
                 "mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
                 "  WHERE User=\"%(username)s\";'" % context
             )
     
-    def delete(self, database):
+    def delete(self, user):
-        if database.type == database.MYSQL:
+        if user.type == user.MYSQL:
             context = self.get_context(database)
             self.append(
                 "mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
             )
     
-    def get_context(self, database):
+    def get_context(self, user):
         return {
-            'username': database.username,
+            'username': user.username,
-            'password': database.password,
+            'password': user.password,
             'host': settings.DATABASES_DEFAULT_HOST,
         }
 

orchestra/apps/databases/forms.py

@@ -30,9 +30,9 @@ class DatabaseUserCreationForm(forms.ModelForm):
     
     def save(self, commit=True):
         user = super(DatabaseUserCreationForm, self).save(commit=False)
-        user.set_password(self.cleaned_data["password1"])
+#        user.set_password(self.cleaned_data["password1"])
-        if commit:
+#        if commit:
-            user.save()
+#            user.save()
         return user
 
 
@@ -89,16 +89,16 @@ class DatabaseCreationForm(DatabaseUserCreationForm):
     
     def save(self, commit=True):
         db = super(DatabaseUserCreationForm, self).save(commit=False)
-        user = self.cleaned_data['user']
+#        if commit:
-        if commit:
+#            user = self.cleaned_data['user']
-            if not user:
+#            if not user:
-                user = DatabaseUser(
+#                user = DatabaseUser(
-                    username=self.cleaned_data['username'],
+#                    username=self.cleaned_data['username'],
-                    type=self.cleaned_data['type'],
+#                    type=self.cleaned_data['type'],
-                )
+#                )
-                user.set_password(self.cleaned_data["password1"])
+#                user.set_password(self.cleaned_data["password1"])
-            user.save()
+#            user.save()
-            role, __ = Role.objects.get_or_create(database=db, user=user)
+#            role, __ = Role.objects.get_or_create(database=db, user=user)
         return db
 
 

orchestra/apps/databases/models.py

@@ -13,7 +13,7 @@ class Database(models.Model):
     MYSQL = 'mysql'
     POSTGRESQL = 'postgresql'
     
-    name = models.CharField(_("name"), max_length=128,
+    name = models.CharField(_("name"), max_length=64, # MySQL limit
             validators=[validators.validate_name])
     users = models.ManyToManyField('databases.DatabaseUser',
             verbose_name=_("users"),
@@ -53,9 +53,7 @@ class Role(models.Model):
             msg = _("Database and user type doesn't match")
             raise validators.ValidationError(msg)
         roles = self.database.roles.values('id')
-        print roles
         if not roles or (len(roles) == 1 and roles[0].id == self.id):
-            print 'seld'
             self.is_owner = True
 
 
@@ -63,9 +61,9 @@ class DatabaseUser(models.Model):
     MYSQL = 'mysql'
     POSTGRESQL = 'postgresql'
     
-    username = models.CharField(_("username"), max_length=128,
+    username = models.CharField(_("username"), max_length=16, # MySQL usernames 16 char long
             validators=[validators.validate_name])
-    password = models.CharField(_("password"), max_length=128)
+    password = models.CharField(_("password"), max_length=256)
     type = models.CharField(_("type"), max_length=32,
             choices=settings.DATABASES_TYPE_CHOICES,
             default=settings.DATABASES_DEFAULT_TYPE)
@@ -87,8 +85,7 @@ class DatabaseUser(models.Model):
             # MySQL stores sha1(sha1(password).binary).hex
             binary = hashlib.sha1(password).digest()
             hexdigest = hashlib.sha1(binary).hexdigest()
-            password = '*%s' % hexdigest.upper()
+            self.password = '*%s' % hexdigest.upper()
-            self.password = password
         else:
             raise TypeError("Database type '%s' not supported" % self.type)
 

orchestra/apps/databases/serializers.py

@@ -8,7 +8,7 @@ from orchestra.core.validators import validate_password
 from .models import Database, DatabaseUser, Role
 
 
-class UserSerializer(serializers.HyperlinkedModelSerializer):
+class UserRoleSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Role
         fields = ('user', 'is_owner',)
@@ -21,11 +21,11 @@ class RoleSerializer(serializers.HyperlinkedModelSerializer):
 
 
 class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
-    users = UserSerializer(source='roles', many=True)
+    roles = UserRoleSerializer(many=True)
     
     class Meta:
         model = Database
-        fields = ('url', 'name', 'type', 'users')
+        fields = ('url', 'name', 'type', 'roles')
 
 
 class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):

orchestra/apps/databases/tests/functional_tests/tests.py

@@ -1,5 +1,6 @@
 import MySQLdb
 import os
+import time
 from functools import partial
 
 from django.conf import settings as djsettings
@@ -52,7 +53,7 @@ class DatabaseTestMixin(object):
     
     def test_add(self):
         dbname = '%s_database' % random_ascii(5)
-        username = '%s_dbuser' % random_ascii(10)
+        username = '%s_dbuser' % random_ascii(5)
         password = '@!?%spppP001' % random_ascii(5)
         self.add(dbname, username, password)
         self.validate_create_table(dbname, username, password)
@@ -61,6 +62,10 @@ class DatabaseTestMixin(object):
 class MySQLBackendMixin(object):
     db_type = 'mysql'
     
+    def setUp(self):
+        super(MySQLBackendMixin, self).setUp()
+        settings.DATABASES_DEFAULT_HOST = '10.228.207.207'
+    
     def add_route(self):
         server = Server.objects.create(name=self.MASTER_SERVER)
         backend = backends.MySQLBackend.get_name()
@@ -73,14 +78,13 @@ class MySQLBackendMixin(object):
     def validate_create_table(self, name, username, password):
         db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name)
         cur = db.cursor()
-        cur.execute('CREATE TABLE test;')
+        cur.execute('CREATE TABLE test ( id INT ) ;')
     
     def validate_delete(self, name, username, password):
         self.asseRaises(MySQLdb.ConnectionError,
                 self.validate_create_table, name, username, password)
 
 
-
 class RESTDatabaseMixin(DatabaseTestMixin):
     def setUp(self):
         super(RESTDatabaseMixin, self).setUp()
@@ -89,7 +93,8 @@ class RESTDatabaseMixin(DatabaseTestMixin):
     @save_response_on_error
     def add(self, dbname, username, password):
         user = self.rest.databaseusers.create(username=username, password=password)
-        self.rest.databases.create(name=dbname, user=user, type=self.db_type)
+        # TODO fucking nested objects
+        self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type)
 
 
 class AdminDatabaseMixin(DatabaseTestMixin):

orchestra/apps/lists/backends.py

@@ -6,11 +6,92 @@ from orchestra.apps.orchestration import ServiceController
 from orchestra.apps.resources import ServiceMonitor
 
 from . import settings
+from .models import List
 
 
 class MailmanBackend(ServiceController):
     verbose_name = "Mailman"
     model = 'lists.List'
+    
+    def include_virtual_alias_domain(self, context):
+        if context['address_domain']:
+            self.append(textwrap.dedent("""
+                [[ $(grep "^\s*%(address_domain)s\s*$" %(virtual_alias_domains)s) ]] || {
+                    echo "%(address_domain)s" >> %(virtual_alias_domains)s
+                    UPDATED_VIRTUAL_ALIAS_DOMAINS=1
+                }""" % context
+            ))
+    
+    def exclude_virtual_alias_domain(self, context):
+        address_domain = context['address_domain']
+        if not List.objects.filter(address_domain=address_domain).exists():
+            self.append('sed -i "/^%(address_domain)s\s*/d" %(virtual_alias_domains)s' % context)
+    
+    def get_virtual_aliases(self, context):
+        aliases = []
+        addresses = [
+            '',
+            '-admin',
+            '-bounces',
+            '-confirm',
+            '-join',
+            '-leave',
+            '-owner',
+            '-request',
+            '-subscribe',
+            '-unsubscribe'
+        ]
+        for address in addresses:
+            context['address'] = address
+            aliases.append("%(address_name)s%(address)s@%(domain)s\t%(name)s%(address)s" % context)
+        return '\n'.join(aliases)
+    
+    def save(self, mail_list):
+        if not getattr(mail_list, 'password', None):
+            # TODO
+            # Create only support for now
+            return
+        context = self.get_context(mail_list)
+        self.append("newlist --quiet --emailhost='%(domain)s' '%(name)s' '%(admin)s' '%(password)s'" % context)
+        if mail_list.address:
+            context['aliases'] = self.get_virtual_aliases(context)
+            self.append(
+                "if [[ ! $(grep '^\s*%(name)s\s' %(virtual_alias)s) ]]; then\n"
+                "   echo '# %(banner)s\n%(aliases)s\n' >> %(virtual_alias)s\n"
+                "   UPDATED_VIRTUAL_ALIAS=1\n"
+                "fi" % context
+            )
+        self.include_virtual_alias_domain(context)
+    
+    def delete(self, mail_list):
+        pass
+    
+    def commit(self):
+        context = self.get_context_files()
+        self.append(textwrap.dedent("""
+            [[ $UPDATED_VIRTUAL_ALIAS == 1 ]] && { postmap %(virtual_alias)s; }
+            [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
+            """ % context
+        ))
+    
+    def get_context_files(self):
+        return {
+            'virtual_alias': settings.LISTS_VIRTUAL_ALIAS_PATH,
+            'virtual_alias_domains': settings.MAILS_VIRTUAL_ALIAS_DOMAINS_PATH,
+        }
+    
+    def get_context(self, mail_list):
+        context = self.get_context_files()
+        context.update({
+            'banner': self.get_banner(),
+            'name': mail_list.name,
+            'password': mail_list.password,
+            'domain': mail_list.address_domain or settings.LISTS_DEFAULT_DOMAIN,
+            'address_name': mail_list.address_name,
+            'address_domain': mail_list.address_domain,
+            'admin': mail_list.admin_email,
+        })
+        return context
 
 
 class MailmanTraffic(ServiceMonitor):

orchestra/apps/lists/models.py

@@ -7,9 +7,11 @@ from orchestra.core.validators import validate_name
 from . import settings
 
 
+# TODO address and domain, perhaps allow only domain?
+
 class List(models.Model):
-    name = models.CharField(_("name"), max_length=128, unique=True,
+    name = models.CharField(_("name"), max_length=128, unique=True, validators=[validate_name],
-            validators=[validate_name])
+            help_text=_("Default list address <name>@%s") % settings.LISTS_DEFAULT_DOMAIN)
     address_name = models.CharField(_("address name"), max_length=128,
             validators=[validate_name], blank=True)
     address_domain = models.ForeignKey(settings.LISTS_DOMAIN_MODEL,
@@ -23,7 +25,13 @@ class List(models.Model):
         unique_together = ('address_name', 'address_domain')
     
     def __unicode__(self):
-        return "%s@%s" % (self.address_name, self.address_domain)
+        return self.name
+    
+    @property
+    def address(self):
+        if self.address_name and self.address_domain:
+            return "%s@%s" % (self.address_name, self.address_domain)
+        return ''
     
     def get_username(self):
         return self.name

orchestra/apps/lists/serializers.py

@@ -1,11 +1,34 @@
+from django.forms import widgets
+from django.utils.translation import ugettext, ugettext_lazy as _
 from rest_framework import serializers
 
 from orchestra.apps.accounts.serializers import AccountSerializerMixin
+from orchestra.core.validators import validate_password
 
 from .models import List
 
 
+# TODO create PasswordSerializerMixin
+
 class ListSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    password = serializers.CharField(max_length=128, label=_('Password'),
+            validators=[validate_password], write_only=True, required=False,
+            widget=widgets.PasswordInput)
+    
     class Meta:
         model = List
-        fields = ('url', 'name', 'address_name', 'address_domain')
+        fields = ('url', 'name', 'address_name', 'address_domain', 'admin_email')
+    
+    def validate_password(self, attrs, source):
+        """ POST only password """
+        if self.object:
+            if 'password' in attrs:
+                raise serializers.ValidationError(_("Can not set password"))
+        elif 'password' not in attrs:
+            raise serializers.ValidationError(_("Password required"))
+        return attrs
+    
+    def save_object(self, obj, **kwargs):
+        if not obj.pk:
+            obj.set_password(self.init_data.get('password', ''))
+        super(ListSerializer, self).save_object(obj, **kwargs)

orchestra/apps/lists/settings.py

@@ -1,11 +1,20 @@
 from django.conf import settings
 
 
-# Data access
 
 LISTS_DOMAIN_MODEL = getattr(settings, 'LISTS_DOMAIN_MODEL', 'domains.Domain')
 
-LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'grups.orchestra.lan')
+
+LISTS_DEFAULT_DOMAIN = getattr(settings, 'LIST_DEFAULT_DOMAIN', 'lists.orchestra.lan')
+
 
 LISTS_MAILMAN_POST_LOG_PATH = getattr(settings, 'LISTS_MAILMAN_POST_LOG_PATH',
         '/var/log/mailman/post')
+
+
+LISTS_VIRTUAL_ALIAS_PATH = getattr(settings, 'LISTS_VIRTUAL_ALIAS_PATH',
+        '/etc/postfix/mailman_virtual_aliases')
+
+
+MAILS_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_DOMAINS_PATH',
+        '/etc/postfix/mailman_virtual_domains')

orchestra/apps/lists/tests/functional_tests/tests.py

@@ -0,0 +1,158 @@
+import email.utils
+import os
+import smtplib
+import time
+import textwrap
+from email.mime.text import MIMEText
+
+from django.conf import settings as djsettings
+from django.contrib.contenttypes.models import ContentType
+from django.core.management.base import CommandError
+from django.core.urlresolvers import reverse
+from selenium.webdriver.support.select import Select
+
+from orchestra.apps.accounts.models import Account
+from orchestra.apps.domains.models import Domain
+from orchestra.apps.orchestration.models import Server, Route
+from orchestra.apps.resources.models import Resource
+from orchestra.utils.system import run, sshrun
+from orchestra.utils.tests import BaseLiveServerTestCase, random_ascii, snapshot_on_error, save_response_on_error
+
+from ... import backends, settings
+from ...models import List
+
+
+class ListMixin(object):
+    MASTER_SERVER = os.environ.get('ORCHESTRA_SLAVE_SERVER', 'localhost')
+    DEPENDENCIES = (
+        'orchestra.apps.orchestration',
+        'orchestra.apps.domains',
+        'orchestra.apps.lists',
+    )
+    
+    def setUp(self):
+        super(ListMixin, self).setUp()
+        self.add_route()
+        djsettings.DEBUG = True
+    
+    def validate_add(self, name, address=None):
+        sshrun(self.MASTER_SERVER, 'list_members %s' % name, display=False)
+        if not address:
+            address = "%s@%s" % (name, settings.LISTS_DEFAULT_DOMAIN)
+        subscribe_address = "{}-subscribe@{}".format(*address.split('@'))
+        self.subscribe(subscribe_address)
+        time.sleep(2)
+        sshrun(self.MASTER_SERVER,
+            'grep -v ":\|^\s\|^$\|-\|\.\|\s" /var/spool/mail/nobody | base64 -d | grep "%s"' % address, display=False)
+    
+    def subscribe(self, subscribe_address):
+        msg = MIMEText('')
+        msg['To'] = subscribe_address
+        msg['From'] = 'root@%s' % self.MASTER_SERVER
+        msg['Subject'] = 'subscribe'
+        server = smtplib.SMTP(self.MASTER_SERVER, 25)
+        try:
+            server.ehlo()
+            server.starttls()
+            server.ehlo()
+            server.sendmail(msg['From'], msg['To'], msg.as_string())
+        finally:
+            server.quit()
+    
+    def add_route(self):
+        server = Server.objects.create(name=self.MASTER_SERVER)
+        backend = backends.MailmanBackend.get_name()
+        Route.objects.create(backend=backend, match=True, host=server)
+    
+    def atest_add(self):
+        name = '%s_list' % random_ascii(10)
+        password = '@!?%spppP001' % random_ascii(5)
+        admin_email = 'root@test3.orchestra.lan'
+        self.add(name, password, admin_email)
+        self.validate_add(name)
+#        self.addCleanup(self.delete, username)
+    
+    def test_add_with_address(self):
+        name = '%s_list' % random_ascii(10)
+        password = '@!?%spppP001' % random_ascii(5)
+        print password
+        admin_email = 'root@test3.orchestra.lan'
+        address_name = '%s_name' % random_ascii(10)
+        domain_name = '%sdomain.lan' % random_ascii(10)
+        address_domain = Domain.objects.create(name=domain_name, account=self.account)
+        self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain)
+        self.validate_add(name, address="%s@%s" % (address_name, address_domain))
+
+
+class RESTListMixin(ListMixin):
+    def setUp(self):
+        super(RESTListMixin, self).setUp()
+        self.rest_login()
+    
+    @save_response_on_error
+    def add(self, name, password, admin_email, address_name=None, address_domain=None):
+        extra = {}
+        if address_name:
+            extra.update({
+                'address_name': address_name,
+                'address_domain': self.rest.domains.retrieve(name=address_domain.name).get().url,
+            })
+        self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra)
+    
+    @save_response_on_error
+    def delete(self, username):
+        list = self.rest.lists.retrieve(name=username).get()
+        list.delete()
+
+
+class AdminListMixin(ListMixin):
+    def setUp(self):
+        super(AdminListMixin, self).setUp()
+        self.admin_login()
+    
+    @snapshot_on_error
+    def add(self, name, password, admin_email):
+        url = self.live_server_url + reverse('admin:mails_List_add')
+        self.selenium.get(url)
+        
+        account_input = self.selenium.find_element_by_id('id_account')
+        account_select = Select(account_input)
+        account_select.select_by_value(str(self.account.pk))
+        
+        name_field = self.selenium.find_element_by_id('id_name')
+        name_field.send_keys(username)
+        
+        password_field = self.selenium.find_element_by_id('id_password1')
+        password_field.send_keys(password)
+        password_field = self.selenium.find_element_by_id('id_password2')
+        password_field.send_keys(password)
+        
+        if quota is not None:
+            quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
+            quota_field = self.selenium.find_element_by_id(quota_id)
+            quota_field.clear()
+            quota_field.send_keys(quota)
+        
+        if filtering is not None:
+            filtering_input = self.selenium.find_element_by_id('id_filtering')
+            filtering_select = Select(filtering_input)
+            filtering_select.select_by_value("CUSTOM")
+            filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
+            filtering_inline.click()
+            time.sleep(0.5)
+            filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
+            filtering_field.send_keys(filtering)
+        
+        name_field.submit()
+        self.assertNotEqual(url, self.selenium.current_url)
+
+
+class RESTListTest(RESTListMixin, BaseLiveServerTestCase):
+    pass
+
+
+#class AdminListTest(AdminListMixin, BaseLiveServerTestCase):
+#    pass
+
+
+

orchestra/apps/mails/admin.py

@@ -9,10 +9,9 @@ from django.utils.translation import ugettext_lazy as _
 from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
 from orchestra.admin.utils import admin_link, change_url
 from orchestra.apps.accounts.admin import SelectAccountAdminMixin, AccountAdminMixin
-from orchestra.forms import UserCreationForm, UserChangeForm
 
 from .filters import HasMailboxListFilter, HasForwardListFilter, HasAddressListFilter
-from .forms import MailboxCreationForm, AddressForm
+from .forms import MailboxCreationForm, MailboxChangeForm, AddressForm
 from .models import Mailbox, Address, Autoresponse
 
 
@@ -28,36 +27,34 @@ class AutoresponseInline(admin.StackedInline):
 
 class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
     list_display = (
-        'name', 'account_link', 'uses_custom_filtering', 'display_addresses'
+        'name', 'account_link', 'filtering', 'display_addresses'
     )
-    list_filter = (HasAddressListFilter,)
+    list_filter = (HasAddressListFilter, 'filtering')
     add_fieldsets = (
         (None, {
-            'fields': ('account', 'name', 'password1', 'password2'),
+            'fields': ('account', 'name', 'password1', 'password2', 'filtering'),
         }),
-        (_("Filtering"), {
+        (_("Custom filtering"), {
             'classes': ('collapse',),
             'fields': ('custom_filtering',),
         }),
     )
     fieldsets = (
         (None, {
-            'classes': ('wide',),
+            'fields': ('name', 'password', 'is_active', 'account_link', 'filtering'),
-            'fields': ('name', 'password', 'is_active', 'account_link'),
         }),
-        (_("Filtering"), {
+        (_("Custom filtering"), {
             'classes': ('collapse',),
             'fields': ('custom_filtering',),
         }),
         (_("Addresses"), {
-            'classes': ('wide',),
             'fields': ('addresses_field',)
         }),
     )
     readonly_fields = ('account_link', 'display_addresses', 'addresses_field')
     change_readonly_fields = ('name',)
     add_form = MailboxCreationForm
-    form = UserChangeForm
+    form = MailboxChangeForm
     
     def display_addresses(self, mailbox):
         addresses = []
@@ -68,16 +65,10 @@ class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdm
     display_addresses.short_description = _("Addresses")
     display_addresses.allow_tags = True
     
-    def uses_custom_filtering(self, mailbox):
-        return bool(mailbox.custom_filtering)
-    uses_custom_filtering.short_description = _("Custom filter")
-    uses_custom_filtering.boolean = True
-    uses_custom_filtering.admin_order_field = 'custom_filtering'
-    
     def get_fieldsets(self, request, obj=None):
         """ not collapsed filtering when exists """
         fieldsets = super(MailboxAdmin, self).get_fieldsets(request, obj=obj)
-        if obj and obj.custom_filtering:
+        if obj and obj.filtering == obj.CUSTOM:
             fieldsets = copy.deepcopy(fieldsets)
             fieldsets[1][1]['classes'] = fieldsets[0][1]['fields'] + ('open',)
         return fieldsets
@@ -97,7 +88,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, AccountAdminMixin, ExtendedModelAdm
             name = '%s@%s' % (name, domain)
             value += '<li><a href="%s">%s</a></li>' % (url, name)
         value = '<ul>%s</ul>' % value
-        return mark_safe('<div style="padding-left: 100px;">%s</div>' % value)
+        return mark_safe('<div style="padding-left: 10px;">%s</div>' % value)
     addresses_field.short_description = _("Addresses")
     addresses_field.allow_tags = True
 

orchestra/apps/mails/backends.py

@@ -1,3 +1,4 @@
+import logging
 import textwrap
 import os
 
@@ -13,11 +14,12 @@ from .models import Address
 
 # TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
 # TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
-# TODO Set first/last_valid_uid/gid settings to contain only the range actually used by mail processes
-# TODO Insert "/./" inside the returned home directory, eg.: home=/home/./user to chroot into /home, or home=/home/user/./ to chroot into /home/user.
 # TODO mount the filesystem with "nosuid" option
 
 
+logger = logging.getLogger(__name__)
+
+
 class PasswdVirtualUserBackend(ServiceController):
     verbose_name = _("Mail virtual user (passwd-file)")
     model = 'mails.Mailbox'
@@ -36,22 +38,27 @@ class PasswdVirtualUserBackend(ServiceController):
         self.append("mkdir -p %(home)s" % context)
         self.append("chown %(uid)s.%(gid)s %(home)s" % context)
     
+    def set_mailbox(self, context):
+        self.append(textwrap.dedent("""
+            if [[ ! $(grep "^%(username)s@%(mailbox_domain)s\s" %(virtual_mailbox_maps)s) ]]; then
+                echo "%(username)s@%(mailbox_domain)s\tOK" >> %(virtual_mailbox_maps)s
+                UPDATED_VIRTUAL_MAILBOX_MAPS=1
+            fi""" % context))
+    
     def generate_filter(self, mailbox, context):
-        now = timezone.now().strftime("%B %d, %Y, %H:%M")
+        self.append("doveadm mailbox create -u %(username)s Spam" % context) # TODO override webmail filters???
-        context['filtering'] = (
+        context['filtering_path'] = os.path.join(context['home'], '.dovecot.sieve')
-            "# Sieve Filter\n"
+        filtering = mailbox.get_filtering()
-            "# Generated by Orchestra %s\n\n" % now
+        if filtering:
-        )
+            context['filtering'] = '# %(banner)s\n' + filtering
-        if mailbox.custom_filtering:
+            self.append("echo '%(filtering)s' > %(filtering_path)s" % context)
-            context['filtering'] += mailbox.custom_filtering
         else:
-            context['filtering'] += settings.MAILS_DEFAUL_FILTERING
+            self.append("rm -f %(filtering_path)s" % context)
-        context['filter_path'] = os.path.join(context['home'], '.orchestra.sieve')
-        self.append("echo '%(filtering)s' > %(filter_path)s" % context)
     
     def save(self, mailbox):
         context = self.get_context(mailbox)
         self.set_user(context)
+        self.set_mailbox(context)
         self.generate_filter(mailbox, context)
     
     def delete(self, mailbox):
@@ -59,6 +66,8 @@ class PasswdVirtualUserBackend(ServiceController):
         self.append("{ sleep 2 && killall -u %(uid)s -s KILL; } &" % context)
         self.append("killall -u %(uid)s || true" % context)
         self.append("sed -i '/^%(username)s:.*/d' %(passwd_path)s" % context)
+        self.append("sed -i '/^%(username)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s" % context)
+        self.append("UPDATED_VIRTUAL_MAILBOX_MAPS=1")
         # TODO delete
         context['deleted'] = context['home'].rstrip('/') + '.deleted'
         self.append("mv %(home)s %(deleted)s" % context)
@@ -75,6 +84,15 @@ class PasswdVirtualUserBackend(ServiceController):
         unit = mailbox.resources.disk.unit[0].upper()
         return 'userdb_quota_rule=*:bytes=%i%s' % (quota, unit)
     
+    def commit(self):
+        context = {
+            'virtual_mailbox_maps': settings.MAILS_VIRTUAL_MAILBOX_MAPS_PATH
+        }
+        self.append(
+            "[[ $UPDATED_VIRTUAL_MAILBOX_MAPS == 1 ]] && { postmap %(virtual_mailbox_maps)s; }"
+             % context
+        )
+    
     def get_context(self, mailbox):
         context = {
             'name': mailbox.name,
@@ -86,9 +104,12 @@ class PasswdVirtualUserBackend(ServiceController):
             'quota': self.get_quota(mailbox),
             'passwd_path': settings.MAILS_PASSWD_PATH,
             'home': mailbox.get_home(),
+            'banner': self.get_banner(),
+            'virtual_mailbox_maps': settings.MAILS_VIRTUAL_MAILBOX_MAPS_PATH,
+            'mailbox_domain': settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
         }
         context['extra_fields'] = self.get_extra_fields(mailbox, context)
-        context['passwd'] = '{username}:{password}:{uid}:{gid}:,,,:{home}:{extra_fields}'.format(**context)
+        context['passwd'] = '{username}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context)
         return context
 
 
@@ -96,62 +117,76 @@ class PostfixAddressBackend(ServiceController):
     verbose_name = _("Postfix address")
     model = 'mails.Address'
     
-    def include_virtdomain(self, context):
+    def include_virtual_alias_domain(self, context):
-        self.append(
-            '[[ $(grep "^\s*%(domain)s\s*$" %(virtdomains)s) ]]'
-            '   || { echo "%(domain)s" >> %(virtdomains)s; UPDATED_VIRTDOMAINS=1; }' % context
-        )
-    
-    def exclude_virtdomain(self, context):
-        domain = context['domain']
-        if not Address.objects.filter(domain=domain).exists():
-            self.append('sed -i "s/^%(domain)s//" %(virtdomains)s' % context)
-    
-    def update_virtusertable(self, context):
         self.append(textwrap.dedent("""
-            LINE="%(email)s\t%(destination)s"
+            [[ $(grep "^\s*%(domain)s\s*$" %(virtual_alias_domains)s) ]] || {
-            if [[ ! $(grep "^%(email)s\s" %(virtusertable)s) ]]; then
+                echo "%(domain)s" >> %(virtual_alias_domains)s
-               echo "${LINE}" >> %(virtusertable)s
+                UPDATED_VIRTUAL_ALIAS_DOMAINS=1
-               UPDATED_VIRTUSERTABLE=1
+            }""" % context
-            else
-               if [[ ! $(grep "^${LINE}$" %(virtusertable)s) ]]; then
-                   sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtusertable)s
-                   UPDATED_VIRTUSERTABLE=1
-               fi
-            fi""" % context
         ))
     
-    def exclude_virtusertable(self, context):
+    def exclude_virtual_alias_domain(self, context):
+        domain = context['domain']
+        if not Address.objects.filter(domain=domain).exists():
+            self.append('sed -i "/^%(domain)s\s*/d" %(virtual_alias_domains)s' % context)
+    
+    def update_virtual_alias_maps(self, address, context):
+        destination = []
+        for mailbox in address.get_mailboxes():
+            context['mailbox'] = mailbox
+            destination.append("%(mailbox)s@%(mailbox_domain)s" % context)
+        for forward in address.forward:
+            if '@' in forward:
+                destination.append(forward)
+        if destination:
+            context['destination'] = ' '.join(destination)
+            self.append(textwrap.dedent("""
+                LINE="%(email)s\t%(destination)s"
+                if [[ ! $(grep "^%(email)s\s" %(virtual_alias_maps)s) ]]; then
+                   echo "${LINE}" >> %(virtual_alias_maps)s
+                   UPDATED_VIRTUAL_ALIAS_MAPS=1
+                else
+                   if [[ ! $(grep "^${LINE}$" %(virtual_alias_maps)s) ]]; then
+                       sed -i "s/^%(email)s\s.*$/${LINE}/" %(virtual_alias_maps)s
+                       UPDATED_VIRTUAL_ALIAS_MAPS=1
+                   fi
+                fi""" % context
+            ))
+        else:
+            logger.warning("Address %i is empty" % address.pk)
+            self.append('sed -i "/^%(email)s\s/d" %(virtual_alias_maps)s')
+            self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1')
+    
+    def exclude_virtual_alias_maps(self, context):
         self.append(textwrap.dedent("""
             if [[ $(grep "^%(email)s\s") ]]; then
-               sed -i "s/^%(email)s\s.*$//" %(virtusertable)s
+               sed -i "/^%(email)s\s.*$/d" %(virtual_alias_maps)s
-               UPDATED=1
+               UPDATED_VIRTUAL_ALIAS_MAPS=1
             fi"""
         ))
     
     def save(self, address):
         context = self.get_context(address)
-        self.include_virtdomain(context)
+        self.include_virtual_alias_domain(context)
-        self.update_virtusertable(context)
+        self.update_virtual_alias_maps(address, context)
     
     def delete(self, address):
         context = self.get_context(address)
-        self.exclude_virtdomain(context)
+        self.exclude_virtual_alias_domain(context)
-        self.exclude_virtusertable(context)
+        self.exclude_virtual_alias_maps(context)
     
     def commit(self):
         context = self.get_context_files()
         self.append(textwrap.dedent("""
-            [[ $UPDATED_VIRTUSERTABLE == 1 ]] && { postmap %(virtusertable)s; }
+            [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; }
-            # TODO not sure if always needed
+            [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
-            [[ $UPDATED_VIRTDOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
             """ % context
         ))
     
     def get_context_files(self):
         return {
-            'virtdomains': settings.MAILS_VIRTDOMAINS_PATH,
+            'virtual_alias_domains': settings.MAILS_VIRTUAL_ALIAS_DOMAINS_PATH,
-            'virtusertable': settings.MAILS_VIRTUSERTABLE_PATH,
+            'virtual_alias_maps': settings.MAILS_VIRTUAL_ALIAS_MAPS_PATH
         }
     
     def get_context(self, address):
@@ -159,7 +194,7 @@ class PostfixAddressBackend(ServiceController):
         context.update({
             'domain': address.domain,
             'email': address.email,
-            'destination': address.destination,
+            'mailbox_domain': settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
         })
         return context
 

orchestra/apps/mails/forms.py

@@ -1,9 +1,23 @@
 from django import forms
+from django.utils.translation import ugettext_lazy as _
 
-from orchestra.forms import UserCreationForm
+from orchestra.forms import UserCreationForm, UserChangeForm
 
 
-class MailboxCreationForm(UserCreationForm):
+class CleanCustomFilteringMixin(object):
+    def clean_custom_filtering(self):
+        filtering = self.cleaned_data['filtering']
+        custom_filtering = self.cleaned_data['custom_filtering']
+        if filtering == self._meta.model.CUSTOM and not custom_filtering:
+            raise forms.ValidationError(_("You didn't provide any custom filtering"))
+        return custom_filtering
+
+
+class MailboxChangeForm(CleanCustomFilteringMixin, UserChangeForm):
+    pass
+
+
+class MailboxCreationForm(CleanCustomFilteringMixin, UserCreationForm):
     def clean_name(self):
         # Since model.clean() will check this, this is redundant,
         # but it sets a nicer error message than the ORM and avoids conflicts with contrib.auth

orchestra/apps/mails/models.py

@@ -11,6 +11,8 @@ from . import validators, settings
 # TODO rename app to mailboxes
 
 class Mailbox(models.Model):
+    CUSTOM = 'CUSTOM'
+    
     name = models.CharField(_("name"), max_length=64, unique=True,
             help_text=_("Required. 30 characters or fewer. Letters, digits and "
                         "@/./+/-/_ only."),
@@ -19,6 +21,9 @@ class Mailbox(models.Model):
     password = models.CharField(_("password"), max_length=128)
     account = models.ForeignKey('accounts.Account', verbose_name=_("account"),
             related_name='mailboxes')
+    filtering = models.CharField(max_length=16,
+            choices=[(k, v[0]) for k,v in settings.MAILS_MAILBOX_FILTERINGS.iteritems()],
+            default=settings.MAILS_MAILBOX_DEFAULT_FILTERING)
     custom_filtering = models.TextField(_("filtering"), blank=True,
             validators=[validators.validate_sieve],
             help_text=_("Arbitrary email filtering in sieve language. "
@@ -51,6 +56,28 @@ class Mailbox(models.Model):
         }
         home = settings.MAILS_HOME % context
         return home.rstrip('/')
+    
+    def clean(self):
+        if self.custom_filtering and self.filtering != self.CUSTOM:
+            self.custom_filtering = ''
+    
+    def get_filtering(self):
+        __, filtering = settings.MAILS_MAILBOX_FILTERINGS[self.filtering]
+        if isinstance(filtering, basestring):
+            return filtering
+        return filtering(self)
+    
+    def delete(self, *args, **kwargs):
+        super(Mailbox, self).delete(*args, **kwargs)
+        # Cleanup related addresses
+        for address in Address.objects.filter(forward__regex=r'.*(^|\s)+%s($|\s)+.*' % self.name):
+            forward = address.forward.split()
+            forward.remove(self.name)
+            address.forward = ' '.join(forward)
+            if not address.destination:
+                address.delete()
+            else:
+                address.save()
 
 
 class Address(models.Model):
@@ -63,7 +90,8 @@ class Address(models.Model):
             verbose_name=_("mailboxes"),
             related_name='addresses', blank=True)
     forward = models.CharField(_("forward"), max_length=256, blank=True,
-            validators=[validators.validate_forward], help_text=_("Space separated email addresses"))
+            validators=[validators.validate_forward],
+            help_text=_("Space separated email addresses or mailboxes"))
     account = models.ForeignKey('accounts.Account', verbose_name=_("Account"),
             related_name='addresses')
     
@@ -78,12 +106,26 @@ class Address(models.Model):
     def email(self):
         return "%s@%s" % (self.name, self.domain)
     
-    @property
+#    @property
-    def destination(self):
+#    def destination(self):
-        destinations = list(self.mailboxes.values_list('name', flat=True))
+#        destinations = list(self.mailboxes.values_list('name', flat=True))
-        if self.forward:
+#        if self.forward:
-            destinations.append(self.forward)
+#            destinations.append(self.forward)
-        return ' '.join(destinations)
+#        return ' '.join(destinations)
+    
+    def get_forward_mailboxes(self):
+        for forward in self.forward.split():
+            if '@' not in forward:
+                try:
+                    yield Mailbox.objects.get(name=forward)
+                except Mailbox.DoesNotExist:
+                    pass
+    
+    def get_mailboxes(self):
+        for mailbox in self.mailboxes.all():
+            yield mailbox
+        for mailbox in self.get_forward_mailboxes():
+            yield mailbox
 
 
 class Autoresponse(models.Model):

orchestra/apps/mails/serializers.py

@@ -1,15 +1,23 @@
+from django.forms import widgets
+from django.utils.translation import ugettext, ugettext_lazy as _
 from rest_framework import serializers
 
 from orchestra.apps.accounts.serializers import AccountSerializerMixin
+from orchestra.core.validators import validate_password
 
 from .models import Mailbox, Address
 
 
 class MailboxSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
+    password = serializers.CharField(max_length=128, label=_('Password'),
+            validators=[validate_password], write_only=True, required=False,
+            widget=widgets.PasswordInput)
+    
     class Meta:
         model = Mailbox
-        # TODO 'use_custom_filtering', 
+        fields = (
-        fields = ('url', 'name', 'password', 'custom_filtering', 'addresses', 'is_active')
+            'url', 'name', 'password', 'filtering', 'custom_filtering', 'addresses', 'is_active'
+        )
     
     def validate_password(self, attrs, source):
         """ POST only password """

orchestra/apps/mails/settings.py

@@ -1,4 +1,7 @@
+import textwrap
+
 from django.conf import settings
+from django.utils.translation import ugettext_lazy as _
 
 
 MAILS_DOMAIN_MODEL = getattr(settings, 'MAILS_DOMAIN_MODEL', 'domains.Domain')
@@ -14,23 +17,43 @@ MAILS_SIEVETEST_BIN_PATH = getattr(settings, 'MAILS_SIEVETEST_BIN_PATH',
         '%(orchestra_root)s/bin/sieve-test')
 
 
-MAILS_VIRTUSERTABLE_PATH = getattr(settings, 'MAILS_VIRTUSERTABLE_PATH',
+MAILS_VIRTUAL_MAILBOX_MAPS_PATH = getattr(settings, 'MAILS_VIRTUAL_MAILBOX_MAPS_PATH',
-        '/etc/postfix/virtusertable')
+        '/etc/postfix/virtual_mailboxes')
+        
+
+MAILS_VIRTUAL_ALIAS_MAPS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_MAPS_PATH',
+        '/etc/postfix/virtual_aliases')
 
 
-MAILS_VIRTDOMAINS_PATH = getattr(settings, 'MAILS_VIRTDOMAINS_PATH',
+MAILS_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILS_VIRTUAL_ALIAS_DOMAINS_PATH',
-        '/etc/postfix/virtdomains')
+        '/etc/postfix/virtual_domains')
 
 
+MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN = getattr(settings, 'MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN', 
+        'orchestra.lan')
+
 MAILS_PASSWD_PATH = getattr(settings, 'MAILS_PASSWD_PATH',
-        '/etc/dovecot/virtual_users')
+        '/etc/dovecot/passwd')
 
 
-MAILS_DEFAUL_FILTERING = getattr(settings, 'MAILS_DEFAULT_FILTERING',
+
-    'require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];\n'
+MAILS_MAILBOX_FILTERINGS = getattr(settings, 'MAILS_MAILBOX_FILTERINGS', {
-    '\n'
+    # value: (verbose_name, filter)
-    'if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {\n'
+    'DISABLE': (_("Disable"), ''),
-    '    fileinto "Junk";\n'
+    'REJECT': (_("Reject spam"), textwrap.dedent("""
-    '    discard;\n'
+         require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
-    '}'
+         if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {
-)
+            discard;
+            stop;
+        }""")),
+    'REDIRECT': (_("Archive spam"), textwrap.dedent("""
+        require ["fileinto","regex","envelope","vacation","reject","relational","comparator-i;ascii-numeric"];
+        if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-Score" "5" {
+            fileinto "Spam";
+            stop;
+        }""")),
+    'CUSTOM': (_("Custom filtering"), lambda mailbox: mailbox.custom_filtering),
+})
+
+
+MAILS_MAILBOX_DEFAULT_FILTERING = getattr(settings, 'MAILS_MAILBOX_DEFAULT_FILTERING', 'REDIRECT')

orchestra/apps/mails/tests/functional_tests/tests.py

@@ -4,8 +4,10 @@ import os
 import poplib
 import smtplib
 import time
+import textwrap
 from email.mime.text import MIMEText
 
+from django.apps import apps
 from django.conf import settings as djsettings
 from django.contrib.contenttypes.models import ContentType
 from django.core.management.base import CommandError
@@ -33,8 +35,6 @@ class MailboxMixin(object):
     def setUp(self):
         super(MailboxMixin, self).setUp()
         self.add_route()
-        # TODO fix this
-        from django.apps import apps
         # clean resource relation from other tests
         apps.get_app_config('resources').reload_relations()
         djsettings.DEBUG = True
@@ -92,7 +92,7 @@ class MailboxMixin(object):
     def send_email(self, to, token):
         msg = MIMEText(token)
         msg['To'] = to
-        msg['From'] = 'orchestra@test.orchestra.lan'
+        msg['From'] = 'orchestra@%s' % self.MASTER_SERVER
         msg['Subject'] = 'test'
         server = smtplib.SMTP(self.MASTER_SERVER, 25)
         try:
@@ -176,7 +176,7 @@ class MailboxMixin(object):
         password = '@!?%spppP001' % random_ascii(5)
         self.add(username, password)
         self.validate_mailbox(username)
-        self.addCleanup(self.delete, username)
+#        self.addCleanup(self.delete, username)
         imap = self.login_imap(username, password)
         self.disable(username)
         self.assertRaises(imap.error, self.login_imap, username, password)
@@ -211,6 +211,27 @@ class MailboxMixin(object):
         self.delete_address(username)
         self.send_email("%s@%s" % (name, domain), token)
         self.validate_email(username, token)
+    
+    def test_custom_filtering(self):
+        username = '%s_mailbox' % random_ascii(10)
+        password = '@!?%spppP001' % random_ascii(5)
+        folder = random_ascii(5)
+        filtering = textwrap.dedent("""
+            require "fileinto";
+            if true { 
+                fileinto "%s";
+                stop;
+            }""" % folder)
+        self.add(username, password, filtering=filtering)
+        self.addCleanup(self.delete, username)
+        imap = self.login_imap(username, password)
+        imap.create(folder)
+        self.validate_mailbox(username)
+        token = random_ascii(100)
+        self.send_email("%s@%s" % (username, settings.MAILS_VIRTUAL_MAILBOX_DEFAULT_DOMAIN), token)
+        home = Mailbox.objects.get(name=username).get_home()
+        sshrun(self.MASTER_SERVER,
+               "grep '%s' %s/Maildir/.%s/new/*" % (token, home, folder), display=False)
 
 
 class RESTMailboxMixin(MailboxMixin):
@@ -219,17 +240,22 @@ class RESTMailboxMixin(MailboxMixin):
         self.rest_login()
     
     @save_response_on_error
-    def add(self, username, password, quota=None):
+    def add(self, username, password, quota=None, filtering=None):
         extra = {}
         if quota:
-            extra = {
+            extra.update({
                 "resources": [
                     {
                         "name": "disk",
                         "allocated": quota
                     },
                 ]
-            }
+            })
+        if filtering:
+            extra.update({
+                'filtering': 'CUSTOM',
+                'custom_filtering': filtering,
+            })
         self.rest.mailboxes.create(name=username, password=password, **extra)
     
     @save_response_on_error
@@ -270,7 +296,7 @@ class AdminMailboxMixin(MailboxMixin):
         self.admin_login()
     
     @snapshot_on_error
-    def add(self, username, password, quota=None):
+    def add(self, username, password, quota=None, filtering=None):
         url = self.live_server_url + reverse('admin:mails_mailbox_add')
         self.selenium.get(url)
         
@@ -285,17 +311,23 @@ class AdminMailboxMixin(MailboxMixin):
         password_field.send_keys(password)
         password_field = self.selenium.find_element_by_id('id_password2')
         password_field.send_keys(password)
+        
         if quota is not None:
-            from orchestra.admin.utils import get_modeladmin
+            quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
-            m = get_modeladmin(Mailbox)
+            quota_field = self.selenium.find_element_by_id(quota_id)
-            print 't', type(m).inlines
-            print 'm', m.inlines
-            self.take_screenshot()
-            quota_field = self.selenium.find_element_by_id(
-                'id_resources-resourcedata-content_type-object_id-0-allocated')
             quota_field.clear()
             quota_field.send_keys(quota)
         
+        if filtering is not None:
+            filtering_input = self.selenium.find_element_by_id('id_filtering')
+            filtering_select = Select(filtering_input)
+            filtering_select.select_by_value("CUSTOM")
+            filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
+            filtering_inline.click()
+            time.sleep(0.5)
+            filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
+            filtering_field.send_keys(filtering)
+        
         name_field.submit()
         self.assertNotEqual(url, self.selenium.current_url)
     

orchestra/apps/mails/validators.py

@@ -2,6 +2,7 @@ import hashlib
 import os
 import re
 
+from django.core.management.base import CommandError
 from django.core.validators import ValidationError, EmailValidator
 from django.utils.translation import ugettext_lazy as _
 
@@ -22,38 +23,33 @@ def validate_emailname(value):
         raise ValidationError(msg)
 
 
-#def validate_destination(value):
-#    """ space separated mailboxes or emails """
-#    for destination in value.split():
-#        msg = _("'%s' is not an existent mailbox" % destination)
-#        if '@' in destination:
-#            if not destination[-1].isalpha():
-#                raise ValidationError(msg)
-#            EmailValidator(destination)
-#        else:
-#            from .models import Mailbox
-#            if not Mailbox.objects.filter(user__username=destination).exists():
-#                raise ValidationError(msg)
-#            validate_emailname(destination)
-
-
 def validate_forward(value):
     """ space separated mailboxes or emails """
+    from .models import Mailbox
     for destination in value.split():
-        EmailValidator(destination)
+        msg = _("'%s' is not an existent mailbox" % destination)
+        if '@' in destination:
+            if not destination[-1].isalpha():
+                raise ValidationError(msg)
+            EmailValidator(destination)
+        else:
+            if not Mailbox.objects.filter(user__username=destination).exists():
+                raise ValidationError(msg)
+            validate_emailname(destination)
 
 
 def validate_sieve(value):
     sieve_name = '%s.sieve' % hashlib.md5(value).hexdigest()
-    path = os.path.join(settings.EMAILS_SIEVETEST_PATH, sieve_name)
+    path = os.path.join(settings.MAILS_SIEVETEST_PATH, sieve_name)
     with open(path, 'wb') as f:
         f.write(value)
     context = {
         'orchestra_root': paths.get_orchestra_root()
     }
-    sievetest = settings.EMAILS_SIEVETEST_BIN_PATH % context
+    sievetest = settings.MAILS_SIEVETEST_BIN_PATH % context
-    test = run(' '.join([sievetest, path, '/dev/null']), display=False)
+    try:
-    if test.return_code:
+        test = run(' '.join([sievetest, path, '/dev/null']), display=False)
+    except CommandError:
         errors = []
         for line in test.stderr.splitlines():
             error = re.match(r'^.*(line\s+[0-9]+:.*)', line)

orchestra/apps/orchestration/admin.py

@@ -46,8 +46,8 @@ class RouteAdmin(admin.ModelAdmin):
 
 class BackendOperationInline(admin.TabularInline):
     model = BackendOperation
-    fields = ('action', 'instance_link')
+    fields = ('action', 'content_object_link')
-    readonly_fields = ('action', 'instance_link')
+    readonly_fields = ('action', 'content_object_link')
     extra = 0
     can_delete = False
     
@@ -56,22 +56,22 @@ class BackendOperationInline(admin.TabularInline):
             'all': ('orchestra/css/hide-inline-id.css',)
         }
     
-    def instance_link(self, operation):
+    def content_object_link(self, operation):
         try:
-            return admin_link('instance')(self, operation)
+            return admin_link('content_object')(self, operation)
         except:
             return _("deleted {0} {1}").format(
                 escape(operation.content_type), escape(operation.object_id)
             )
-    instance_link.allow_tags = True
+    content_object_link.allow_tags = True
-    instance_link.short_description = _("Instance")
+    content_object_link.short_description = _("Content_object")
     
     def has_add_permission(self, *args, **kwargs):
         return False
     
     def get_queryset(self, request):
         queryset = super(BackendOperationInline, self).get_queryset(request)
-        return queryset.prefetch_related('instance')
+        return queryset.prefetch_related('content_object')
 
 
 def display_mono(field):

orchestra/apps/orchestration/manager.py

@@ -14,9 +14,12 @@ logger = logging.getLogger(__name__)
 
 def as_task(execute):
     def wrapper(*args, **kwargs):
-        with db.transaction.commit_manually():
+        db.transaction.set_autocommit(False)
+        try:
             log = execute(*args, **kwargs)
+        finally:
             db.transaction.commit()
+            db.transaction.set_autocommit(True)
         if log.state != log.SUCCESS:
             send_report(execute, args, log)
         return log
@@ -25,7 +28,6 @@ def as_task(execute):
 
 def close_connection(execute):
     """ Threads have their own connection pool, closing it when finishing """
-    # TODO rewrite as context manager
     def wrapper(*args, **kwargs):
         log = execute(*args, **kwargs)
         db.connection.close()

orchestra/apps/orchestration/middlewares.py

@@ -84,7 +84,12 @@ class OperationsMiddleware(object):
                             if not execute:
                                 continue
                 instance = copy.copy(instance)
-                pending_operations.add(Operation.create(backend, instance, action))
+                operation = Operation.create(backend, instance, action)
+                if action != Operation.DELETE:
+                    # usually we expect to be using last object state,
+                    # except when we are deleting it
+                    pending_operations.discard(operation)
+                pending_operations.add(operation)
     
     def process_request(self, request):
         """ Store request on a thread local variable """

orchestra/apps/orchestration/models.py

@@ -102,19 +102,23 @@ class BackendOperation(models.Model):
     content_type = models.ForeignKey(ContentType)
     object_id = models.PositiveIntegerField()
     
-    instance = generic.GenericForeignKey('content_type', 'object_id')
+    content_object = generic.GenericForeignKey('content_type', 'object_id')
     
     class Meta:
         verbose_name = _("Operation")
         verbose_name_plural = _("Operations")
     
+    def __init__(self, *args, **kwargs):
+        self.instance = kwargs.pop('instance', None)
+        super(BackendOperation, self).__init__(*args, **kwargs)
+    
     def __unicode__(self):
-        return '%s.%s(%s)' % (self.backend, self.action, self.instance)
+        return '%s.%s(%s)' % (self.backend, self.action, self.instance or self.content_object)
     
     def __hash__(self):
         """ set() """
         backend = getattr(self, 'backend', self.backend)
-        return hash(backend) + hash(self.instance) + hash(self.action)
+        return hash(backend) + hash(self.instance or self.content_object) + hash(self.action)
     
     def __eq__(self, operation):
         """ set() """
@@ -122,7 +126,7 @@ class BackendOperation(models.Model):
     
     @classmethod
     def create(cls, backend, instance, action):
-        op = cls(backend=backend.get_name(), instance=instance, action=action)
+        op = cls(backend=backend.get_name(), instance=instance, content_object=instance, action=action)
         op.backend = backend
         return op
     

orchestra/apps/resources/admin.py

@@ -7,6 +7,7 @@ from orchestra.admin import ExtendedModelAdmin
 from orchestra.admin.filters import UsedContentTypeFilter
 from orchestra.admin.utils import insertattr, get_modeladmin, admin_link, admin_date
 from orchestra.core import services
+from orchestra.utils import database_ready
 
 from .forms import ResourceForm
 from .models import Resource, ResourceData, MonitorData
@@ -135,7 +136,6 @@ def resource_inline_factory(resources):
     return ResourceInline
 
 
-from orchestra.utils import database_ready
 def insert_resource_inlines():
     # Clean previous state
     for related in Resource._related:
@@ -144,14 +144,12 @@ def insert_resource_inlines():
         for inline in getattr(modeladmin_class, 'inlines', []):
             if inline.__name__ == 'ResourceInline':
                 modeladmin_class.inlines.remove(inline)
-        modeladmin.inlines = modeladmin_class.inlines
     
     for ct, resources in Resource.objects.group_by('content_type').iteritems():
         inline = resource_inline_factory(resources)
         model = ct.model_class()
-        modeladmin = get_modeladmin(model)
         insertattr(model, 'inlines', inline)
-        modeladmin.inlines = type(modeladmin).inlines
+
 
 if database_ready():
     insert_resource_inlines()

orchestra/apps/resources/models.py

@@ -101,10 +101,9 @@ class Resource(models.Model):
             elif task.crontab != self.crontab:
                 task.crontab = self.crontab
                 task.save(update_fields=['crontab'])
-        if created:
+        # This only work on tests (multiprocessing used on real deployments)
-            # This only work on tests because of multiprocessing used on real deployments
+        apps.get_app_config('resources').reload_relations()
-            print 'saved'
+        # TODO touch wsgi.py for code reloading?
-            apps.get_app_config('resources').reload_relations()
     
     def delete(self, *args, **kwargs):
         super(Resource, self).delete(*args, **kwargs)

orchestra/apps/services/models.py

@@ -230,10 +230,11 @@ class Service(models.Model):
     def get_services(cls, instance):
         cache = caches.get_request_cache()
         ct = ContentType.objects.get_for_model(instance)
-        services = cache.get(ct)
+        key = 'services.Service-%i' % ct.pk
+        services = cache.get(key)
         if services is None:
             services = cls.objects.filter(content_type=ct, is_active=True)
-            cache.set(ct, services)
+            cache.set(key, services)
         return services
     
     # FIXME some times caching is nasty, do we really have to? make get_plugin more efficient?

orchestra/bin/orchestra-admin

@@ -130,8 +130,7 @@ function install_requirements () {
          libxml2-dev \
          libxslt1-dev \
          wkhtmltopdf \
-         xvfb \
+         xvfb"
-         python-mysqldb"
     
     PIP="django==1.7 \
          django-celery-email==1.0.4 \
@@ -159,7 +158,8 @@ function install_requirements () {
     if $testing; then
         APT="${APT} \
             iceweasel \
-            dnsutils"
+            dnsutils \
+            python-mysqldb"
         PIP="${PIP} \
             selenium \
             xvfbwrapper \

scripts/services/postfix.md

@@ -6,7 +6,7 @@
 
 
 
-apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve
+apt-get install dovecot-core dovecot-imapd dovecot-pop3d dovecot-lmtpd dovecot-sieve dovecot-managesieved
 
 echo 'mail_location = maildir:~/Maildir
 mail_plugins = quota
@@ -42,3 +42,6 @@ echo 'mailbox_transport = lmtp:unix:private/dovecot-lmtp' >> /etc/postfix/main.c
 /etc/init.d/dovecot restart
 /etc/init.d/postfix restart
 
+# TODO check postfix and dovecot configs
+
+# TODO crontab that deletes message +30 days on spam folders