Lots of random fixes

TODO.md

@@ -280,4 +280,9 @@ https://code.djangoproject.com/ticket/24576
 
 # migrations accounts, bill, orders, auth -> migrate the rest (contacts lambda error)
 
-# MultiCHoiceField proper serialization
+* MultiCHoiceField proper serialization
+
+# Apache restart fails: detect if appache running, and execute start
+# PHP backend is retarded does not detect well the version
+# Change crons, create cron for deleted webapps and users
+* UNIFY PHP FPM settings name

orchestra/contrib/domains/admin.py

@@ -103,14 +103,16 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin):
         """ Order by structured name and imporve performance """
         qs = super(DomainAdmin, self).get_queryset(request)
         qs = qs.select_related('top', 'account')
-        # For some reason if we do this we know for sure that join table will be called T4
+        # Order by structured name
-        query = str(qs.query)
+        if request.method == 'GET':
-        table = re.findall(r'(T\d+)\."account_id"', query)[0]
+            # For some reason if we do this we know for sure that join table will be called T4
-        qs = qs.extra(
+            query = str(qs.query)
-            select={
+            table = re.findall(r'(T\d+)\."account_id"', query)[0]
-                'structured_name': 'CONCAT({table}.name, domains_domain.name)'.format(table=table)
+            qs = qs.extra(
-            },
+                select={
-        ).order_by('structured_name')
+                    'structured_name': 'CONCAT({table}.name, domains_domain.name)'.format(table=table)
+                },
+            ).order_by('structured_name')
         if apps.isinstalled('orchestra.contrib.websites'):
             qs = qs.prefetch_related('websites')
         return qs

orchestra/contrib/domains/forms.py

@@ -59,6 +59,7 @@ class BatchDomainCreationAdminForm(forms.ModelForm):
 class RecordInlineFormSet(forms.models.BaseInlineFormSet):
     def clean(self):
         """ Checks if everything is consistent """
+        super(RecordInlineFormSet, self).clean()
         if any(self.errors):
             return
         if self.instance.name:

orchestra/contrib/domains/helpers.py

@@ -9,7 +9,7 @@ def domain_for_validation(instance, records):
     so when validation calls render_zone() it will use the new provided data
     """
     domain = copy.copy(instance)
-    def get_records():
+    def get_records(records=records):
         for data in records:
             yield Record(type=data['type'], value=data['value'])
     domain.get_records = get_records
@@ -19,7 +19,8 @@ def domain_for_validation(instance, records):
         domain.top = domain.get_parent(top=True)
     if domain.top:
         # is a subdomain
-        subdomains = [sub for sub in domain.top.subdomains.all() if sub.pk != domain.pk]
+        subdomains = domain.top.subdomains.select_related('top').prefetch_related('records').all()
+        subdomains = [sub for sub in subdomains if sub.pk != domain.pk]
         domain.top.get_subdomains = lambda: subdomains + [domain]
     elif not domain.pk:
         # is a new top domain

orchestra/contrib/lists/admin.py

@@ -6,17 +6,20 @@ from django.utils.translation import ugettext_lazy as _
 from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
 from orchestra.admin.utils import admin_link
 from orchestra.contrib.accounts.admin import SelectAccountAdminMixin
+from orchestra.contrib.accounts.filters import IsActiveListFilter
 
 from .forms import ListCreationForm, ListChangeForm
 from .models import List
 
 
 class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModelAdmin):
-    list_display = ('name', 'address_name', 'address_domain_link', 'account_link')
+    list_display = (
+        'name', 'address_name', 'address_domain_link', 'account_link', 'display_active'
+    )
     add_fieldsets = (
         (None, {
             'classes': ('wide',),
-            'fields': ('account_link', 'name',)
+            'fields': ('account_link', 'name', 'is_active')
         }),
         (_("Address"), {
             'classes': ('wide',),
@@ -30,7 +33,7 @@ class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModel
     fieldsets = (
         (None, {
             'classes': ('wide',),
-            'fields': ('account_link', 'name',)
+            'fields': ('account_link', 'name', 'is_active')
         }),
         (_("Address"), {
             'classes': ('wide',),
@@ -42,6 +45,7 @@ class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModel
         }),
     )
     search_fields = ('name', 'address_name', 'address_domain__name', 'account__username')
+    list_filter = (IsActiveListFilter,)
     readonly_fields = ('account_link',)
     change_readonly_fields = ('name',)
     form = ListChangeForm

orchestra/contrib/lists/backends.py

@@ -119,7 +119,7 @@ class MailmanBackend(ServiceController):
                 postmap %(virtual_alias)s
             fi
             if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then
-                /etc/init.d/postfix reload
+                service postfix reload
             fi""") % context
         )
     

orchestra/contrib/mailboxes/backends.py

@@ -118,14 +118,17 @@ class DovecotPostfixPasswdVirtualUserBackend(ServiceController):
     
     def delete(self, mailbox):
         context = self.get_context(mailbox)
-        self.append("{ sleep 2 && killall -u %(uid)s -s KILL; } &" % context)
+        self.append(textwrap.dedent("""\
-        self.append("killall -u %(uid)s || true" % context)
+            { sleep 2 && killall -u %(uid)s -s KILL; } &
-        self.append("sed -i '/^%(user)s:.*/d' %(passwd_path)s" % context)
+            killall -u %(uid)s || true
-        self.append("sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s" % context)
+            sed -i '/^%(user)s:.*/d' %(passwd_path)s
-        self.append("UPDATED_VIRTUAL_MAILBOX_MAPS=1")
+            sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s
-        # TODO delete
+            UPDATED_VIRTUAL_MAILBOX_MAPS=1""") % context
-        context['deleted'] = context['home'].rstrip('/') + '.deleted'
+        )
-        self.append("mv %(home)s %(deleted)s" % context)
+        if context['deleted_home']:
+            self.append("mv %(home)s %(deleted_home)s || exit_code=1" % context)
+        else:
+            self.append("rm -fr %(home)s" % context)
     
     def get_extra_fields(self, mailbox, context):
         context['quota'] = self.get_quota(mailbox)
@@ -159,13 +162,16 @@ class DovecotPostfixPasswdVirtualUserBackend(ServiceController):
             'group': self.DEFAULT_GROUP,
             'quota': self.get_quota(mailbox),
             'passwd_path': settings.MAILBOXES_PASSWD_PATH,
-            'home': mailbox.get_home().rstrip('/'),
+            'home': mailbox.get_home(),
             'banner': self.get_banner(),
             'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH,
             'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
         }
         context['extra_fields'] = self.get_extra_fields(mailbox, context)
-        context['passwd'] = '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context)
+        context.update({
+            'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context),
+            'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context,
+        })
         return replace(context, "'", '"')
 
 
@@ -177,11 +183,13 @@ class PostfixAddressBackend(ServiceController):
     )
     
     def include_virtual_alias_domain(self, context):
-        self.append(textwrap.dedent("""
+        if context['domain'] != context['local_domain']:
-            [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || {
+            self.append(textwrap.dedent("""
-                echo '%(domain)s' >> %(virtual_alias_domains)s
+                [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || {
-                UPDATED_VIRTUAL_ALIAS_DOMAINS=1
+                    echo '%(domain)s' >> %(virtual_alias_domains)s
-            }""") % context)
+                    UPDATED_VIRTUAL_ALIAS_DOMAINS=1
+                }""") % context
+            )
     
     def exclude_virtual_alias_domain(self, context):
         domain = context['domain']
@@ -193,7 +201,7 @@ class PostfixAddressBackend(ServiceController):
 #        destination = []
 #        for mailbox in address.get_mailboxes():
 #            context['mailbox'] = mailbox
-#            destination.append("%(mailbox)s@%(mailbox_domain)s" % context)
+#            destination.append("%(mailbox)s@%(local_domain)s" % context)
 #        for forward in address.forward:
 #            if '@' in forward:
 #                destination.append(forward)
@@ -237,7 +245,7 @@ class PostfixAddressBackend(ServiceController):
         context = self.get_context_files()
         self.append(textwrap.dedent("""
             [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; }
-            [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; }
+            [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { service postfix reload; }
             """) % context
         )
         self.append('exit 0')
@@ -253,7 +261,7 @@ class PostfixAddressBackend(ServiceController):
         context.update({
             'domain': address.domain,
             'email': address.email,
-            'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN,
+            'local_domain': settings.MAILBOXES_LOCAL_DOMAIN,
         })
         return replace(context, "'", '"')
 
@@ -344,11 +352,13 @@ class PostfixMailscannerTraffic(ServiceMonitor):
             def inside_period(month, day, time, ini_date):
                 global months
                 global end_datetime
-                # Mar 19 17:13:22
+                # Mar  9 17:13:22
                 month = months[month]
                 year = end_datetime.year
                 if month == '12' and end_datetime.month == 1:
                     year = year+1
+                if len(day) == 1:
+                    day = '0' + day
                 date = str(year) + month + day
                 date += time.replace(':', '')
                 return ini_date < int(date) < end_date

orchestra/contrib/mailboxes/settings.py

@@ -47,7 +47,7 @@ MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILBOXES_VIRTUAL_ALIA
 )
 
 
-MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN = getattr(settings, 'MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN', 
+MAILBOXES_LOCAL_DOMAIN = getattr(settings, 'MAILBOXES_LOCAL_DOMAIN', 
     ORCHESTRA_BASE_DOMAIN
 )
 
@@ -94,3 +94,8 @@ MAILBOXES_LOCAL_ADDRESS_DOMAIN = getattr(settings, 'MAILBOXES_LOCAL_ADDRESS_DOMA
 MAILBOXES_MAIL_LOG_PATH = getattr(settings, 'MAILBOXES_MAIL_LOG_PATH',
     '/var/log/mail.log'
 )
+
+
+MAILBOXES_MOVE_ON_DELETE_PATH = getattr(settings, 'MAILBOXES_MOVE_ON_DELETE_PATH',
+    ''
+)

orchestra/contrib/miscellaneous/admin.py

@@ -54,7 +54,7 @@ class MiscServiceAdmin(ExtendedModelAdmin):
 
 class MiscellaneousAdmin(AccountAdminMixin, SelectPluginAdminMixin, admin.ModelAdmin):
     list_display = (
-        '__str__', 'service_link', 'amount', 'dispaly_active', 'account_link'
+        '__str__', 'service_link', 'amount', 'dispaly_active', 'account_link', 'is_active'
     )
     list_filter = ('service__name', 'is_active')
     list_select_related = ('service', 'account')

orchestra/contrib/miscellaneous/models.py

@@ -56,10 +56,7 @@ class Miscellaneous(models.Model):
     
     @cached_property
     def active(self):
-        try:
+        return self.is_active and self.service.is_active and self.account.is_active
-            return self.is_active and self.account.is_active
-        except type(self).account.field.rel.to.DoesNotExist:
-            return self.is_active
     
     def get_description(self):
         return ' '.join((str(self.amount), self.service.description or self.service.verbose_name))

orchestra/contrib/orchestration/__init__.py

@@ -50,7 +50,7 @@ class Operation():
             if hasattr(self.backend, 'get_context'):
                 self.backend().get_context(self.instance)
     
-    def create(self, log):
+    def store(self, log):
         from .models import BackendOperation
         return BackendOperation.objects.create(
             log=log,

orchestra/contrib/orchestration/management/commands/orchestrate.py

@@ -18,11 +18,14 @@ class Command(BaseCommand):
             help='Tells Django to NOT prompt the user for input of any kind.')
         parser.add_argument('--action', action='store', dest='action',
             default='save', help='Executes action. Defaults to "save".')
+        parser.add_argument('--dry-run', action='store_true', dest='dry', default=False,
+            help='Only prints scrtipt.')
     
     def handle(self, *args, **options):
         model = get_model(*options['model'].split('.'))
         action = options.get('action')
         interactive = options.get('interactive')
+        dry = options.get('dry')
         kwargs = {}
         for comp in options.get('query', []):
             comps = iter(comp.split('='))
@@ -42,7 +45,9 @@ class Command(BaseCommand):
             servers.append(server.name)
             sys.stdout.write('# Execute on %s\n' % server.name)
             for method, commands in backend.scripts:
-                sys.stdout.write('\n'.join(commands) + '\n')
+                script = '\n'.join(commands) + '\n'
+                script = script.encode('ascii', errors='replace')
+                sys.stdout.write(script.decode('ascii'))
         if interactive:
             context = {
                 'servers': ', '.join(servers),
@@ -56,4 +61,10 @@ class Command(BaseCommand):
                 if confirm == 'no':
                     return
                 break
-#        manager.execute(scripts, block=block)
+        if not dry:
+            logs = manager.execute(scripts, block=block)
+            for log in logs:
+                print(log.stdout)
+                sys.stderr.write(log.stderr)
+            for log in logs:
+                print(log.backend, log.state)

orchestra/contrib/orchestration/manager.py

@@ -125,7 +125,7 @@ def execute(scripts, block=False, async=False):
                 logger.info("Executed %s" % str(operation))
                 if operation.instance.pk:
                     # Not all backends are called with objects saved on the database
-                    operation.create(execution.log)
+                    operation.store(execution.log)
             stdout = execution.log.stdout.strip()
             stdout and logger.debug('STDOUT %s', stdout)
             stderr = execution.log.stderr.strip()

orchestra/contrib/plans/admin.py

@@ -14,8 +14,8 @@ class RateInline(admin.TabularInline):
 
 
 class PlanAdmin(ExtendedModelAdmin):
-    list_display = ('name', 'is_default', 'is_combinable', 'allow_multiple')
+    list_display = ('name', 'is_default', 'is_combinable', 'allow_multiple', 'is_active')
-    list_filter = ('is_default', 'is_combinable', 'allow_multiple')
+    list_filter = ('is_default', 'is_combinable', 'allow_multiple', 'is_active')
     fields = ('verbose_name', 'name', 'is_default', 'is_combinable', 'allow_multiple')
     prepopulated_fields = {
         'name': ('verbose_name',)

orchestra/contrib/resources/admin.py

@@ -126,7 +126,7 @@ class ResourceDataAdmin(ExtendedModelAdmin):
     display_unit.admin_order_field = 'resource__unit'
     
     def display_used(self, data):
-        if not data.used:
+        if data.used is None:
             return ''
         url = reverse('admin:resources_resourcedata_used_monitordata', args=(data.pk,))
         return '<a href="%s">%s</a>' % (url, data.used)

orchestra/contrib/resources/aggregations.py

@@ -19,12 +19,13 @@ class Aggregation(plugins.Plugin, metaclass=plugins.PluginMount):
 
 
 class Last(Aggregation):
+    """ Sum of the last value of all monitors """
     name = 'last'
     verbose_name = _("Last value")
     
     def filter(self, dataset):
         try:
-            return dataset.order_by('object_id', '-id').distinct('object_id')
+            return dataset.order_by('object_id', '-id').distinct('monitor')
         except dataset.model.DoesNotExist:
             return dataset.none()
     
@@ -38,6 +39,7 @@ class Last(Aggregation):
 
 
 class MonthlySum(Last):
+    """ Monthly sum the values of all monitors """
     name = 'monthly-sum'
     verbose_name = _("Monthly Sum")
     
@@ -50,9 +52,14 @@ class MonthlySum(Last):
 
 
 class MonthlyAvg(MonthlySum):
+    """ sum of the monthly averages of each monitor """
     name = 'monthly-avg'
     verbose_name = _("Monthly AVG")
     
+    def filter(self, dataset):
+        qs = super(MonthlyAvg, self).filter(dataset)
+        return qs.order_by('created_at')
+    
     def get_epoch(self):
         today = timezone.now()
         return datetime(
@@ -64,21 +71,27 @@ class MonthlyAvg(MonthlySum):
     
     def compute_usage(self, dataset):
         result = 0
-        try:
+        has_result = False
-            last = dataset.latest()
+        for monitor, dataset in dataset.group_by('monitor').items():
-        except dataset.model.DoesNotExist:
+            try:
+                last = dataset[-1]
+            except IndexError:
+                continue
+            epoch = self.get_epoch()
+            total = (last.created_at-epoch).total_seconds()
+            ini = epoch
+            for data in dataset:
+                has_result = True
+                slot = (data.created_at-ini).total_seconds()
+                result += data.value * decimal.Decimal(str(slot/total))
+                ini = data.created_at
+        if has_result:
             return result
-        epoch = self.get_epoch()
+        return None
-        total = (last.created_at-epoch).total_seconds()
-        ini = epoch
-        for data in dataset:
-            slot = (data.created_at-ini).total_seconds()
-            result += data.value * decimal.Decimal(str(slot/total))
-            ini = data.created_at
-        return result
 
 
 class Last10DaysAvg(MonthlyAvg):
+    """ sum of the last 10 days averages of each monitor """
     name = 'last-10-days-avg'
     verbose_name = _("Last 10 days AVG")
     days = 10
@@ -88,4 +101,5 @@ class Last10DaysAvg(MonthlyAvg):
         return today - datetime.timedelta(days=self.days)
     
     def filter(self, dataset):
-        return dataset.filter(created_at__gt=self.get_epoch())
+        epoch = self.get_epoch()
+        return dataset.filter(created_at__gt=epoch).order_by('created_at')

orchestra/contrib/resources/models.py

@@ -262,6 +262,10 @@ class ResourceData(models.Model):
         return datasets
 
 
+class MonitorDataQuerySet(models.QuerySet):
+    group_by = queryset.group_by
+
+
 class MonitorData(models.Model):
     """ Stores monitored data """
     monitor = models.CharField(_("monitor"), max_length=256,
@@ -272,6 +276,7 @@ class MonitorData(models.Model):
     value = models.DecimalField(_("value"), max_digits=16, decimal_places=2)
     
     content_object = GenericForeignKey()
+    objects = MonitorDataQuerySet.as_manager()
     
     class Meta:
         get_latest_by = 'id'

orchestra/contrib/resources/tasks.py

@@ -27,7 +27,7 @@ def monitor(resource_id, ids=None, async=True):
         # Execute monitor
         monitorings = []
         for obj in model.objects.filter(**kwargs):
-            op = Operation.create(backend, obj, Operation.MONITOR)
+            op = Operation(backend, obj, Operation.MONITOR)
             operations.append(op)
             monitorings.append(op)
         # TODO async=True only when running with celery
@@ -44,10 +44,10 @@ def monitor(resource_id, ids=None, async=True):
             a = data.used
             b = data.allocated
             if data.used > (data.allocated or 0):
-                op = Operation.create(backend, obj, Operation.EXCEEDED)
+                op = Operation(backend, obj, Operation.EXCEEDED)
                 triggers.append(op)
             elif data.used < (data.allocated or 0):
-                op = Operation.create(backend, obj, Operation.RECOVERY)
+                op = Operation(backend, obj, Operation.RECOVERY)
                 triggers.append(op)
     Operation.execute(triggers)
     return operations

orchestra/contrib/saas/backends/wordpressmu.py

@@ -13,10 +13,6 @@ class WordpressMuBackend(ServiceController):
     model = 'webapps.WebApp'
     default_route_match = "webapp.type == 'wordpress-mu'"
     
-    @property
-    def script(self):
-        return self.cmds
-    
     def login(self, session):
         base_url = self.get_base_url()
         login_url = base_url + '/wp-login.php'
@@ -113,11 +109,7 @@ class WordpressMuBackend(ServiceController):
             self.validate_response(response)
     
     def save(self, webapp):
-        if webapp.type != 'wordpress-mu':
-            return
         self.append(self.create_blog, webapp)
     
     def delete(self, webapp):
-        if webapp.type != 'wordpress-mu':
-            return
         self.append(self.delete_blog, webapp)

orchestra/contrib/systemusers/backends.py

@@ -39,9 +39,9 @@ class UNIXUserBackend(ServiceController):
             )
         for member in settings.SYSTEMUSERS_DEFAULT_GROUP_MEMBERS:
             context['member'] = member
-            self.append('usermod -a -G %(user)s %(member)s' % context)
+            self.append('usermod -a -G %(user)s %(member)s || exit_code=$?' % context)
         if not user.is_main:
-            self.append('usermod -a -G %(user)s %(mainuser)s' % context)
+            self.append('usermod -a -G %(user)s %(mainuser)s || exit_code=$?' % context)
     
     def delete(self, user):
         context = self.get_context(user)
@@ -52,9 +52,12 @@ class UNIXUserBackend(ServiceController):
             killall -u %(user)s || true
             userdel %(user)s || exit_code=1
             groupdel %(group)s || exit_code=1
-            mv %(base_home)s %(base_home)s.deleted || exit_code=1
             """) % context
         )
+        if context['deleted_home']:
+            self.append("mv %(base_home)s %(deleted_home)s || exit_code=1" % context)
+        else:
+            self.append("rm -fr %(base_home)s" % context)
     
     def grant_permission(self, user):
         context = self.get_context(user)
@@ -76,6 +79,7 @@ class UNIXUserBackend(ServiceController):
             'home': user.get_home(),
             'base_home': user.get_base_home(),
         }
+        context['deleted_home'] = settings.SYSTEMUSERS_MOVE_ON_DELETE_PATH % context
         return replace(context, "'", '"')
 
 

orchestra/contrib/systemusers/forms.py

@@ -60,6 +60,7 @@ class SystemUserFormMixin(object):
         }
     
     def clean(self):
+        super(SystemUserFormMixin, self).clean()
         home = self.cleaned_data.get('home')
         if home and self.MOCK_USERNAME in home:
             username = self.cleaned_data.get('username', '')

orchestra/contrib/systemusers/settings.py

@@ -39,3 +39,8 @@ SYSTEMUSERS_MAIL_LOG_PATH = getattr(settings, 'SYSTEMUSERS_MAIL_LOG_PATH',
 SYSTEMUSERS_DEFAULT_GROUP_MEMBERS = getattr(settings, 'SYSTEMUSERS_DEFAULT_GROUP_MEMBERS',
     ('www-data',)
 )
+
+
+SYSTEMUSERS_MOVE_ON_DELETE_PATH = getattr(settings, 'SYSTEMUSERS_MOVE_ON_DELETE_PATH',
+    ''
+)

orchestra/contrib/webapps/admin.py

@@ -5,14 +5,15 @@ from django.utils.encoding import force_text
 from django.utils.translation import ugettext, ugettext_lazy as _
 
 from orchestra.admin import ExtendedModelAdmin
-from orchestra.admin.utils import change_url
+from orchestra.admin.utils import change_url, get_modeladmin
 from orchestra.contrib.accounts.admin import AccountAdminMixin
 from orchestra.forms.widgets import DynamicHelpTextSelect
 from orchestra.plugins.admin import SelectPluginAdminMixin
 
+from .filters import HasWebsiteListFilter
+from .models import WebApp, WebAppOption
 from .options import AppOption
 from .types import AppType
-from .models import WebApp, WebAppOption
 
 
 class WebAppOptionInline(admin.TabularInline):
@@ -36,7 +37,9 @@ class WebAppOptionInline(admin.TabularInline):
                 plugin = self.parent_object.type_class
             else:
                 request = kwargs['request']
-                plugin = AppType.get(request.GET['type'])
+                webapp_modeladmin = get_modeladmin(self.parent_model)
+                plugin_value = webapp_modeladmin.get_plugin_value(request)
+                plugin = AppType.get(plugin_value)
             kwargs['choices'] = plugin.get_options_choices()
             # Help text based on select widget
             target = 'this.id.replace("name", "value")'
@@ -46,7 +49,7 @@ class WebAppOptionInline(admin.TabularInline):
 
 class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
     list_display = ('name', 'type', 'display_detail', 'display_websites', 'account_link')
-    list_filter = ('type',)
+    list_filter = ('type', HasWebsiteListFilter)
     inlines = [WebAppOptionInline]
     readonly_fields = ('account_link', )
     change_readonly_fields = ('name', 'type', 'display_websites')

orchestra/contrib/webapps/backends/__init__.py

@@ -29,7 +29,10 @@ class WebAppServiceMixin(object):
             )
     
     def delete_webapp_dir(self, context):
-        self.append("rm -fr %(app_path)s" % context)
+        if context['deleted_app_path']:
+            self.append("mv %(app_path)s %(deleted_app_path)s || exit_code=1" % context)
+        else:
+            self.append("rm -fr %(app_path)s" % context)
     
     def get_context(self, webapp):
         context = {
@@ -37,11 +40,12 @@ class WebAppServiceMixin(object):
             'group': webapp.get_groupname(),
             'app_name': webapp.name,
             'type': webapp.type,
-            'app_path': webapp.get_path().rstrip('/'),
+            'app_path': webapp.get_path(),
             'banner': self.get_banner(),
             'under_construction_path': settings.settings.WEBAPPS_UNDER_CONSTRUCTION_PATH,
             'is_mounted': webapp.content_set.exists(),
         }
+        context['deleted_app_path'] = settings.WEBAPPS_MOVE_ON_DELETE_PATH % context
         return replace(context, "'", '"')
 
 

orchestra/contrib/webapps/backends/php.py

@@ -17,6 +17,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
     
     def save(self, webapp):
         context = self.get_context(webapp)
+        self.create_webapp_dir(context)
+        self.set_under_construction(context)
         if webapp.type_instance.is_fpm:
             self.save_fpm(webapp, context)
             self.delete_fcgid(webapp, context)
@@ -25,8 +27,6 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
             self.delete_fpm(webapp, context)
     
     def save_fpm(self, webapp, context):
-        self.create_webapp_dir(context)
-        self.set_under_construction(context)
         self.append(textwrap.dedent("""\
             fpm_config='%(fpm_config)s'
             {
@@ -39,8 +39,6 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         )
     
     def save_fcgid(self, webapp, context):
-        self.create_webapp_dir(context)
-        self.set_under_construction(context)
         self.append("mkdir -p %(wrapper_dir)s" % context)
         self.append(textwrap.dedent("""\
             wrapper='%(wrapper)s'
@@ -104,7 +102,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         merge = settings.WEBAPPS_MERGE_PHP_WEBAPPS
         context.update({
             'init_vars': webapp.type_instance.get_php_init_vars(merge=self.MERGE),
-            'max_children': webapp.get_options().get('processes', False),
+            'max_children': webapp.get_options().get('processes',
+                settings.WEBAPPS_FPM_DEFAULT_MAX_CHILDREN),
             'request_terminate_timeout': webapp.get_options().get('timeout', False),
         })
         context['fpm_listen'] = webapp.type_instance.FPM_LISTEN % context
@@ -119,7 +118,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
             listen.group = {{ group }}
             pm = ondemand
             pm.max_requests = {{ max_requests }}
-            {% if max_children %}pm.max_children = {{ max_children }}{% endif %}
+            pm.max_children = {{ max_children }}
             {% if request_terminate_timeout %}request_terminate_timeout = {{ request_terminate_timeout }}{% endif %}
             {% for name, value in init_vars.iteritems %}
             php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}
@@ -133,7 +132,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         init_vars = opt.get_php_init_vars(merge=self.MERGE)
         if init_vars:
             init_vars = [ "-d %s='%s'" % (k, v.replace("'", '"')) for k,v in init_vars.items() ]
-        init_vars = ', '.join(init_vars)
+        init_vars = ' \\\n    '.join(init_vars)
         context.update({
             'php_binary': os.path.normpath(settings.WEBAPPS_PHP_CGI_BINARY_PATH % context),
             'php_rc': os.path.normpath(settings.WEBAPPS_PHP_CGI_RC_DIR % context),

orchestra/contrib/webapps/backends/symboliclink.py

@@ -1,23 +1,28 @@
+import textwrap
+
 from django.utils.translation import ugettext_lazy as _
 
 from orchestra.contrib.orchestration import ServiceController, replace
 
-from . import WebAppServiceMixin
+from .php import PHPBackend
 
 
-class SymbolicLinkBackend(WebAppServiceMixin, ServiceController):
+class SymbolicLinkBackend(PHPBackend, ServiceController):
     verbose_name = _("Symbolic link webapp")
     model = 'webapps.WebApp'
     default_route_match = "webapp.type == 'symbolic-link'"
     
-    def save(self, webapp):
+    def create_webapp_dir(self, context):
-        context = self.get_context(webapp)
+        self.append(textwrap.dedent("""\
-        self.append("ln -s '%(link_path)s' %(app_path)s" % context)
+            if [[ ! -e %(app_path)s ]]; then
-        self.append("chown -h %(user)s:%(group)s %(app_path)s" % context)
+                ln -s '%(link_path)s' %(app_path)s
+            fi
+            chown -h %(user)s:%(group)s %(app_path)s
+            """) % context
+        )
     
-    def delete(self, webapp):
+    def set_under_construction(self, context):
-        context = self.get_context(webapp)
+        pass
-        self.delete_webapp_dir(context)
     
     def get_context(self, webapp):
         context = super(SymbolicLinkBackend, self).get_context(webapp)

orchestra/contrib/webapps/filters.py

@@ -0,0 +1,22 @@
+from django.contrib.admin import SimpleListFilter
+from django.utils.translation import ugettext_lazy as _
+
+
+class HasWebsiteListFilter(SimpleListFilter):
+    title = _("Has website")
+    parameter_name = 'has_website'
+    
+    def lookups(self, request, model_admin):
+        return (
+            ('True', _("True")),
+            ('False', _("False")),
+        )
+    
+    def queryset(self, request, queryset):
+        if self.value() == 'True':
+            return queryset.filter(content__isnull=False)
+        elif self.value() == 'False':
+            return queryset.filter(content__isnull=True)
+        return queryset
+
+

orchestra/contrib/webapps/options.py

@@ -180,14 +180,6 @@ class PHPMaginQuotesSybase(PHPAppOption):
     regex = r'^(On|Off|on|off)$'
 
 
-class PHPMaxExecutonTime(PHPAppOption):
-    name = 'max_execution_time'
-    verbose_name = _("Max execution time")
-    help_text = _("Maximum time in seconds a script is allowed to run before it is terminated by "
-                "the parser (Integer between 0 and 999).")
-    regex = r'^[0-9]{1,3}$'
-
-
 class PHPMaxInputTime(PHPAppOption):
     name = 'max_input_time'
     verbose_name = _("Max input time")

orchestra/contrib/webapps/settings.py

@@ -13,6 +13,11 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN',
     '/opt/php/5.4/socks/%(user)s-%(app_name)s.sock'
 )
 
+WEBAPPS_FPM_DEFAULT_MAX_CHILDREN = getattr(settings, 'WEBAPPS_FPM_DEFAULT_MAX_CHILDREN',
+    3
+)
+
+
 WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
     '/etc/php5/fpm/pool.d/%(user)s-%(app_name)s.conf')
 
@@ -145,7 +150,6 @@ WEBAPPS_ENABLED_OPTIONS = getattr(settings, 'WEBAPPS_ENABLED_OPTIONS', (
     'orchestra.contrib.webapps.options.PHPMagicQuotesGPC',
     'orchestra.contrib.webapps.options.PHPMagicQuotesRuntime',
     'orchestra.contrib.webapps.options.PHPMaginQuotesSybase',
-    'orchestra.contrib.webapps.options.PHPMaxExecutonTime',
     'orchestra.contrib.webapps.options.PHPMaxInputTime',
     'orchestra.contrib.webapps.options.PHPMaxInputVars',
     'orchestra.contrib.webapps.options.PHPMemoryLimit',
@@ -171,3 +175,8 @@ WEBAPPS_ENABLED_OPTIONS = getattr(settings, 'WEBAPPS_ENABLED_OPTIONS', (
 WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST = getattr(settings, 'WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST',
     'mysql.{}'.format(ORCHESTRA_BASE_DOMAIN)
 )
+
+
+WEBAPPS_MOVE_ON_DELETE_PATH = getattr(settings, 'WEBAPPS_MOVE_ON_DELETE_PATH',
+    ''
+)

orchestra/contrib/webapps/types/php.py

@@ -77,15 +77,16 @@ class PHPApp(AppType):
             php_version = self.get_php_version()
             webapps = self.instance.account.webapps.filter(type=self.instance.type)
             for webapp in webapps:
-                if webapp.type_instance.get_php_version == php_version:
+                if webapp.type_instance.get_php_version() == php_version:
                     options += list(webapp.options.all())
         php_options = [option.name for option in self.get_php_options()]
         enabled_functions = set()
         for opt in options:
             if opt.name in php_options:
-                init_vars[opt.name] = opt.value
+                if opt.name == 'enabled_functions':
-            elif opt.name == 'enabled_functions':
+                    enabled_functions = enabled_functions.union(set(opt.value.split(',')))
-                enabled_functions.union(set(opt.value.split(',')))
+                else:
+                    init_vars[opt.name] = opt.value
         if enabled_functions:
             disabled_functions = []
             for function in self.PHP_DISABLED_FUNCTIONS:
@@ -94,7 +95,9 @@ class PHPApp(AppType):
             init_vars['dissabled_functions'] = ','.join(disabled_functions)
         timeout = self.instance.options.filter(name='timeout').first()
         if timeout:
-            init_vars['max_execution_time'] = timeout.value
+            # Give a little slack here
+            timeout = str(int(timeout.value)-2)
+            init_vars['max_execution_time'] = timeout
         if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars:
             context = self.get_directive_context()
             error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context)

orchestra/contrib/websites/backends/apache.py

@@ -40,6 +40,7 @@ class Apache2Backend(ServiceController):
         context['extra_conf'] = '\n'.join([conf for location, conf in extra_conf])
         return Template(textwrap.dedent("""\
             <VirtualHost {{ ip }}:{{ port }}>
+                IncludeOptional /etc/apache2/site[s]-override/{{ site_unique_name }}.con[f]
                 ServerName {{ site.domains.all|first }}\
             {% if site.domains.all|slice:"1:" %}
                 ServerAlias {{ site.domains.all|slice:"1:"|join:' ' }}{% endif %}\
@@ -50,7 +51,6 @@ class Apache2Backend(ServiceController):
                 SuexecUserGroup {{ user }} {{ group }}\
             {% for line in extra_conf.splitlines %}
                 {{ line | safe }}{% endfor %}
-                IncludeOptional /etc/apache2/extra-vhos[t]/{{ site_unique_name }}.con[f]
             </VirtualHost>
             """)
         ).render(Context(context))
@@ -181,8 +181,8 @@ class Apache2Backend(ServiceController):
         
     def get_security(self, directives):
         security = []
-        for rules in directives.get('sec-rule-remove', []):
+        for values in directives.get('sec-rule-remove', []):
-            for rule in rules.value.split():
+            for rule in values.split():
                 sec_rule = "SecRuleRemoveById %i" % int(rule)
                 security.append(('', sec_rule))
         for location in directives.get('sec-engine', []):
@@ -267,12 +267,12 @@ class Apache2Backend(ServiceController):
             'site': site,
             'site_name': site.name,
             'ip': settings.WEBSITES_DEFAULT_IP,
-            'site_unique_name': site.unique_name,
+            'site_unique_name': '0-'+site.unique_name,
             'user': self.get_username(site),
             'group': self.get_groupname(site),
             # TODO remove '0-'
             'sites_enabled': "%s.conf" % os.path.join(sites_enabled, '0-'+site.unique_name),
-            'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name),
+            'sites_available': "%s.conf" % os.path.join(sites_available, '0-'+site.unique_name),
             'access_log': site.get_www_access_log_path(),
             'error_log': site.get_www_error_log_path(),
             'banner': self.get_banner(),

orchestra/contrib/websites/forms.py

@@ -9,6 +9,7 @@ from .validators import validate_domain_protocol
 class WebsiteAdminForm(forms.ModelForm):
     def clean(self):
         """ Prevent multiples domains on the same protocol """
+        super(WebsiteAdminForm, self).clean()
         domains = self.cleaned_data.get('domains')
         if not domains:
             return self.cleaned_data

orchestra/management/commands/makemessages.py

@@ -51,7 +51,7 @@ class Command(makemessages.Command):
             tmpcontent = '\n'.join(tmpcontent) + '\n'
             filename = 'database_%s.sql.py' % name
             self.database_files.append(filename)
-            with open(filename, 'w') as tmpfile:
+            with open(filename, 'wb') as tmpfile:
                 tmpfile.write(tmpcontent.encode('utf-8'))
     
     def remove_database_files(self):

orchestra/plugins/admin.py

@@ -1,3 +1,5 @@
+import re
+
 from django.conf.urls import patterns, url
 from django.contrib.admin.utils import unquote
 from django.shortcuts import render, redirect
@@ -58,10 +60,19 @@ class SelectPluginAdminMixin(object):
         template = 'admin/plugins/select_plugin.html'
         return render(request, template, context)
     
+    def get_plugin_value(self, request):
+        plugin_value = request.GET.get(self.plugin_field) or request.POST.get(self.plugin_field)
+        if not plugin_value and request.method == 'POST':
+            # HACK baceuse django add_preserved_filters removes extising queryargs
+            value = re.search(r"type=([^&^']+)[&']", request.META.get('HTTP_REFERER', ''))
+            if value:
+                plugin_value = value.groups()[0]
+        return plugin_value
+    
     def add_view(self, request, form_url='', extra_context=None):
         """ Redirects to select account view if required """
         if request.user.is_superuser:
-            plugin_value = request.GET.get(self.plugin_field) or request.POST.get(self.plugin_field)
+            plugin_value = self.get_plugin_value(request)
             if plugin_value or len(self.plugin.get_plugins()) == 1:
                 self.plugin_value = plugin_value
                 if not plugin_value:

orchestra/plugins/forms.py

@@ -29,6 +29,7 @@ class PluginDataForm(forms.ModelForm):
                     self.fields[field].widget = ReadOnlyWidget(value, display)
     
     def clean(self):
+        super(PluginDataForm, self).clean()
         data = {}
         # Update data fields
         for field in self.declared_fields: