Improved admin UI performance

TODO.md

@@ -1,5 +1,4 @@
-TODO
+TODO ====
-====
 
 * scape strings before executing scripts in order to prevent exploits: django templates automatically scapes things. Most important is to ensuer that all escape ' to &quot
 * Don't store passwords and other service parameters that can be changed by the services i.e. mailman, vps etc. Find an execution mechanism that trigger `change_password()`
@@ -163,3 +162,21 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
 
 
 * Domain validation has to be done with injected records and subdomains
+
+* Names: lower andupper case allow or disallow ? webapps/account.username etc
+
+* Split plans into a separate app (plans and rates / services ) ? 
+
+* reconsider binding webapps to systemusers (pangea multiple users wordpress-ftp, moodle-pangea, etc)
+
+* sync() ServiceController method that synchronizes orchestra and servers (delete or import)
+
+* validate address.forward: if mailbox in account.mailboxes then: _("Please use mailboxes field") or consider removing mailbox support on forward (user@pangea.org instead)
+* reespell systemuser to system_user
+* remove order in account admin and others
+
+* create admin prefetch_related on ExtendedModelAdmin
+
+* Databases.User add reverse M2M databases widget (like mailbox.addresses)
+
+* One domain zone validation for each save, not one per subdomain, maybe on modeladmin.save_related? prevent save on model_related, and save it on save_related()

orchestra/admin/options.py

@@ -154,7 +154,13 @@ class ChangeAddFieldsMixin(object):
 
 
 class ExtendedModelAdmin(ChangeViewActionsMixin, ChangeAddFieldsMixin, admin.ModelAdmin):
-    pass
+    prefetch_related = None
+    
+    def get_queryset(self, request):
+        qs = super(ExtendedModelAdmin, self).get_queryset(request)
+        if self.prefetch_related:
+            qs = qs.prefetch_related(*self.prefetch_related)
+        return qs
 
 
 class SelectPluginAdminMixin(object):

orchestra/apps/bills/settings.py

@@ -58,7 +58,7 @@ BILLS_ORDER_MODEL = getattr(settings, 'BILLS_ORDER_MODEL', 'orders.Order')
 BILLS_CONTACT_DEFAULT_CITY = getattr(settings, 'BILLS_CONTACT_DEFAULT_CITY', 'Barcelona')
 
 
-BILLS_CONTACT_COUNTRIES = getattr(settings, 'BILLS_CONTACT_COUNTRIES', data.COUNTRIES)
+BILLS_CONTACT_COUNTRIES = getattr(settings, 'BILLS_CONTACT_COUNTRIES', ((k,v) for k,v in data.COUNTRIES.iteritems()))
 
 
 BILLS_CONTACT_DEFAULT_COUNTRY = getattr(settings, 'BILLS_CONTACT_DEFAULT_COUNTRY', 'ES')

orchestra/apps/contacts/models.py

@@ -52,7 +52,8 @@ class Contact(models.Model):
             validators=[RegexValidator(r'^[0-9,A-Z]{3,10}$',
                         _("Enter a valid zipcode."), 'invalid')])
     country = models.CharField(_("country"), max_length=20, blank=True,
-            choices=settings.CONTACTS_COUNTRIES)
+            choices=settings.CONTACTS_COUNTRIES,
+            default=settings.CONTACTS_DEFAULT_COUNTRY)
     
     def __unicode__(self):
         return self.short_name

orchestra/apps/contacts/settings.py

@@ -10,7 +10,7 @@ CONTACTS_DEFAULT_EMAIL_USAGES = getattr(settings, 'CONTACTS_DEFAULT_EMAIL_USAGES
 CONTACTS_DEFAULT_CITY = getattr(settings, 'CONTACTS_DEFAULT_CITY', 'Barcelona')
 
 
-CONTACTS_COUNTRIES = getattr(settings, 'CONTACTS_COUNTRIES', data.COUNTRIES)
+CONTACTS_COUNTRIES = getattr(settings, 'CONTACTS_COUNTRIES', ((k,v) for k,v in data.COUNTRIES.iteritems()))
 
 
 CONTACTS_DEFAULT_COUNTRY = getattr(settings, 'CONTACTS_DEFAULT_COUNTRY', 'ES')

orchestra/apps/databases/admin.py

@@ -41,13 +41,15 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
     add_form = DatabaseCreationForm
     readonly_fields = ('account_link', 'display_users',)
     filter_horizontal = ['users']
+    filter_by_account_fields = ('users',)
+    prefetch_related = ('users',)
     
     def display_users(self, db):
         links = []
         for user in db.users.all():
             link = '<a href="%s">%s</a>' % (change_url(user), user.username)
             links.append(link)
-        return ', '.join(links)
+        return '<br>'.join(links)
     display_users.short_description = _("Users")
     display_users.allow_tags = True
     display_users.admin_order_field = 'users__username'
@@ -87,13 +89,15 @@ class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, Exten
         }),
     )
     readonly_fields = ('account_link', 'display_databases',)
+    filter_by_account_fields = ('databases',)
+    prefetch_related = ('databases',)
     
     def display_databases(self, user):
         links = []
         for db in user.databases.all():
             link = '<a href="%s">%s</a>' % (change_url(db), db.name)
             links.append(link)
-        return ', '.join(links)
+        return '<br>'.join(links)
     display_databases.short_description = _("Databases")
     display_databases.allow_tags = True
     display_databases.admin_order_field = 'databases__name'

orchestra/apps/mailboxes/admin.py

@@ -59,6 +59,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo
     change_readonly_fields = ('name',)
     add_form = MailboxCreationForm
     form = MailboxChangeForm
+    prefetch_related = ('addresses__domain',)
     
     def display_addresses(self, mailbox):
         addresses = []
@@ -104,6 +105,7 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
     filter_by_account_fields = ('domain', 'mailboxes')
     filter_horizontal = ['mailboxes']
     form = AddressForm
+    prefetch_related = ('mailboxes', 'domain')
     
     domain_link = admin_link('domain', order='domain__name')
     
@@ -133,11 +135,6 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
             kwargs['widget'] = forms.TextInput(attrs={'size':'118'})
         return super(AddressAdmin, self).formfield_for_dbfield(db_field, **kwargs)
     
-    def get_queryset(self, request):
-        """ Select related for performance """
-        qs = super(AddressAdmin, self).get_queryset(request)
-        return qs.select_related('domain')
-    
     def get_fields(self, request, obj=None):
         """ Remove mailboxes field when creating address from a popup i.e. from mailbox add form """
         fields = super(AddressAdmin, self).get_fields(request, obj=obj)

orchestra/apps/mailboxes/forms.py

@@ -13,7 +13,7 @@ class MailboxForm(forms.ModelForm):
     """ hacky form for adding reverse M2M form field for Mailbox.addresses """
     # TODO keep track of this ticket for future reimplementation
     #      https://code.djangoproject.com/ticket/897
-    addresses = forms.ModelMultipleChoiceField(queryset=Address.objects, required=False,
+    addresses = forms.ModelMultipleChoiceField(queryset=Address.objects.select_related('domain'), required=False,
             widget=widgets.FilteredSelectMultiple(verbose_name=_('addresses'), is_stacked=False))
     
     def __init__(self, *args, **kwargs):

orchestra/apps/mailboxes/models.py

@@ -76,8 +76,9 @@ class Mailbox(models.Model):
 
 
 class Address(models.Model):
-    name = models.CharField(_("name"), max_length=64,
+    name = models.CharField(_("name"), max_length=64, blank=True,
-            validators=[validators.validate_emailname])
+            validators=[validators.validate_emailname],
+            help_text=_("Address name, left blank for a <i>catch-all</i> address"))
     domain = models.ForeignKey(settings.MAILBOXES_DOMAIN_MODEL,
             verbose_name=_("domain"),
             related_name='addresses')

orchestra/apps/mailboxes/validators.py

@@ -26,7 +26,11 @@ def validate_emailname(value):
 def validate_forward(value):
     """ space separated mailboxes or emails """
     from .models import Mailbox
+    destinations = []
     for destination in value.split():
+        if destination in destinations:
+            raise ValidationError(_("'%s' is already present.") % destination)
+        destinations.append(destination)
         msg = _("'%s' is not an existent mailbox" % destination)
         if '@' in destination:
             if not destination[-1].isalpha():

orchestra/apps/orchestration/methods.py

@@ -20,7 +20,6 @@ transports = {}
 
 def BashSSH(backend, log, server, cmds):
     from .models import BackendLog
-    # TODO save remote file into a root read only directory to avoid users sniffing passwords and stuff
     
     script = '\n'.join(['set -e', 'set -o pipefail'] + cmds + ['exit 0'])
     script = script.replace('\r', '')
@@ -36,8 +35,8 @@ def BashSSH(backend, log, server, cmds):
         logger.debug('%s is going to be executed on %s' % (backend, server))
         # Avoid "Argument list too long" on large scripts by genereting a file
         # and scping it to the remote server
-        with open(path, 'w') as script_file:
+        with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as handle:
-            script_file.write(script)
+            handle.write(script)
         
         # ssh connection
         ssh = paramiko.SSHClient()

orchestra/apps/resources/admin.py

@@ -87,6 +87,7 @@ class ResourceDataAdmin(ExtendedModelAdmin):
     actions = (run_monitor,)
     change_view_actions = actions
     ordering = ('-updated_at',)
+    prefetch_related = ('content_object',)
 
     resource_link = admin_link('resource')
     content_object_link = admin_link('content_object')
@@ -97,10 +98,6 @@ class ResourceDataAdmin(ExtendedModelAdmin):
     display_unit.short_description = _("Unit")
     display_unit.admin_order_field = 'resource__unit'
 
-    def get_queryset(self, request):
-        queryset = super(ResourceDataAdmin, self).get_queryset(request)
-        return queryset.prefetch_related('content_object')
-
 
 class MonitorDataAdmin(ExtendedModelAdmin):
     list_display = ('id', 'monitor', 'display_created', 'value', 'content_object_link')

orchestra/apps/webapps/admin.py

@@ -34,15 +34,17 @@ class WebAppAdmin(AccountAdminMixin, ExtendedModelAdmin):
     inlines = [WebAppOptionInline]
     readonly_fields = ('account_link',)
     change_readonly_fields = ('name', 'type')
+    prefetch_related = ('content_set__website',)
     
     def display_websites(self, webapp):
         websites = []
-        for content in webapp.content_set.all().select_related('website'):
+        for content in webapp.content_set.all():
             website = content.website
             url = change_url(website)
             name = "%s on %s" % (website.name, content.path)
             websites.append('<a href="%s">%s</a>' % (url, name))
         add_url = reverse('admin:websites_website_add')
+        # TODO support for preselecting related we app on website
         add_url += '?account=%s' % webapp.account_id
         plus = '<strong style="color:green; font-size:12px">+</strong>'
         websites.append('<a href="%s">%s%s</a>' % (add_url, plus, ugettext("Add website")))

orchestra/apps/webapps/settings.py

@@ -217,6 +217,10 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', {
         _("FCGI - IO timeout"),
         r'^[0-9]{1,3}$'
     ),
+    'FcgidProcessLifeTime': (
+        _("FCGI - IO timeout"),
+        r'^[0-9]{1,4}$'
+    ),
 })
 
 

orchestra/apps/websites/admin.py

@@ -55,6 +55,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
         }),
     )
     filter_by_account_fields = ['domains']
+    prefetch_related = ('domains', 'content_set__webapp')
     
     def display_domains(self, website):
         domains = []
@@ -67,7 +68,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
     
     def display_webapps(self, website):
         webapps = []
-        for content in website.content_set.all().select_related('webapp'):
+        for content in website.content_set.all():
             webapp = content.webapp
             url = change_url(webapp)
             name = "%s on %s" % (webapp.get_type_display(), content.path)
@@ -81,10 +82,5 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
             kwargs['widget'] = forms.TextInput(attrs={'size':'100'})
         return super(WebsiteAdmin, self).formfield_for_dbfield(db_field, **kwargs)
 
-    def get_queryset(self, request):
-        """ Select related for performance """
-        qs = super(WebsiteAdmin, self).get_queryset(request)
-        return qs.prefetch_related('domains')
-
 
 admin.site.register(Website, WebsiteAdmin)

orchestra/core/validators.py

@@ -44,8 +44,8 @@ def validate_name(value):
     """
     A single non-empty line of free-form text with no whitespace.
     """
-    validators.RegexValidator('^[\.\w\-]+$',
+    validators.RegexValidator('^[\.\_\-0-9a-z]+$',
-        _("Enter a valid name (text without whitspaces)."), 'invalid')(value)
+        _("Enter a valid name (spaceless lowercase text including _.-)."), 'invalid')(value)
 
 
 def validate_ascii(value):