From d1163602126fa806420c487777e1c1eb742c0e4d Mon Sep 17 00:00:00 2001 From: root Date: Sun, 2 Nov 2014 14:33:55 +0000 Subject: [PATCH] Improved admin UI performance --- TODO.md | 21 +++++++++++++++++++-- orchestra/admin/options.py | 8 +++++++- orchestra/apps/bills/settings.py | 2 +- orchestra/apps/contacts/models.py | 3 ++- orchestra/apps/contacts/settings.py | 2 +- orchestra/apps/databases/admin.py | 8 ++++++-- orchestra/apps/mailboxes/admin.py | 7 ++----- orchestra/apps/mailboxes/forms.py | 2 +- orchestra/apps/mailboxes/models.py | 5 +++-- orchestra/apps/mailboxes/validators.py | 4 ++++ orchestra/apps/orchestration/methods.py | 5 ++--- orchestra/apps/resources/admin.py | 7 ++----- orchestra/apps/webapps/admin.py | 4 +++- orchestra/apps/webapps/settings.py | 4 ++++ orchestra/apps/websites/admin.py | 8 ++------ orchestra/core/validators.py | 4 ++-- 16 files changed, 61 insertions(+), 33 deletions(-) diff --git a/TODO.md b/TODO.md index 5b3c4475..1dea9801 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,4 @@ -TODO -==== +TODO ==== * scape strings before executing scripts in order to prevent exploits: django templates automatically scapes things. Most important is to ensuer that all escape ' to " * Don't store passwords and other service parameters that can be changed by the services i.e. mailman, vps etc. Find an execution mechanism that trigger `change_password()` @@ -163,3 +162,21 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * Domain validation has to be done with injected records and subdomains + +* Names: lower andupper case allow or disallow ? webapps/account.username etc + +* Split plans into a separate app (plans and rates / services ) ? + +* reconsider binding webapps to systemusers (pangea multiple users wordpress-ftp, moodle-pangea, etc) + +* sync() ServiceController method that synchronizes orchestra and servers (delete or import) + +* validate address.forward: if mailbox in account.mailboxes then: _("Please use mailboxes field") or consider removing mailbox support on forward (user@pangea.org instead) +* reespell systemuser to system_user +* remove order in account admin and others + +* create admin prefetch_related on ExtendedModelAdmin + +* Databases.User add reverse M2M databases widget (like mailbox.addresses) + +* One domain zone validation for each save, not one per subdomain, maybe on modeladmin.save_related? prevent save on model_related, and save it on save_related() diff --git a/orchestra/admin/options.py b/orchestra/admin/options.py index 0e281921..f99d8f3a 100644 --- a/orchestra/admin/options.py +++ b/orchestra/admin/options.py @@ -154,7 +154,13 @@ class ChangeAddFieldsMixin(object): class ExtendedModelAdmin(ChangeViewActionsMixin, ChangeAddFieldsMixin, admin.ModelAdmin): - pass + prefetch_related = None + + def get_queryset(self, request): + qs = super(ExtendedModelAdmin, self).get_queryset(request) + if self.prefetch_related: + qs = qs.prefetch_related(*self.prefetch_related) + return qs class SelectPluginAdminMixin(object): diff --git a/orchestra/apps/bills/settings.py b/orchestra/apps/bills/settings.py index 000c18b2..3357ada8 100644 --- a/orchestra/apps/bills/settings.py +++ b/orchestra/apps/bills/settings.py @@ -58,7 +58,7 @@ BILLS_ORDER_MODEL = getattr(settings, 'BILLS_ORDER_MODEL', 'orders.Order') BILLS_CONTACT_DEFAULT_CITY = getattr(settings, 'BILLS_CONTACT_DEFAULT_CITY', 'Barcelona') -BILLS_CONTACT_COUNTRIES = getattr(settings, 'BILLS_CONTACT_COUNTRIES', data.COUNTRIES) +BILLS_CONTACT_COUNTRIES = getattr(settings, 'BILLS_CONTACT_COUNTRIES', ((k,v) for k,v in data.COUNTRIES.iteritems())) BILLS_CONTACT_DEFAULT_COUNTRY = getattr(settings, 'BILLS_CONTACT_DEFAULT_COUNTRY', 'ES') diff --git a/orchestra/apps/contacts/models.py b/orchestra/apps/contacts/models.py index 9e10d95b..e2a76e10 100644 --- a/orchestra/apps/contacts/models.py +++ b/orchestra/apps/contacts/models.py @@ -52,7 +52,8 @@ class Contact(models.Model): validators=[RegexValidator(r'^[0-9,A-Z]{3,10}$', _("Enter a valid zipcode."), 'invalid')]) country = models.CharField(_("country"), max_length=20, blank=True, - choices=settings.CONTACTS_COUNTRIES) + choices=settings.CONTACTS_COUNTRIES, + default=settings.CONTACTS_DEFAULT_COUNTRY) def __unicode__(self): return self.short_name diff --git a/orchestra/apps/contacts/settings.py b/orchestra/apps/contacts/settings.py index 8c663d15..00ed01cb 100644 --- a/orchestra/apps/contacts/settings.py +++ b/orchestra/apps/contacts/settings.py @@ -10,7 +10,7 @@ CONTACTS_DEFAULT_EMAIL_USAGES = getattr(settings, 'CONTACTS_DEFAULT_EMAIL_USAGES CONTACTS_DEFAULT_CITY = getattr(settings, 'CONTACTS_DEFAULT_CITY', 'Barcelona') -CONTACTS_COUNTRIES = getattr(settings, 'CONTACTS_COUNTRIES', data.COUNTRIES) +CONTACTS_COUNTRIES = getattr(settings, 'CONTACTS_COUNTRIES', ((k,v) for k,v in data.COUNTRIES.iteritems())) CONTACTS_DEFAULT_COUNTRY = getattr(settings, 'CONTACTS_DEFAULT_COUNTRY', 'ES') diff --git a/orchestra/apps/databases/admin.py b/orchestra/apps/databases/admin.py index 65044492..8bbd05bd 100644 --- a/orchestra/apps/databases/admin.py +++ b/orchestra/apps/databases/admin.py @@ -41,13 +41,15 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): add_form = DatabaseCreationForm readonly_fields = ('account_link', 'display_users',) filter_horizontal = ['users'] + filter_by_account_fields = ('users',) + prefetch_related = ('users',) def display_users(self, db): links = [] for user in db.users.all(): link = '%s' % (change_url(user), user.username) links.append(link) - return ', '.join(links) + return '
'.join(links) display_users.short_description = _("Users") display_users.allow_tags = True display_users.admin_order_field = 'users__username' @@ -87,13 +89,15 @@ class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, Exten }), ) readonly_fields = ('account_link', 'display_databases',) + filter_by_account_fields = ('databases',) + prefetch_related = ('databases',) def display_databases(self, user): links = [] for db in user.databases.all(): link = '%s' % (change_url(db), db.name) links.append(link) - return ', '.join(links) + return '
'.join(links) display_databases.short_description = _("Databases") display_databases.allow_tags = True display_databases.admin_order_field = 'databases__name' diff --git a/orchestra/apps/mailboxes/admin.py b/orchestra/apps/mailboxes/admin.py index 4b10fbba..0dbeafc5 100644 --- a/orchestra/apps/mailboxes/admin.py +++ b/orchestra/apps/mailboxes/admin.py @@ -59,6 +59,7 @@ class MailboxAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedMo change_readonly_fields = ('name',) add_form = MailboxCreationForm form = MailboxChangeForm + prefetch_related = ('addresses__domain',) def display_addresses(self, mailbox): addresses = [] @@ -104,6 +105,7 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): filter_by_account_fields = ('domain', 'mailboxes') filter_horizontal = ['mailboxes'] form = AddressForm + prefetch_related = ('mailboxes', 'domain') domain_link = admin_link('domain', order='domain__name') @@ -133,11 +135,6 @@ class AddressAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): kwargs['widget'] = forms.TextInput(attrs={'size':'118'}) return super(AddressAdmin, self).formfield_for_dbfield(db_field, **kwargs) - def get_queryset(self, request): - """ Select related for performance """ - qs = super(AddressAdmin, self).get_queryset(request) - return qs.select_related('domain') - def get_fields(self, request, obj=None): """ Remove mailboxes field when creating address from a popup i.e. from mailbox add form """ fields = super(AddressAdmin, self).get_fields(request, obj=obj) diff --git a/orchestra/apps/mailboxes/forms.py b/orchestra/apps/mailboxes/forms.py index 3ea407af..60d19a7c 100644 --- a/orchestra/apps/mailboxes/forms.py +++ b/orchestra/apps/mailboxes/forms.py @@ -13,7 +13,7 @@ class MailboxForm(forms.ModelForm): """ hacky form for adding reverse M2M form field for Mailbox.addresses """ # TODO keep track of this ticket for future reimplementation # https://code.djangoproject.com/ticket/897 - addresses = forms.ModelMultipleChoiceField(queryset=Address.objects, required=False, + addresses = forms.ModelMultipleChoiceField(queryset=Address.objects.select_related('domain'), required=False, widget=widgets.FilteredSelectMultiple(verbose_name=_('addresses'), is_stacked=False)) def __init__(self, *args, **kwargs): diff --git a/orchestra/apps/mailboxes/models.py b/orchestra/apps/mailboxes/models.py index 05b42be1..708d32d9 100644 --- a/orchestra/apps/mailboxes/models.py +++ b/orchestra/apps/mailboxes/models.py @@ -76,8 +76,9 @@ class Mailbox(models.Model): class Address(models.Model): - name = models.CharField(_("name"), max_length=64, - validators=[validators.validate_emailname]) + name = models.CharField(_("name"), max_length=64, blank=True, + validators=[validators.validate_emailname], + help_text=_("Address name, left blank for a catch-all address")) domain = models.ForeignKey(settings.MAILBOXES_DOMAIN_MODEL, verbose_name=_("domain"), related_name='addresses') diff --git a/orchestra/apps/mailboxes/validators.py b/orchestra/apps/mailboxes/validators.py index ce57a501..8bee96e2 100644 --- a/orchestra/apps/mailboxes/validators.py +++ b/orchestra/apps/mailboxes/validators.py @@ -26,7 +26,11 @@ def validate_emailname(value): def validate_forward(value): """ space separated mailboxes or emails """ from .models import Mailbox + destinations = [] for destination in value.split(): + if destination in destinations: + raise ValidationError(_("'%s' is already present.") % destination) + destinations.append(destination) msg = _("'%s' is not an existent mailbox" % destination) if '@' in destination: if not destination[-1].isalpha(): diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index dd1bf784..3bc65fb1 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -20,7 +20,6 @@ transports = {} def BashSSH(backend, log, server, cmds): from .models import BackendLog - # TODO save remote file into a root read only directory to avoid users sniffing passwords and stuff script = '\n'.join(['set -e', 'set -o pipefail'] + cmds + ['exit 0']) script = script.replace('\r', '') @@ -36,8 +35,8 @@ def BashSSH(backend, log, server, cmds): logger.debug('%s is going to be executed on %s' % (backend, server)) # Avoid "Argument list too long" on large scripts by genereting a file # and scping it to the remote server - with open(path, 'w') as script_file: - script_file.write(script) + with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT, 0600), 'w') as handle: + handle.write(script) # ssh connection ssh = paramiko.SSHClient() diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 4cc4555e..b848dbbb 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -87,7 +87,8 @@ class ResourceDataAdmin(ExtendedModelAdmin): actions = (run_monitor,) change_view_actions = actions ordering = ('-updated_at',) - + prefetch_related = ('content_object',) + resource_link = admin_link('resource') content_object_link = admin_link('content_object') display_updated = admin_date('updated_at', short_description=_("Updated")) @@ -96,10 +97,6 @@ class ResourceDataAdmin(ExtendedModelAdmin): return data.unit display_unit.short_description = _("Unit") display_unit.admin_order_field = 'resource__unit' - - def get_queryset(self, request): - queryset = super(ResourceDataAdmin, self).get_queryset(request) - return queryset.prefetch_related('content_object') class MonitorDataAdmin(ExtendedModelAdmin): diff --git a/orchestra/apps/webapps/admin.py b/orchestra/apps/webapps/admin.py index 549a1e9a..c5370350 100644 --- a/orchestra/apps/webapps/admin.py +++ b/orchestra/apps/webapps/admin.py @@ -34,15 +34,17 @@ class WebAppAdmin(AccountAdminMixin, ExtendedModelAdmin): inlines = [WebAppOptionInline] readonly_fields = ('account_link',) change_readonly_fields = ('name', 'type') + prefetch_related = ('content_set__website',) def display_websites(self, webapp): websites = [] - for content in webapp.content_set.all().select_related('website'): + for content in webapp.content_set.all(): website = content.website url = change_url(website) name = "%s on %s" % (website.name, content.path) websites.append('%s' % (url, name)) add_url = reverse('admin:websites_website_add') + # TODO support for preselecting related we app on website add_url += '?account=%s' % webapp.account_id plus = '+' websites.append('%s%s' % (add_url, plus, ugettext("Add website"))) diff --git a/orchestra/apps/webapps/settings.py b/orchestra/apps/webapps/settings.py index e013737c..7170d997 100644 --- a/orchestra/apps/webapps/settings.py +++ b/orchestra/apps/webapps/settings.py @@ -217,6 +217,10 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', { _("FCGI - IO timeout"), r'^[0-9]{1,3}$' ), + 'FcgidProcessLifeTime': ( + _("FCGI - IO timeout"), + r'^[0-9]{1,4}$' + ), }) diff --git a/orchestra/apps/websites/admin.py b/orchestra/apps/websites/admin.py index 74436223..a0083bdb 100644 --- a/orchestra/apps/websites/admin.py +++ b/orchestra/apps/websites/admin.py @@ -55,6 +55,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): }), ) filter_by_account_fields = ['domains'] + prefetch_related = ('domains', 'content_set__webapp') def display_domains(self, website): domains = [] @@ -67,7 +68,7 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): def display_webapps(self, website): webapps = [] - for content in website.content_set.all().select_related('webapp'): + for content in website.content_set.all(): webapp = content.webapp url = change_url(webapp) name = "%s on %s" % (webapp.get_type_display(), content.path) @@ -80,11 +81,6 @@ class WebsiteAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): if db_field.name == 'root': kwargs['widget'] = forms.TextInput(attrs={'size':'100'}) return super(WebsiteAdmin, self).formfield_for_dbfield(db_field, **kwargs) - - def get_queryset(self, request): - """ Select related for performance """ - qs = super(WebsiteAdmin, self).get_queryset(request) - return qs.prefetch_related('domains') admin.site.register(Website, WebsiteAdmin) diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index 2f237bfb..76f3dbac 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -44,8 +44,8 @@ def validate_name(value): """ A single non-empty line of free-form text with no whitespace. """ - validators.RegexValidator('^[\.\w\-]+$', - _("Enter a valid name (text without whitspaces)."), 'invalid')(value) + validators.RegexValidator('^[\.\_\-0-9a-z]+$', + _("Enter a valid name (spaceless lowercase text including _.-)."), 'invalid')(value) def validate_ascii(value):