From d85ada93e7a0e2169b5e527021e09f927039c185 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Mon, 27 Oct 2014 14:31:04 +0000 Subject: [PATCH] Improved mailman backend --- TODO.md | 29 ++------------ orchestra/apps/accounts/admin.py | 4 +- .../apps/accounts/migrations/0001_initial.py | 39 +++++++++++++++++++ .../migrations/0002_auto_20141027_1414.py | 34 ++++++++++++++++ .../apps/accounts/migrations/__init__.py | 0 orchestra/apps/accounts/models.py | 13 +++---- orchestra/apps/accounts/serializers.py | 3 +- orchestra/apps/bills/models.py | 6 ++- .../templates/bills/microspective-fee.html | 2 +- .../bills/templates/bills/microspective.html | 4 +- orchestra/apps/lists/backends.py | 16 +++++--- orchestra/apps/orchestration/manager.py | 13 +++++-- orchestra/apps/resources/serializers.py | 3 +- orchestra/apps/websites/backends/apache.py | 4 +- orchestra/apps/websites/models.py | 6 +++ orchestra/apps/websites/settings.py | 4 +- 16 files changed, 126 insertions(+), 54 deletions(-) create mode 100644 orchestra/apps/accounts/migrations/0001_initial.py create mode 100644 orchestra/apps/accounts/migrations/0002_auto_20141027_1414.py create mode 100644 orchestra/apps/accounts/migrations/__init__.py diff --git a/TODO.md b/TODO.md index 926ab0e8..92c4c39d 100644 --- a/TODO.md +++ b/TODO.md @@ -126,45 +126,21 @@ Remember that, as always with QuerySets, any subsequent chained methods which im -# Required-Start: $network $local_fs $remote_fs postgresql celeryd -# Required-Stop: $network $local_fs $remote_fs postgresql celeryd - * for list virtual_domains cleaning up we need to know the old domain name when a list changes its address domain, but this is not possible with the current design. * regenerate virtual_domains every time (configure a separate file for orchestra on postfix) * update_fields=[] doesn't trigger post save! -* lists -> SaaS ? - -* move bill contact to bills apps - - * Backend optimization * fields = () * ignore_fields = () * based on a merge set of save(update_fields) - * textwrap.dedent( \\) - -* accounts - * short name / long name, account name really needed? address? only minimal info.. - * contact inlines - * autocreate stuff (email/.orchestra.lan/plans) - * account username should be domain freiendly withot lines - - * parmiko write to a channel instead of transfering files? http://sysadmin.circularvale.com/programming/paramiko-channel-hangs/ -* strip leading and trailing whitre spaces of most input fields - -* better modeling of the interdependency between webapps and websites (settings) -* webapp options cfig agnostic - -* service.name / verbose_name instead of .description ? -* miscellaneous.name / verbose_name * proforma without billing contact? -* remove contact addresss, and use invoice contact for it (maybe move to contacts app again) - * env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python manage.py test orchestra.apps.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest * Pangea modifications: domain registered/non-registered list_display and field with register link: inconsistent, what happen to related objects with a domain that is converted to register-only? @@ -175,7 +151,7 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * REST PERMISSIONS -* caching based on def text2int(textnum, numwords={}): +* caching based on def text2int(textnum, numwords={}) ?: * Subdomain saving should not trigger bind slave @@ -184,3 +160,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * prevent adding local email addresses on account.contacts account.email * Resource monitoring without ROUTE alert or explicit error + + +* account.full_name account.short_name diff --git a/orchestra/apps/accounts/admin.py b/orchestra/apps/accounts/admin.py index 26460916..2883d605 100644 --- a/orchestra/apps/accounts/admin.py +++ b/orchestra/apps/accounts/admin.py @@ -34,7 +34,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) 'fields': ('username', 'password1', 'password2',), }), (_("Personal info"), { - 'fields': ('first_name', 'last_name', 'email', ('type', 'language'), 'comments'), + 'fields': ('short_name', 'full_name', 'email', ('type', 'language'), 'comments'), }), (_("Permissions"), { 'fields': ('is_superuser',) @@ -45,7 +45,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) 'fields': ('username', 'password', 'main_systemuser_link') }), (_("Personal info"), { - 'fields': ('first_name', 'last_name', 'email', ('type', 'language'), 'comments'), + 'fields': ('short_name', 'full_name', 'email', ('type', 'language'), 'comments'), }), (_("Permissions"), { 'fields': ('is_superuser', 'is_active') diff --git a/orchestra/apps/accounts/migrations/0001_initial.py b/orchestra/apps/accounts/migrations/0001_initial.py new file mode 100644 index 00000000..97154ba7 --- /dev/null +++ b/orchestra/apps/accounts/migrations/0001_initial.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +import django.utils.timezone +import django.core.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('systemusers', '__first__'), + ] + + operations = [ + migrations.CreateModel( + name='Account', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(default=django.utils.timezone.now, verbose_name='last login')), + ('username', models.CharField(help_text='Required. 30 characters or fewer. Letters, digits and ./-/_ only.', unique=True, max_length=64, verbose_name='username', validators=[django.core.validators.RegexValidator(b'^[\\w.-]+$', 'Enter a valid username.', b'invalid')])), + ('first_name', models.CharField(max_length=30, verbose_name='first name', blank=True)), + ('last_name', models.CharField(max_length=30, verbose_name='last name', blank=True)), + ('email', models.EmailField(help_text='Used for password recovery', max_length=75, verbose_name='email address')), + ('type', models.CharField(default=b'INDIVIDUAL', max_length=32, verbose_name='type', choices=[(b'INDIVIDUAL', 'Individual'), (b'ASSOCIATION', 'Association'), (b'CUSTOMER', 'Customer'), (b'STAFF', 'Staff')])), + ('language', models.CharField(default=b'ca', max_length=2, verbose_name='language', choices=[(b'ca', 'Catalan'), (b'es', 'Spanish'), (b'en', 'English')])), + ('comments', models.TextField(max_length=256, verbose_name='comments', blank=True)), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this account should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('main_systemuser', models.ForeignKey(related_name='accounts_main', to='systemusers.SystemUser', null=True)), + ], + options={ + 'abstract': False, + }, + bases=(models.Model,), + ), + ] diff --git a/orchestra/apps/accounts/migrations/0002_auto_20141027_1414.py b/orchestra/apps/accounts/migrations/0002_auto_20141027_1414.py new file mode 100644 index 00000000..7cced87f --- /dev/null +++ b/orchestra/apps/accounts/migrations/0002_auto_20141027_1414.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='account', + name='first_name', + ), + migrations.RemoveField( + model_name='account', + name='last_name', + ), + migrations.AddField( + model_name='account', + name='full_name', + field=models.CharField(default='', max_length=30, verbose_name='full name'), + preserve_default=False, + ), + migrations.AddField( + model_name='account', + name='short_name', + field=models.CharField(default='', max_length=30, verbose_name='short name', blank=True), + preserve_default=False, + ), + ] diff --git a/orchestra/apps/accounts/migrations/__init__.py b/orchestra/apps/accounts/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/accounts/models.py b/orchestra/apps/accounts/models.py index 23b4fea0..b2891956 100644 --- a/orchestra/apps/accounts/models.py +++ b/orchestra/apps/accounts/models.py @@ -19,8 +19,8 @@ class Account(auth.AbstractBaseUser): _("Enter a valid username."), 'invalid')]) main_systemuser = models.ForeignKey(settings.ACCOUNTS_SYSTEMUSER_MODEL, null=True, related_name='accounts_main') - first_name = models.CharField(_("first name"), max_length=30, blank=True) - last_name = models.CharField(_("last name"), max_length=30, blank=True) + short_name = models.CharField(_("short name"), max_length=30, blank=True) + full_name = models.CharField(_("full name"), max_length=30) email = models.EmailField(_('email address'), help_text=_("Used for password recovery")) type = models.CharField(_("type"), choices=settings.ACCOUNTS_TYPES, max_length=32, default=settings.ACCOUNTS_DEFAULT_TYPE) @@ -69,8 +69,8 @@ class Account(auth.AbstractBaseUser): self.save(update_fields=['main_systemuser']) def clean(self): - self.first_name = self.first_name.strip() - self.last_name = self.last_name.strip() + self.short_name = self.short_name.strip() + self.full_name = self.full_name.strip() def disable(self): self.is_active = False @@ -93,12 +93,11 @@ class Account(auth.AbstractBaseUser): send_email_template(template, context, email_to, html=html, attachments=attachments) def get_full_name(self): - full_name = '%s %s' % (self.first_name, self.last_name) - return full_name.strip() or self.username + return self.full_name or self.short_name or self.username def get_short_name(self): """ Returns the short name for the user """ - return self.first_name + return self.short_name or self.username or self.full_name def has_perm(self, perm, obj=None): """ diff --git a/orchestra/apps/accounts/serializers.py b/orchestra/apps/accounts/serializers.py index e58dd34e..8fb621e6 100644 --- a/orchestra/apps/accounts/serializers.py +++ b/orchestra/apps/accounts/serializers.py @@ -7,7 +7,8 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Account fields = ( - 'url', 'username', 'type', 'language', 'date_joined', 'is_active' + 'url', 'username', 'type', 'language', 'short_name', 'full_name', 'date_joined', + 'is_active' ) diff --git a/orchestra/apps/bills/models.py b/orchestra/apps/bills/models.py index 60d6302d..72406880 100644 --- a/orchestra/apps/bills/models.py +++ b/orchestra/apps/bills/models.py @@ -19,7 +19,8 @@ from . import settings class BillContact(models.Model): account = models.OneToOneField('accounts.Account', verbose_name=_("account"), related_name='billcontact') - name = models.CharField(_("name"), max_length=256) + name = models.CharField(_("name"), max_length=256, blank=True, + help_text=_("Account full name will be used when not provided")) address = models.TextField(_("address")) city = models.CharField(_("city"), max_length=128, default=settings.BILLS_CONTACT_DEFAULT_CITY) @@ -30,6 +31,9 @@ class BillContact(models.Model): def __unicode__(self): return self.name + + def get_name(self): + return self.name or self.account.get_full_name() class BillManager(models.Manager): diff --git a/orchestra/apps/bills/templates/bills/microspective-fee.html b/orchestra/apps/bills/templates/bills/microspective-fee.html index 87625dab..7c4d4da0 100644 --- a/orchestra/apps/bills/templates/bills/microspective-fee.html +++ b/orchestra/apps/bills/templates/bills/microspective-fee.html @@ -87,7 +87,7 @@ hr {
- {{ buyer.name }}
+ {{ buyer.get_name }}
{{ buyer.vat }}
{{ buyer.address }}
{{ buyer.zipcode }} - {{ buyer.city }}
diff --git a/orchestra/apps/bills/templates/bills/microspective.html b/orchestra/apps/bills/templates/bills/microspective.html index 06e2fa05..1916bd2d 100644 --- a/orchestra/apps/bills/templates/bills/microspective.html +++ b/orchestra/apps/bills/templates/bills/microspective.html @@ -23,7 +23,7 @@
- {{ seller.name }} + {{ seller.get_name }}

{{ seller.address }}
@@ -58,7 +58,7 @@

- {{ buyer.name }}
+ {{ buyer.get_name }}
{{ buyer.vat }}
{{ buyer.address }}
{{ buyer.zipcode }} - {{ buyer.city }}
diff --git a/orchestra/apps/lists/backends.py b/orchestra/apps/lists/backends.py index 1b202b1a..65c5f96b 100644 --- a/orchestra/apps/lists/backends.py +++ b/orchestra/apps/lists/backends.py @@ -43,7 +43,7 @@ class MailmanBackend(ServiceController): def exclude_virtual_alias_domain(self, context): address_domain = context['address_domain'] if not List.objects.filter(address_domain=address_domain).exists(): - self.append('sed -i "/^%(address_domain)s\s*/d" %(virtual_alias_domains)s' % context) + self.append('sed -i "/^%(address_domain)s\s*$/d" %(virtual_alias_domains)s' % context) def get_virtual_aliases(self, context): aliases = [] @@ -72,7 +72,7 @@ class MailmanBackend(ServiceController): UPDATED_VIRTUAL_ALIAS=1 else if [[ ! $(grep '^\s*%(address_name)s@%(address_domain)s\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then - sed -i "s/^.*\s%(name)s\s*$//" %(virtual_alias)s + sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s echo '# %(banner)s\n%(aliases)s ' >> %(virtual_alias)s UPDATED_VIRTUAL_ALIAS=1 @@ -88,7 +88,7 @@ class MailmanBackend(ServiceController): # Cleanup shit self.append(textwrap.dedent("""\ if [[ ! $(grep '\s\s*%(name)s\s*$' %(virtual_alias)s) ]]; then - sed -i "s/^.*\s%(name)s\s*$//" %(virtual_alias)s + sed -i "/^.*\s%(name)s\s*$/d" %(virtual_alias)s fi""" % context )) # Update @@ -99,11 +99,11 @@ class MailmanBackend(ServiceController): def delete(self, mail_list): context = self.get_context(mail_list) self.exclude_virtual_alias_domain(context) + self.append('sed -i "/^\s*Generated by.*%(name)s\s*$/d" %(virtual_alias)s' % context) for address in self.addresses: context['address'] = address - self.append('sed -i "s/^.*\s%(name)s%(address)s\s*$//" %(virtual_alias)s' % context) - # TODO remove - self.append("echo rmlist -a %(name)s" % context) + self.append('sed -i "/^.*\s%(name)s%(address)s\s*$/d" %(virtual_alias)s' % context) + self.append("rmlist -a %(name)s" % context) def commit(self): context = self.get_context_files() @@ -119,6 +119,10 @@ class MailmanBackend(ServiceController): 'virtual_alias_domains': settings.LISTS_VIRTUAL_ALIAS_DOMAINS_PATH, } + def get_banner(self, mail_list): + banner = super(MailmanBackend, self).get_banner() + return '%s %s' % (banner, mail_list.name) + def get_context(self, mail_list): context = self.get_context_files() context.update({ diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index 6c1e85d3..7c770d88 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -30,10 +30,15 @@ def as_task(execute): def close_connection(execute): """ Threads have their own connection pool, closing it when finishing """ def wrapper(*args, **kwargs): - log = execute(*args, **kwargs) - db.connection.close() - # Using the wrapper function as threader messenger for the execute output - wrapper.log = log + try: + log = execute(*args, **kwargs) + except: + raise + else: + # Using the wrapper function as threader messenger for the execute output + wrapper.log = log + finally: + db.connection.close() return wrapper diff --git a/orchestra/apps/resources/serializers.py b/orchestra/apps/resources/serializers.py index 8ee14239..4dc2ee68 100644 --- a/orchestra/apps/resources/serializers.py +++ b/orchestra/apps/resources/serializers.py @@ -8,10 +8,11 @@ from .models import Resource, ResourceData class ResourceSerializer(serializers.ModelSerializer): name = serializers.SerializerMethodField('get_name') + unit = serializers.Field() class Meta: model = ResourceData - fields = ('name', 'used', 'allocated') + fields = ('name', 'used', 'allocated', 'unit') read_only_fields = ('used',) def from_native(self, raw_data, files=None): diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 6be3cbe5..24bceaf2 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -167,7 +167,7 @@ class Apache2Backend(ServiceController): 'group': site.get_groupname(), 'sites_enabled': sites_enabled, 'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name), - 'logs': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name), + 'logs': site.get_www_log_path(), 'banner': self.get_banner(), } return context @@ -237,7 +237,7 @@ class Apache2Traffic(ServiceMonitor): def get_context(self, site): last_date = self.get_last_date(site.pk) return { - 'log_file': os.path.join(settings.WEBSITES_BASE_APACHE_LOGS, site.unique_name), + 'log_file': '%s{,.1}' % site.get_www_log_path(), 'last_date': last_date.strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': site.pk, } diff --git a/orchestra/apps/websites/models.py b/orchestra/apps/websites/models.py index 372aa98f..57de95d8 100644 --- a/orchestra/apps/websites/models.py +++ b/orchestra/apps/websites/models.py @@ -55,6 +55,12 @@ class Website(models.Model): def get_groupname(self): return self.get_username() + + def get_www_log_path(self): + context = { + 'unique_name': self.unique_name + } + return settings.WEBSITES_WEBSITE_WWW_LOG_PATH % context class WebsiteOption(models.Model): diff --git a/orchestra/apps/websites/settings.py b/orchestra/apps/websites/settings.py index ffcaea49..1068f45c 100644 --- a/orchestra/apps/websites/settings.py +++ b/orchestra/apps/websites/settings.py @@ -50,5 +50,5 @@ WEBSITES_WEBALIZER_PATH = getattr(settings, 'WEBSITES_WEBALIZER_PATH', '/home/httpd/webalizer/') -WEBSITES_BASE_APACHE_LOGS = getattr(settings, 'WEBSITES_BASE_APACHE_LOGS', - '/var/log/apache2/virtual/') +WEBSITES_WEBSITE_WWW_LOG_PATH = getattr(settings, 'WEBSITES_WEBSITE_WWW_LOG_PATH', + '/var/log/apache2/virtual/%(unique_name)s')