From a04f5cc5da82117cdf71bc7a456e84f131d1a8a7 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Thu, 9 Apr 2015 14:32:10 +0000 Subject: [PATCH] Lots of random fixes --- TODO.md | 7 ++- orchestra/contrib/domains/admin.py | 18 +++---- orchestra/contrib/domains/forms.py | 1 + orchestra/contrib/domains/helpers.py | 5 +- orchestra/contrib/lists/admin.py | 10 ++-- orchestra/contrib/lists/backends.py | 2 +- orchestra/contrib/mailboxes/backends.py | 48 +++++++++++-------- orchestra/contrib/mailboxes/settings.py | 7 ++- orchestra/contrib/miscellaneous/admin.py | 2 +- orchestra/contrib/miscellaneous/models.py | 5 +- orchestra/contrib/orchestration/__init__.py | 2 +- .../management/commands/orchestrate.py | 15 +++++- orchestra/contrib/orchestration/manager.py | 2 +- orchestra/contrib/plans/admin.py | 4 +- orchestra/contrib/resources/admin.py | 2 +- orchestra/contrib/resources/aggregations.py | 40 +++++++++++----- orchestra/contrib/resources/models.py | 5 ++ orchestra/contrib/resources/tasks.py | 6 +-- .../contrib/saas/backends/wordpressmu.py | 8 ---- orchestra/contrib/systemusers/backends.py | 10 ++-- orchestra/contrib/systemusers/forms.py | 1 + orchestra/contrib/systemusers/settings.py | 5 ++ orchestra/contrib/webapps/admin.py | 11 +++-- .../contrib/webapps/backends/__init__.py | 8 +++- orchestra/contrib/webapps/backends/php.py | 13 +++-- .../contrib/webapps/backends/symboliclink.py | 23 +++++---- orchestra/contrib/webapps/filters.py | 22 +++++++++ orchestra/contrib/webapps/options.py | 8 ---- orchestra/contrib/webapps/settings.py | 11 ++++- orchestra/contrib/webapps/types/php.py | 13 +++-- orchestra/contrib/websites/backends/apache.py | 10 ++-- orchestra/contrib/websites/forms.py | 1 + orchestra/management/commands/makemessages.py | 2 +- orchestra/plugins/admin.py | 13 ++++- orchestra/plugins/forms.py | 1 + 35 files changed, 224 insertions(+), 117 deletions(-) create mode 100644 orchestra/contrib/webapps/filters.py diff --git a/TODO.md b/TODO.md index dfac45c0..662822ab 100644 --- a/TODO.md +++ b/TODO.md @@ -280,4 +280,9 @@ https://code.djangoproject.com/ticket/24576 # migrations accounts, bill, orders, auth -> migrate the rest (contacts lambda error) -# MultiCHoiceField proper serialization +* MultiCHoiceField proper serialization + +# Apache restart fails: detect if appache running, and execute start +# PHP backend is retarded does not detect well the version +# Change crons, create cron for deleted webapps and users +* UNIFY PHP FPM settings name diff --git a/orchestra/contrib/domains/admin.py b/orchestra/contrib/domains/admin.py index d55dfb38..bb910d17 100644 --- a/orchestra/contrib/domains/admin.py +++ b/orchestra/contrib/domains/admin.py @@ -103,14 +103,16 @@ class DomainAdmin(AccountAdminMixin, ExtendedModelAdmin): """ Order by structured name and imporve performance """ qs = super(DomainAdmin, self).get_queryset(request) qs = qs.select_related('top', 'account') - # For some reason if we do this we know for sure that join table will be called T4 - query = str(qs.query) - table = re.findall(r'(T\d+)\."account_id"', query)[0] - qs = qs.extra( - select={ - 'structured_name': 'CONCAT({table}.name, domains_domain.name)'.format(table=table) - }, - ).order_by('structured_name') + # Order by structured name + if request.method == 'GET': + # For some reason if we do this we know for sure that join table will be called T4 + query = str(qs.query) + table = re.findall(r'(T\d+)\."account_id"', query)[0] + qs = qs.extra( + select={ + 'structured_name': 'CONCAT({table}.name, domains_domain.name)'.format(table=table) + }, + ).order_by('structured_name') if apps.isinstalled('orchestra.contrib.websites'): qs = qs.prefetch_related('websites') return qs diff --git a/orchestra/contrib/domains/forms.py b/orchestra/contrib/domains/forms.py index 1b853a06..f1c2ad66 100644 --- a/orchestra/contrib/domains/forms.py +++ b/orchestra/contrib/domains/forms.py @@ -59,6 +59,7 @@ class BatchDomainCreationAdminForm(forms.ModelForm): class RecordInlineFormSet(forms.models.BaseInlineFormSet): def clean(self): """ Checks if everything is consistent """ + super(RecordInlineFormSet, self).clean() if any(self.errors): return if self.instance.name: diff --git a/orchestra/contrib/domains/helpers.py b/orchestra/contrib/domains/helpers.py index 07b6e2c0..f1e1842b 100644 --- a/orchestra/contrib/domains/helpers.py +++ b/orchestra/contrib/domains/helpers.py @@ -9,7 +9,7 @@ def domain_for_validation(instance, records): so when validation calls render_zone() it will use the new provided data """ domain = copy.copy(instance) - def get_records(): + def get_records(records=records): for data in records: yield Record(type=data['type'], value=data['value']) domain.get_records = get_records @@ -19,7 +19,8 @@ def domain_for_validation(instance, records): domain.top = domain.get_parent(top=True) if domain.top: # is a subdomain - subdomains = [sub for sub in domain.top.subdomains.all() if sub.pk != domain.pk] + subdomains = domain.top.subdomains.select_related('top').prefetch_related('records').all() + subdomains = [sub for sub in subdomains if sub.pk != domain.pk] domain.top.get_subdomains = lambda: subdomains + [domain] elif not domain.pk: # is a new top domain diff --git a/orchestra/contrib/lists/admin.py b/orchestra/contrib/lists/admin.py index 00c26473..5a643579 100644 --- a/orchestra/contrib/lists/admin.py +++ b/orchestra/contrib/lists/admin.py @@ -6,17 +6,20 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin from orchestra.admin.utils import admin_link from orchestra.contrib.accounts.admin import SelectAccountAdminMixin +from orchestra.contrib.accounts.filters import IsActiveListFilter from .forms import ListCreationForm, ListChangeForm from .models import List class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModelAdmin): - list_display = ('name', 'address_name', 'address_domain_link', 'account_link') + list_display = ( + 'name', 'address_name', 'address_domain_link', 'account_link', 'display_active' + ) add_fieldsets = ( (None, { 'classes': ('wide',), - 'fields': ('account_link', 'name',) + 'fields': ('account_link', 'name', 'is_active') }), (_("Address"), { 'classes': ('wide',), @@ -30,7 +33,7 @@ class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModel fieldsets = ( (None, { 'classes': ('wide',), - 'fields': ('account_link', 'name',) + 'fields': ('account_link', 'name', 'is_active') }), (_("Address"), { 'classes': ('wide',), @@ -42,6 +45,7 @@ class ListAdmin(ChangePasswordAdminMixin, SelectAccountAdminMixin, ExtendedModel }), ) search_fields = ('name', 'address_name', 'address_domain__name', 'account__username') + list_filter = (IsActiveListFilter,) readonly_fields = ('account_link',) change_readonly_fields = ('name',) form = ListChangeForm diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py index 033b0682..cf8dbde0 100644 --- a/orchestra/contrib/lists/backends.py +++ b/orchestra/contrib/lists/backends.py @@ -119,7 +119,7 @@ class MailmanBackend(ServiceController): postmap %(virtual_alias)s fi if [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]]; then - /etc/init.d/postfix reload + service postfix reload fi""") % context ) diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 7e933658..4278e056 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -118,14 +118,17 @@ class DovecotPostfixPasswdVirtualUserBackend(ServiceController): def delete(self, mailbox): context = self.get_context(mailbox) - self.append("{ sleep 2 && killall -u %(uid)s -s KILL; } &" % context) - self.append("killall -u %(uid)s || true" % context) - self.append("sed -i '/^%(user)s:.*/d' %(passwd_path)s" % context) - self.append("sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s" % context) - self.append("UPDATED_VIRTUAL_MAILBOX_MAPS=1") - # TODO delete - context['deleted'] = context['home'].rstrip('/') + '.deleted' - self.append("mv %(home)s %(deleted)s" % context) + self.append(textwrap.dedent("""\ + { sleep 2 && killall -u %(uid)s -s KILL; } & + killall -u %(uid)s || true + sed -i '/^%(user)s:.*/d' %(passwd_path)s + sed -i '/^%(user)s@%(mailbox_domain)s\s.*/d' %(virtual_mailbox_maps)s + UPDATED_VIRTUAL_MAILBOX_MAPS=1""") % context + ) + if context['deleted_home']: + self.append("mv %(home)s %(deleted_home)s || exit_code=1" % context) + else: + self.append("rm -fr %(home)s" % context) def get_extra_fields(self, mailbox, context): context['quota'] = self.get_quota(mailbox) @@ -159,13 +162,16 @@ class DovecotPostfixPasswdVirtualUserBackend(ServiceController): 'group': self.DEFAULT_GROUP, 'quota': self.get_quota(mailbox), 'passwd_path': settings.MAILBOXES_PASSWD_PATH, - 'home': mailbox.get_home().rstrip('/'), + 'home': mailbox.get_home(), 'banner': self.get_banner(), 'virtual_mailbox_maps': settings.MAILBOXES_VIRTUAL_MAILBOX_MAPS_PATH, 'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN, } context['extra_fields'] = self.get_extra_fields(mailbox, context) - context['passwd'] = '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context) + context.update({ + 'passwd': '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context), + 'deleted_home': settings.MAILBOXES_MOVE_ON_DELETE_PATH % context, + }) return replace(context, "'", '"') @@ -177,11 +183,13 @@ class PostfixAddressBackend(ServiceController): ) def include_virtual_alias_domain(self, context): - self.append(textwrap.dedent(""" - [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || { - echo '%(domain)s' >> %(virtual_alias_domains)s - UPDATED_VIRTUAL_ALIAS_DOMAINS=1 - }""") % context) + if context['domain'] != context['local_domain']: + self.append(textwrap.dedent(""" + [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || { + echo '%(domain)s' >> %(virtual_alias_domains)s + UPDATED_VIRTUAL_ALIAS_DOMAINS=1 + }""") % context + ) def exclude_virtual_alias_domain(self, context): domain = context['domain'] @@ -193,7 +201,7 @@ class PostfixAddressBackend(ServiceController): # destination = [] # for mailbox in address.get_mailboxes(): # context['mailbox'] = mailbox -# destination.append("%(mailbox)s@%(mailbox_domain)s" % context) +# destination.append("%(mailbox)s@%(local_domain)s" % context) # for forward in address.forward: # if '@' in forward: # destination.append(forward) @@ -237,7 +245,7 @@ class PostfixAddressBackend(ServiceController): context = self.get_context_files() self.append(textwrap.dedent(""" [[ $UPDATED_VIRTUAL_ALIAS_MAPS == 1 ]] && { postmap %(virtual_alias_maps)s; } - [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { /etc/init.d/postfix reload; } + [[ $UPDATED_VIRTUAL_ALIAS_DOMAINS == 1 ]] && { service postfix reload; } """) % context ) self.append('exit 0') @@ -253,7 +261,7 @@ class PostfixAddressBackend(ServiceController): context.update({ 'domain': address.domain, 'email': address.email, - 'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN, + 'local_domain': settings.MAILBOXES_LOCAL_DOMAIN, }) return replace(context, "'", '"') @@ -344,11 +352,13 @@ class PostfixMailscannerTraffic(ServiceMonitor): def inside_period(month, day, time, ini_date): global months global end_datetime - # Mar 19 17:13:22 + # Mar 9 17:13:22 month = months[month] year = end_datetime.year if month == '12' and end_datetime.month == 1: year = year+1 + if len(day) == 1: + day = '0' + day date = str(year) + month + day date += time.replace(':', '') return ini_date < int(date) < end_date diff --git a/orchestra/contrib/mailboxes/settings.py b/orchestra/contrib/mailboxes/settings.py index bd40ad11..8a65e6e6 100644 --- a/orchestra/contrib/mailboxes/settings.py +++ b/orchestra/contrib/mailboxes/settings.py @@ -47,7 +47,7 @@ MAILBOXES_VIRTUAL_ALIAS_DOMAINS_PATH = getattr(settings, 'MAILBOXES_VIRTUAL_ALIA ) -MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN = getattr(settings, 'MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN', +MAILBOXES_LOCAL_DOMAIN = getattr(settings, 'MAILBOXES_LOCAL_DOMAIN', ORCHESTRA_BASE_DOMAIN ) @@ -94,3 +94,8 @@ MAILBOXES_LOCAL_ADDRESS_DOMAIN = getattr(settings, 'MAILBOXES_LOCAL_ADDRESS_DOMA MAILBOXES_MAIL_LOG_PATH = getattr(settings, 'MAILBOXES_MAIL_LOG_PATH', '/var/log/mail.log' ) + + +MAILBOXES_MOVE_ON_DELETE_PATH = getattr(settings, 'MAILBOXES_MOVE_ON_DELETE_PATH', + '' +) diff --git a/orchestra/contrib/miscellaneous/admin.py b/orchestra/contrib/miscellaneous/admin.py index 465b0c57..93c1af92 100644 --- a/orchestra/contrib/miscellaneous/admin.py +++ b/orchestra/contrib/miscellaneous/admin.py @@ -54,7 +54,7 @@ class MiscServiceAdmin(ExtendedModelAdmin): class MiscellaneousAdmin(AccountAdminMixin, SelectPluginAdminMixin, admin.ModelAdmin): list_display = ( - '__str__', 'service_link', 'amount', 'dispaly_active', 'account_link' + '__str__', 'service_link', 'amount', 'dispaly_active', 'account_link', 'is_active' ) list_filter = ('service__name', 'is_active') list_select_related = ('service', 'account') diff --git a/orchestra/contrib/miscellaneous/models.py b/orchestra/contrib/miscellaneous/models.py index 3db29560..663b4232 100644 --- a/orchestra/contrib/miscellaneous/models.py +++ b/orchestra/contrib/miscellaneous/models.py @@ -56,10 +56,7 @@ class Miscellaneous(models.Model): @cached_property def active(self): - try: - return self.is_active and self.account.is_active - except type(self).account.field.rel.to.DoesNotExist: - return self.is_active + return self.is_active and self.service.is_active and self.account.is_active def get_description(self): return ' '.join((str(self.amount), self.service.description or self.service.verbose_name)) diff --git a/orchestra/contrib/orchestration/__init__.py b/orchestra/contrib/orchestration/__init__.py index 297c2b1d..c4d3945d 100644 --- a/orchestra/contrib/orchestration/__init__.py +++ b/orchestra/contrib/orchestration/__init__.py @@ -50,7 +50,7 @@ class Operation(): if hasattr(self.backend, 'get_context'): self.backend().get_context(self.instance) - def create(self, log): + def store(self, log): from .models import BackendOperation return BackendOperation.objects.create( log=log, diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py index 9e4ffb3c..c7afb3f0 100644 --- a/orchestra/contrib/orchestration/management/commands/orchestrate.py +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -18,11 +18,14 @@ class Command(BaseCommand): help='Tells Django to NOT prompt the user for input of any kind.') parser.add_argument('--action', action='store', dest='action', default='save', help='Executes action. Defaults to "save".') + parser.add_argument('--dry-run', action='store_true', dest='dry', default=False, + help='Only prints scrtipt.') def handle(self, *args, **options): model = get_model(*options['model'].split('.')) action = options.get('action') interactive = options.get('interactive') + dry = options.get('dry') kwargs = {} for comp in options.get('query', []): comps = iter(comp.split('=')) @@ -42,7 +45,9 @@ class Command(BaseCommand): servers.append(server.name) sys.stdout.write('# Execute on %s\n' % server.name) for method, commands in backend.scripts: - sys.stdout.write('\n'.join(commands) + '\n') + script = '\n'.join(commands) + '\n' + script = script.encode('ascii', errors='replace') + sys.stdout.write(script.decode('ascii')) if interactive: context = { 'servers': ', '.join(servers), @@ -56,4 +61,10 @@ class Command(BaseCommand): if confirm == 'no': return break -# manager.execute(scripts, block=block) + if not dry: + logs = manager.execute(scripts, block=block) + for log in logs: + print(log.stdout) + sys.stderr.write(log.stderr) + for log in logs: + print(log.backend, log.state) diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index e2cfb735..18c737f6 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -125,7 +125,7 @@ def execute(scripts, block=False, async=False): logger.info("Executed %s" % str(operation)) if operation.instance.pk: # Not all backends are called with objects saved on the database - operation.create(execution.log) + operation.store(execution.log) stdout = execution.log.stdout.strip() stdout and logger.debug('STDOUT %s', stdout) stderr = execution.log.stderr.strip() diff --git a/orchestra/contrib/plans/admin.py b/orchestra/contrib/plans/admin.py index d40bcc72..e30dc1c9 100644 --- a/orchestra/contrib/plans/admin.py +++ b/orchestra/contrib/plans/admin.py @@ -14,8 +14,8 @@ class RateInline(admin.TabularInline): class PlanAdmin(ExtendedModelAdmin): - list_display = ('name', 'is_default', 'is_combinable', 'allow_multiple') - list_filter = ('is_default', 'is_combinable', 'allow_multiple') + list_display = ('name', 'is_default', 'is_combinable', 'allow_multiple', 'is_active') + list_filter = ('is_default', 'is_combinable', 'allow_multiple', 'is_active') fields = ('verbose_name', 'name', 'is_default', 'is_combinable', 'allow_multiple') prepopulated_fields = { 'name': ('verbose_name',) diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py index 5479e2ee..6685490d 100644 --- a/orchestra/contrib/resources/admin.py +++ b/orchestra/contrib/resources/admin.py @@ -126,7 +126,7 @@ class ResourceDataAdmin(ExtendedModelAdmin): display_unit.admin_order_field = 'resource__unit' def display_used(self, data): - if not data.used: + if data.used is None: return '' url = reverse('admin:resources_resourcedata_used_monitordata', args=(data.pk,)) return '%s' % (url, data.used) diff --git a/orchestra/contrib/resources/aggregations.py b/orchestra/contrib/resources/aggregations.py index 7b4c993c..be36e01d 100644 --- a/orchestra/contrib/resources/aggregations.py +++ b/orchestra/contrib/resources/aggregations.py @@ -19,12 +19,13 @@ class Aggregation(plugins.Plugin, metaclass=plugins.PluginMount): class Last(Aggregation): + """ Sum of the last value of all monitors """ name = 'last' verbose_name = _("Last value") def filter(self, dataset): try: - return dataset.order_by('object_id', '-id').distinct('object_id') + return dataset.order_by('object_id', '-id').distinct('monitor') except dataset.model.DoesNotExist: return dataset.none() @@ -38,6 +39,7 @@ class Last(Aggregation): class MonthlySum(Last): + """ Monthly sum the values of all monitors """ name = 'monthly-sum' verbose_name = _("Monthly Sum") @@ -50,9 +52,14 @@ class MonthlySum(Last): class MonthlyAvg(MonthlySum): + """ sum of the monthly averages of each monitor """ name = 'monthly-avg' verbose_name = _("Monthly AVG") + def filter(self, dataset): + qs = super(MonthlyAvg, self).filter(dataset) + return qs.order_by('created_at') + def get_epoch(self): today = timezone.now() return datetime( @@ -64,21 +71,27 @@ class MonthlyAvg(MonthlySum): def compute_usage(self, dataset): result = 0 - try: - last = dataset.latest() - except dataset.model.DoesNotExist: + has_result = False + for monitor, dataset in dataset.group_by('monitor').items(): + try: + last = dataset[-1] + except IndexError: + continue + epoch = self.get_epoch() + total = (last.created_at-epoch).total_seconds() + ini = epoch + for data in dataset: + has_result = True + slot = (data.created_at-ini).total_seconds() + result += data.value * decimal.Decimal(str(slot/total)) + ini = data.created_at + if has_result: return result - epoch = self.get_epoch() - total = (last.created_at-epoch).total_seconds() - ini = epoch - for data in dataset: - slot = (data.created_at-ini).total_seconds() - result += data.value * decimal.Decimal(str(slot/total)) - ini = data.created_at - return result + return None class Last10DaysAvg(MonthlyAvg): + """ sum of the last 10 days averages of each monitor """ name = 'last-10-days-avg' verbose_name = _("Last 10 days AVG") days = 10 @@ -88,4 +101,5 @@ class Last10DaysAvg(MonthlyAvg): return today - datetime.timedelta(days=self.days) def filter(self, dataset): - return dataset.filter(created_at__gt=self.get_epoch()) + epoch = self.get_epoch() + return dataset.filter(created_at__gt=epoch).order_by('created_at') diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index cf09abee..a11d742c 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -262,6 +262,10 @@ class ResourceData(models.Model): return datasets +class MonitorDataQuerySet(models.QuerySet): + group_by = queryset.group_by + + class MonitorData(models.Model): """ Stores monitored data """ monitor = models.CharField(_("monitor"), max_length=256, @@ -272,6 +276,7 @@ class MonitorData(models.Model): value = models.DecimalField(_("value"), max_digits=16, decimal_places=2) content_object = GenericForeignKey() + objects = MonitorDataQuerySet.as_manager() class Meta: get_latest_by = 'id' diff --git a/orchestra/contrib/resources/tasks.py b/orchestra/contrib/resources/tasks.py index 39b95e09..bb1cde0c 100644 --- a/orchestra/contrib/resources/tasks.py +++ b/orchestra/contrib/resources/tasks.py @@ -27,7 +27,7 @@ def monitor(resource_id, ids=None, async=True): # Execute monitor monitorings = [] for obj in model.objects.filter(**kwargs): - op = Operation.create(backend, obj, Operation.MONITOR) + op = Operation(backend, obj, Operation.MONITOR) operations.append(op) monitorings.append(op) # TODO async=True only when running with celery @@ -44,10 +44,10 @@ def monitor(resource_id, ids=None, async=True): a = data.used b = data.allocated if data.used > (data.allocated or 0): - op = Operation.create(backend, obj, Operation.EXCEEDED) + op = Operation(backend, obj, Operation.EXCEEDED) triggers.append(op) elif data.used < (data.allocated or 0): - op = Operation.create(backend, obj, Operation.RECOVERY) + op = Operation(backend, obj, Operation.RECOVERY) triggers.append(op) Operation.execute(triggers) return operations diff --git a/orchestra/contrib/saas/backends/wordpressmu.py b/orchestra/contrib/saas/backends/wordpressmu.py index 604a1353..6fa8f94e 100644 --- a/orchestra/contrib/saas/backends/wordpressmu.py +++ b/orchestra/contrib/saas/backends/wordpressmu.py @@ -13,10 +13,6 @@ class WordpressMuBackend(ServiceController): model = 'webapps.WebApp' default_route_match = "webapp.type == 'wordpress-mu'" - @property - def script(self): - return self.cmds - def login(self, session): base_url = self.get_base_url() login_url = base_url + '/wp-login.php' @@ -113,11 +109,7 @@ class WordpressMuBackend(ServiceController): self.validate_response(response) def save(self, webapp): - if webapp.type != 'wordpress-mu': - return self.append(self.create_blog, webapp) def delete(self, webapp): - if webapp.type != 'wordpress-mu': - return self.append(self.delete_blog, webapp) diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index 4c00b594..ffd50393 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -39,9 +39,9 @@ class UNIXUserBackend(ServiceController): ) for member in settings.SYSTEMUSERS_DEFAULT_GROUP_MEMBERS: context['member'] = member - self.append('usermod -a -G %(user)s %(member)s' % context) + self.append('usermod -a -G %(user)s %(member)s || exit_code=$?' % context) if not user.is_main: - self.append('usermod -a -G %(user)s %(mainuser)s' % context) + self.append('usermod -a -G %(user)s %(mainuser)s || exit_code=$?' % context) def delete(self, user): context = self.get_context(user) @@ -52,9 +52,12 @@ class UNIXUserBackend(ServiceController): killall -u %(user)s || true userdel %(user)s || exit_code=1 groupdel %(group)s || exit_code=1 - mv %(base_home)s %(base_home)s.deleted || exit_code=1 """) % context ) + if context['deleted_home']: + self.append("mv %(base_home)s %(deleted_home)s || exit_code=1" % context) + else: + self.append("rm -fr %(base_home)s" % context) def grant_permission(self, user): context = self.get_context(user) @@ -76,6 +79,7 @@ class UNIXUserBackend(ServiceController): 'home': user.get_home(), 'base_home': user.get_base_home(), } + context['deleted_home'] = settings.SYSTEMUSERS_MOVE_ON_DELETE_PATH % context return replace(context, "'", '"') diff --git a/orchestra/contrib/systemusers/forms.py b/orchestra/contrib/systemusers/forms.py index 84131d64..0eeee379 100644 --- a/orchestra/contrib/systemusers/forms.py +++ b/orchestra/contrib/systemusers/forms.py @@ -60,6 +60,7 @@ class SystemUserFormMixin(object): } def clean(self): + super(SystemUserFormMixin, self).clean() home = self.cleaned_data.get('home') if home and self.MOCK_USERNAME in home: username = self.cleaned_data.get('username', '') diff --git a/orchestra/contrib/systemusers/settings.py b/orchestra/contrib/systemusers/settings.py index ee42bd08..bb658276 100644 --- a/orchestra/contrib/systemusers/settings.py +++ b/orchestra/contrib/systemusers/settings.py @@ -39,3 +39,8 @@ SYSTEMUSERS_MAIL_LOG_PATH = getattr(settings, 'SYSTEMUSERS_MAIL_LOG_PATH', SYSTEMUSERS_DEFAULT_GROUP_MEMBERS = getattr(settings, 'SYSTEMUSERS_DEFAULT_GROUP_MEMBERS', ('www-data',) ) + + +SYSTEMUSERS_MOVE_ON_DELETE_PATH = getattr(settings, 'SYSTEMUSERS_MOVE_ON_DELETE_PATH', + '' +) diff --git a/orchestra/contrib/webapps/admin.py b/orchestra/contrib/webapps/admin.py index db40af67..19f53b16 100644 --- a/orchestra/contrib/webapps/admin.py +++ b/orchestra/contrib/webapps/admin.py @@ -5,14 +5,15 @@ from django.utils.encoding import force_text from django.utils.translation import ugettext, ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin -from orchestra.admin.utils import change_url +from orchestra.admin.utils import change_url, get_modeladmin from orchestra.contrib.accounts.admin import AccountAdminMixin from orchestra.forms.widgets import DynamicHelpTextSelect from orchestra.plugins.admin import SelectPluginAdminMixin +from .filters import HasWebsiteListFilter +from .models import WebApp, WebAppOption from .options import AppOption from .types import AppType -from .models import WebApp, WebAppOption class WebAppOptionInline(admin.TabularInline): @@ -36,7 +37,9 @@ class WebAppOptionInline(admin.TabularInline): plugin = self.parent_object.type_class else: request = kwargs['request'] - plugin = AppType.get(request.GET['type']) + webapp_modeladmin = get_modeladmin(self.parent_model) + plugin_value = webapp_modeladmin.get_plugin_value(request) + plugin = AppType.get(plugin_value) kwargs['choices'] = plugin.get_options_choices() # Help text based on select widget target = 'this.id.replace("name", "value")' @@ -46,7 +49,7 @@ class WebAppOptionInline(admin.TabularInline): class WebAppAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): list_display = ('name', 'type', 'display_detail', 'display_websites', 'account_link') - list_filter = ('type',) + list_filter = ('type', HasWebsiteListFilter) inlines = [WebAppOptionInline] readonly_fields = ('account_link', ) change_readonly_fields = ('name', 'type', 'display_websites') diff --git a/orchestra/contrib/webapps/backends/__init__.py b/orchestra/contrib/webapps/backends/__init__.py index 15b601ea..d6e88dd3 100644 --- a/orchestra/contrib/webapps/backends/__init__.py +++ b/orchestra/contrib/webapps/backends/__init__.py @@ -29,7 +29,10 @@ class WebAppServiceMixin(object): ) def delete_webapp_dir(self, context): - self.append("rm -fr %(app_path)s" % context) + if context['deleted_app_path']: + self.append("mv %(app_path)s %(deleted_app_path)s || exit_code=1" % context) + else: + self.append("rm -fr %(app_path)s" % context) def get_context(self, webapp): context = { @@ -37,11 +40,12 @@ class WebAppServiceMixin(object): 'group': webapp.get_groupname(), 'app_name': webapp.name, 'type': webapp.type, - 'app_path': webapp.get_path().rstrip('/'), + 'app_path': webapp.get_path(), 'banner': self.get_banner(), 'under_construction_path': settings.settings.WEBAPPS_UNDER_CONSTRUCTION_PATH, 'is_mounted': webapp.content_set.exists(), } + context['deleted_app_path'] = settings.WEBAPPS_MOVE_ON_DELETE_PATH % context return replace(context, "'", '"') diff --git a/orchestra/contrib/webapps/backends/php.py b/orchestra/contrib/webapps/backends/php.py index d57b62ac..bf9eb3ab 100644 --- a/orchestra/contrib/webapps/backends/php.py +++ b/orchestra/contrib/webapps/backends/php.py @@ -17,6 +17,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController): def save(self, webapp): context = self.get_context(webapp) + self.create_webapp_dir(context) + self.set_under_construction(context) if webapp.type_instance.is_fpm: self.save_fpm(webapp, context) self.delete_fcgid(webapp, context) @@ -25,8 +27,6 @@ class PHPBackend(WebAppServiceMixin, ServiceController): self.delete_fpm(webapp, context) def save_fpm(self, webapp, context): - self.create_webapp_dir(context) - self.set_under_construction(context) self.append(textwrap.dedent("""\ fpm_config='%(fpm_config)s' { @@ -39,8 +39,6 @@ class PHPBackend(WebAppServiceMixin, ServiceController): ) def save_fcgid(self, webapp, context): - self.create_webapp_dir(context) - self.set_under_construction(context) self.append("mkdir -p %(wrapper_dir)s" % context) self.append(textwrap.dedent("""\ wrapper='%(wrapper)s' @@ -104,7 +102,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController): merge = settings.WEBAPPS_MERGE_PHP_WEBAPPS context.update({ 'init_vars': webapp.type_instance.get_php_init_vars(merge=self.MERGE), - 'max_children': webapp.get_options().get('processes', False), + 'max_children': webapp.get_options().get('processes', + settings.WEBAPPS_FPM_DEFAULT_MAX_CHILDREN), 'request_terminate_timeout': webapp.get_options().get('timeout', False), }) context['fpm_listen'] = webapp.type_instance.FPM_LISTEN % context @@ -119,7 +118,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): listen.group = {{ group }} pm = ondemand pm.max_requests = {{ max_requests }} - {% if max_children %}pm.max_children = {{ max_children }}{% endif %} + pm.max_children = {{ max_children }} {% if request_terminate_timeout %}request_terminate_timeout = {{ request_terminate_timeout }}{% endif %} {% for name, value in init_vars.iteritems %} php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %} @@ -133,7 +132,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): init_vars = opt.get_php_init_vars(merge=self.MERGE) if init_vars: init_vars = [ "-d %s='%s'" % (k, v.replace("'", '"')) for k,v in init_vars.items() ] - init_vars = ', '.join(init_vars) + init_vars = ' \\\n '.join(init_vars) context.update({ 'php_binary': os.path.normpath(settings.WEBAPPS_PHP_CGI_BINARY_PATH % context), 'php_rc': os.path.normpath(settings.WEBAPPS_PHP_CGI_RC_DIR % context), diff --git a/orchestra/contrib/webapps/backends/symboliclink.py b/orchestra/contrib/webapps/backends/symboliclink.py index 6ac994f7..ec9c9459 100644 --- a/orchestra/contrib/webapps/backends/symboliclink.py +++ b/orchestra/contrib/webapps/backends/symboliclink.py @@ -1,23 +1,28 @@ +import textwrap + from django.utils.translation import ugettext_lazy as _ from orchestra.contrib.orchestration import ServiceController, replace -from . import WebAppServiceMixin +from .php import PHPBackend -class SymbolicLinkBackend(WebAppServiceMixin, ServiceController): +class SymbolicLinkBackend(PHPBackend, ServiceController): verbose_name = _("Symbolic link webapp") model = 'webapps.WebApp' default_route_match = "webapp.type == 'symbolic-link'" - def save(self, webapp): - context = self.get_context(webapp) - self.append("ln -s '%(link_path)s' %(app_path)s" % context) - self.append("chown -h %(user)s:%(group)s %(app_path)s" % context) + def create_webapp_dir(self, context): + self.append(textwrap.dedent("""\ + if [[ ! -e %(app_path)s ]]; then + ln -s '%(link_path)s' %(app_path)s + fi + chown -h %(user)s:%(group)s %(app_path)s + """) % context + ) - def delete(self, webapp): - context = self.get_context(webapp) - self.delete_webapp_dir(context) + def set_under_construction(self, context): + pass def get_context(self, webapp): context = super(SymbolicLinkBackend, self).get_context(webapp) diff --git a/orchestra/contrib/webapps/filters.py b/orchestra/contrib/webapps/filters.py new file mode 100644 index 00000000..d0b328d9 --- /dev/null +++ b/orchestra/contrib/webapps/filters.py @@ -0,0 +1,22 @@ +from django.contrib.admin import SimpleListFilter +from django.utils.translation import ugettext_lazy as _ + + +class HasWebsiteListFilter(SimpleListFilter): + title = _("Has website") + parameter_name = 'has_website' + + def lookups(self, request, model_admin): + return ( + ('True', _("True")), + ('False', _("False")), + ) + + def queryset(self, request, queryset): + if self.value() == 'True': + return queryset.filter(content__isnull=False) + elif self.value() == 'False': + return queryset.filter(content__isnull=True) + return queryset + + diff --git a/orchestra/contrib/webapps/options.py b/orchestra/contrib/webapps/options.py index 0b9cae8d..c3732dab 100644 --- a/orchestra/contrib/webapps/options.py +++ b/orchestra/contrib/webapps/options.py @@ -180,14 +180,6 @@ class PHPMaginQuotesSybase(PHPAppOption): regex = r'^(On|Off|on|off)$' -class PHPMaxExecutonTime(PHPAppOption): - name = 'max_execution_time' - verbose_name = _("Max execution time") - help_text = _("Maximum time in seconds a script is allowed to run before it is terminated by " - "the parser (Integer between 0 and 999).") - regex = r'^[0-9]{1,3}$' - - class PHPMaxInputTime(PHPAppOption): name = 'max_input_time' verbose_name = _("Max input time") diff --git a/orchestra/contrib/webapps/settings.py b/orchestra/contrib/webapps/settings.py index a68c342a..58ded0e4 100644 --- a/orchestra/contrib/webapps/settings.py +++ b/orchestra/contrib/webapps/settings.py @@ -13,6 +13,11 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN', '/opt/php/5.4/socks/%(user)s-%(app_name)s.sock' ) +WEBAPPS_FPM_DEFAULT_MAX_CHILDREN = getattr(settings, 'WEBAPPS_FPM_DEFAULT_MAX_CHILDREN', + 3 +) + + WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH', '/etc/php5/fpm/pool.d/%(user)s-%(app_name)s.conf') @@ -145,7 +150,6 @@ WEBAPPS_ENABLED_OPTIONS = getattr(settings, 'WEBAPPS_ENABLED_OPTIONS', ( 'orchestra.contrib.webapps.options.PHPMagicQuotesGPC', 'orchestra.contrib.webapps.options.PHPMagicQuotesRuntime', 'orchestra.contrib.webapps.options.PHPMaginQuotesSybase', - 'orchestra.contrib.webapps.options.PHPMaxExecutonTime', 'orchestra.contrib.webapps.options.PHPMaxInputTime', 'orchestra.contrib.webapps.options.PHPMaxInputVars', 'orchestra.contrib.webapps.options.PHPMemoryLimit', @@ -171,3 +175,8 @@ WEBAPPS_ENABLED_OPTIONS = getattr(settings, 'WEBAPPS_ENABLED_OPTIONS', ( WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST = getattr(settings, 'WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST', 'mysql.{}'.format(ORCHESTRA_BASE_DOMAIN) ) + + +WEBAPPS_MOVE_ON_DELETE_PATH = getattr(settings, 'WEBAPPS_MOVE_ON_DELETE_PATH', + '' +) diff --git a/orchestra/contrib/webapps/types/php.py b/orchestra/contrib/webapps/types/php.py index 99bbab06..e210fcaf 100644 --- a/orchestra/contrib/webapps/types/php.py +++ b/orchestra/contrib/webapps/types/php.py @@ -77,15 +77,16 @@ class PHPApp(AppType): php_version = self.get_php_version() webapps = self.instance.account.webapps.filter(type=self.instance.type) for webapp in webapps: - if webapp.type_instance.get_php_version == php_version: + if webapp.type_instance.get_php_version() == php_version: options += list(webapp.options.all()) php_options = [option.name for option in self.get_php_options()] enabled_functions = set() for opt in options: if opt.name in php_options: - init_vars[opt.name] = opt.value - elif opt.name == 'enabled_functions': - enabled_functions.union(set(opt.value.split(','))) + if opt.name == 'enabled_functions': + enabled_functions = enabled_functions.union(set(opt.value.split(','))) + else: + init_vars[opt.name] = opt.value if enabled_functions: disabled_functions = [] for function in self.PHP_DISABLED_FUNCTIONS: @@ -94,7 +95,9 @@ class PHPApp(AppType): init_vars['dissabled_functions'] = ','.join(disabled_functions) timeout = self.instance.options.filter(name='timeout').first() if timeout: - init_vars['max_execution_time'] = timeout.value + # Give a little slack here + timeout = str(int(timeout.value)-2) + init_vars['max_execution_time'] = timeout if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars: context = self.get_directive_context() error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context) diff --git a/orchestra/contrib/websites/backends/apache.py b/orchestra/contrib/websites/backends/apache.py index b42c3ec1..8174ec71 100644 --- a/orchestra/contrib/websites/backends/apache.py +++ b/orchestra/contrib/websites/backends/apache.py @@ -40,6 +40,7 @@ class Apache2Backend(ServiceController): context['extra_conf'] = '\n'.join([conf for location, conf in extra_conf]) return Template(textwrap.dedent("""\ + IncludeOptional /etc/apache2/site[s]-override/{{ site_unique_name }}.con[f] ServerName {{ site.domains.all|first }}\ {% if site.domains.all|slice:"1:" %} ServerAlias {{ site.domains.all|slice:"1:"|join:' ' }}{% endif %}\ @@ -50,7 +51,6 @@ class Apache2Backend(ServiceController): SuexecUserGroup {{ user }} {{ group }}\ {% for line in extra_conf.splitlines %} {{ line | safe }}{% endfor %} - IncludeOptional /etc/apache2/extra-vhos[t]/{{ site_unique_name }}.con[f] """) ).render(Context(context)) @@ -181,8 +181,8 @@ class Apache2Backend(ServiceController): def get_security(self, directives): security = [] - for rules in directives.get('sec-rule-remove', []): - for rule in rules.value.split(): + for values in directives.get('sec-rule-remove', []): + for rule in values.split(): sec_rule = "SecRuleRemoveById %i" % int(rule) security.append(('', sec_rule)) for location in directives.get('sec-engine', []): @@ -267,12 +267,12 @@ class Apache2Backend(ServiceController): 'site': site, 'site_name': site.name, 'ip': settings.WEBSITES_DEFAULT_IP, - 'site_unique_name': site.unique_name, + 'site_unique_name': '0-'+site.unique_name, 'user': self.get_username(site), 'group': self.get_groupname(site), # TODO remove '0-' 'sites_enabled': "%s.conf" % os.path.join(sites_enabled, '0-'+site.unique_name), - 'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name), + 'sites_available': "%s.conf" % os.path.join(sites_available, '0-'+site.unique_name), 'access_log': site.get_www_access_log_path(), 'error_log': site.get_www_error_log_path(), 'banner': self.get_banner(), diff --git a/orchestra/contrib/websites/forms.py b/orchestra/contrib/websites/forms.py index afbf7110..7c241ea7 100644 --- a/orchestra/contrib/websites/forms.py +++ b/orchestra/contrib/websites/forms.py @@ -9,6 +9,7 @@ from .validators import validate_domain_protocol class WebsiteAdminForm(forms.ModelForm): def clean(self): """ Prevent multiples domains on the same protocol """ + super(WebsiteAdminForm, self).clean() domains = self.cleaned_data.get('domains') if not domains: return self.cleaned_data diff --git a/orchestra/management/commands/makemessages.py b/orchestra/management/commands/makemessages.py index 88f6dcd6..449bc1d7 100644 --- a/orchestra/management/commands/makemessages.py +++ b/orchestra/management/commands/makemessages.py @@ -51,7 +51,7 @@ class Command(makemessages.Command): tmpcontent = '\n'.join(tmpcontent) + '\n' filename = 'database_%s.sql.py' % name self.database_files.append(filename) - with open(filename, 'w') as tmpfile: + with open(filename, 'wb') as tmpfile: tmpfile.write(tmpcontent.encode('utf-8')) def remove_database_files(self): diff --git a/orchestra/plugins/admin.py b/orchestra/plugins/admin.py index e4b80fb1..e58f8d21 100644 --- a/orchestra/plugins/admin.py +++ b/orchestra/plugins/admin.py @@ -1,3 +1,5 @@ +import re + from django.conf.urls import patterns, url from django.contrib.admin.utils import unquote from django.shortcuts import render, redirect @@ -58,10 +60,19 @@ class SelectPluginAdminMixin(object): template = 'admin/plugins/select_plugin.html' return render(request, template, context) + def get_plugin_value(self, request): + plugin_value = request.GET.get(self.plugin_field) or request.POST.get(self.plugin_field) + if not plugin_value and request.method == 'POST': + # HACK baceuse django add_preserved_filters removes extising queryargs + value = re.search(r"type=([^&^']+)[&']", request.META.get('HTTP_REFERER', '')) + if value: + plugin_value = value.groups()[0] + return plugin_value + def add_view(self, request, form_url='', extra_context=None): """ Redirects to select account view if required """ if request.user.is_superuser: - plugin_value = request.GET.get(self.plugin_field) or request.POST.get(self.plugin_field) + plugin_value = self.get_plugin_value(request) if plugin_value or len(self.plugin.get_plugins()) == 1: self.plugin_value = plugin_value if not plugin_value: diff --git a/orchestra/plugins/forms.py b/orchestra/plugins/forms.py index 273b6a49..c9d7780d 100644 --- a/orchestra/plugins/forms.py +++ b/orchestra/plugins/forms.py @@ -29,6 +29,7 @@ class PluginDataForm(forms.ModelForm): self.fields[field].widget = ReadOnlyWidget(value, display) def clean(self): + super(PluginDataForm, self).clean() data = {} # Update data fields for field in self.declared_fields: