From 4593fcc2783e1997f61b4eeed1351187138320a4 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Wed, 5 Aug 2015 22:58:35 +0000 Subject: [PATCH] Fixes on resources, mailer and vps0 --- TODO.md | 48 ++----- orchestra/bin/orchestra-beat | 2 +- orchestra/contrib/lists/forms.py | 30 +---- orchestra/contrib/mailer/admin.py | 26 ++-- orchestra/contrib/mailer/engine.py | 8 +- .../migrations/0004_auto_20150805_1328.py | 24 ++++ orchestra/contrib/mailer/models.py | 10 +- orchestra/contrib/resources/admin.py | 7 +- orchestra/contrib/resources/backends.py | 20 +-- .../migrations/0008_monitordata_state.py | 19 +++ .../migrations/0009_auto_20150804_1450.py | 19 +++ orchestra/contrib/resources/models.py | 6 +- orchestra/contrib/resources/serializers.py | 15 ++- orchestra/contrib/resources/tasks.py | 2 +- orchestra/contrib/saas/backends/phplist.py | 5 +- .../admin/services/service/change_form.html | 2 +- orchestra/contrib/systemusers/backends.py | 87 ++++++------ orchestra/contrib/systemusers/models.py | 2 +- orchestra/contrib/vps/admin.py | 8 +- orchestra/contrib/vps/backends.py | 126 ++++++++++++++---- orchestra/contrib/vps/forms.py | 39 ------ .../vps/migrations/0002_auto_20150804_1524.py | 23 ++++ orchestra/contrib/vps/models.py | 5 +- orchestra/forms/options.py | 9 +- orchestra/utils/sys.py | 6 +- 25 files changed, 331 insertions(+), 217 deletions(-) create mode 100644 orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py create mode 100644 orchestra/contrib/resources/migrations/0008_monitordata_state.py create mode 100644 orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py delete mode 100644 orchestra/contrib/vps/forms.py create mode 100644 orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py diff --git a/TODO.md b/TODO.md index c2727678..bada898b 100644 --- a/TODO.md +++ b/TODO.md @@ -36,8 +36,6 @@ * What fields we really need on contacts? name email phone and what more? -* Redirect junk emails and delete every 30 days? - * DOC: Complitely decouples scripts execution, billing, service definition * init.d celery scripts @@ -58,21 +56,11 @@ * env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python3 manage.py test orchestra.contrib.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture --keepdb * ForeignKey.swappable -* Field.editable -* ManyToManyField.symmetrical = False (user group) * REST PERMISSIONS -* caching based on "def text2int(textnum, numwords={}):" - -* sync() ServiceController method that synchronizes orchestra and servers (delete or import) - -* consider removing mailbox support on forward (user@pangea.org instead) - * Databases.User add reverse M2M databases widget (like mailbox.addresses) -* Grant permissions to systemusers - * Make one dedicated CGI user for each account only for CGI execution (fpm/fcgid). Different from the files owner, and without W permissions, so attackers can not inject backdors and malware. * resource min max allocation with validation @@ -81,24 +69,14 @@ * Directory Protection on webapp and use webapp path as base path (validate) -* validate systemuser.home on server-side - * webapp backend option compatibility check? raise exception, missconfigured error -* admin systemuser home/directory, add default home and empty directory with has_shell on admin - * Resource used_list_display=True, allocated_list_displat=True, allow resources to show up on list_display * BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when) -* Periodic task for cleaning old monitoring data - * Create an admin service_view with icons (like SaaS app) -* Resource graph for each related object - -* SaaS model splitted into SaaSUser and SaaSSite? inherit from SaaS, proxy model? - * prevent @pangea.org email addresses on contacts, enforce at least one email without @pangea.org * forms autocomplete="off", doesn't work in chrome @@ -125,8 +103,6 @@ Php binaries should have this format: /usr/bin/php5.2-cgi * contact.alternative_phone on a phone.tooltip, email:to -* better validate options and directives (url locations, filesystem paths, etc..) - * make sure that you understand the risks * full support for deactivation of services/accounts @@ -295,7 +271,7 @@ https://code.djangoproject.com/ticket/24576 # Settings.parser.changes: if setting.value == default. remove # reload generic admin view ?redirect=http... # inspecting django db connection for asserting db readines? or performing a query -# wake up django mailer on send_mail +* wake up django mailer on send_mail from orchestra.contrib.tasks import task import time, sys @@ -308,14 +284,14 @@ https://code.djangoproject.com/ticket/24576 time.sleep(1) counter.apply_async(10, '/tmp/kakas') -# Provide some fixtures with mocked data +* Provide some fixtures with mocked data TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix TODO mount the filesystem with "nosuid" option -# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask +* uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask # mailboxes.address settings multiple local domains, not only one? # backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data @@ -323,7 +299,7 @@ TODO mount the filesystem with "nosuid" option # smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209') # rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ? -# virtdomains file is not ideal, prevent fake/error on domains there! and make sure to chekc if this file is required! +# virtdomains file is not ideal, prevent user provided fake/error domains there! and make sure to chekc if this file is required! # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery) orchestra-beat support for uwsgi cron @@ -338,11 +314,9 @@ resorce monitoring more efficient, less mem an better queries for calc current d # bill this https://orchestra.pangea.org/admin/orders/order/8236/ should be already billed, <= vs < # Convert rating method from function to PluginClass -# Tests can not run because django.db.utils.ProgrammingError: relation "accounts_account" does not exist # autoresponses on mailboxes, not addresses or remove them -# ACL don't give exec permissions to files! # force save and continue on routes (and others?) # gevent for python3 apt-get install cython3 @@ -384,8 +358,6 @@ uwsgi --reload /tmp/project-master.pid # or if uwsgi was started with touch-reload=/tmp/somefile touch /tmp/somefile -# datetime metric storage granularity: otherwise innacurate detection of billed metric on order.billed_on - # Serializers.validation migration to DRF3: grep -r 'attrs, source' *|grep -v '~' serailzer self.instance on create. @@ -427,14 +399,14 @@ Case # round decimals on every billing operation -# Add SPF record type - -# OVZ TRAFFIC ACCOUNTING!! - # PHPlist cron, bounces and traffic (maybe specific mail script with sitename) - # use "su $user --shell /bin/bash" on backends for security : MKDIR -p... - # model.field.flatchoices + +* This is beta software, please test thoroughly before putting into production and report back any issues. + +# messages SMTP errors: temporary->deferre else Failed + +# Don't enforce one contact per account? remove account.email in favour of contacts? diff --git a/orchestra/bin/orchestra-beat b/orchestra/bin/orchestra-beat index 53c5175b..09d13fa2 100755 --- a/orchestra/bin/orchestra-beat +++ b/orchestra/bin/orchestra-beat @@ -191,7 +191,7 @@ def fire_pending_messages(settings, db): for num, seconds in enumerate(MAILER_DEFERE_SECONDS): delta = timedelta(seconds=seconds) epoch = now-delta - query_or.append("""(mailer_message.retries = %i AND mailer_message.last_retry <= '%s')""" + query_or.append("""(mailer_message.retries = %i AND mailer_message.last_try <= '%s')""" % (num, epoch.isoformat().replace('T', ' '))) query = """\ SELECT 1 FROM mailer_message diff --git a/orchestra/contrib/lists/forms.py b/orchestra/contrib/lists/forms.py index 0ffff1f7..c83b0f3a 100644 --- a/orchestra/contrib/lists/forms.py +++ b/orchestra/contrib/lists/forms.py @@ -1,8 +1,6 @@ -from django import forms from django.utils.translation import ugettext_lazy as _ -from orchestra.core.validators import validate_password -from orchestra.forms.widgets import SpanWidget +from orchestra.forms import UserCreationForm, NonStoredUserChangeForm class CleanAddressMixin(object): @@ -15,25 +13,9 @@ class CleanAddressMixin(object): return domain -class ListCreationForm(CleanAddressMixin, forms.ModelForm): - password1 = forms.CharField(label=_("Password"), validators=[validate_password], - widget=forms.PasswordInput) - password2 = forms.CharField(label=_("Password confirmation"), - widget=forms.PasswordInput, - help_text=_("Enter the same password as above, for verification.")) - - def clean_password2(self): - password1 = self.cleaned_data.get("password1") - password2 = self.cleaned_data.get("password2") - if password1 and password2 and password1 != password2: - msg = _("The two password fields didn't match.") - raise forms.ValidationError(msg) - return password2 - +class ListCreationForm(CleanAddressMixin, UserCreationForm): + pass -class ListChangeForm(CleanAddressMixin, forms.ModelForm): - password = forms.CharField(label=_("Password"), required=False, - widget=SpanWidget(display='Unknown password'), - help_text=_("List passwords are not stored, so there is no way to see this " - "list's password, but you can change the password using " - "this form.")) + +class ListChangeForm(CleanAddressMixin, NonStoredUserChangeForm): + pass diff --git a/orchestra/contrib/mailer/admin.py b/orchestra/contrib/mailer/admin.py index b559d05e..6ee69e21 100644 --- a/orchestra/contrib/mailer/admin.py +++ b/orchestra/contrib/mailer/admin.py @@ -28,13 +28,13 @@ COLORS = { class MessageAdmin(admin.ModelAdmin): list_display = ( 'display_subject', 'colored_state', 'priority', 'to_address', 'from_address', - 'created_at_delta', 'retries', 'last_retry_delta', 'num_logs', + 'created_at_delta', 'display_retries', 'last_try_delta', ) list_filter = ('state', 'priority', 'retries') list_prefetch_related = ('logs__id') fieldsets = ( (None, { - 'fields': ('state', 'priority', ('retries', 'last_retry_delta', 'created_at_delta'), + 'fields': ('state', 'priority', ('retries', 'last_try_delta', 'created_at_delta'), 'display_full_subject', 'display_from', 'display_to', 'display_content'), }), @@ -44,36 +44,36 @@ class MessageAdmin(admin.ModelAdmin): }), ) readonly_fields = ( - 'retries', 'last_retry_delta', 'created_at_delta', 'display_full_subject', + 'retries', 'last_try_delta', 'created_at_delta', 'display_full_subject', 'display_to', 'display_from', 'display_content', ) date_hierarchy = 'created_at' colored_state = admin_colored('state', colors=COLORS) created_at_delta = admin_date('created_at') - last_retry_delta = admin_date('last_retry') + last_try_delta = admin_date('last_try') def display_subject(self, instance): subject = instance.subject - if len(subject) > 32: - return subject[:32] + '…' + if len(subject) > 64: + return subject[:64] + '…' return subject display_subject.short_description = _("Subject") display_subject.admin_order_field = 'subject' display_subject.allow_tags = True - def num_logs(self, instance): - num = instance.logs__count - if num == 1: + def display_retries(self, instance): + num_logs = instance.logs__count + if num_logs == 1: pk = instance.logs.all()[0].id url = reverse('admin:mailer_smtplog_change', args=(pk,)) else: url = reverse('admin:mailer_smtplog_changelist') url += '?&message=%i' % instance.pk - return '%d' % (url, num) - num_logs.short_description = _("Logs") - num_logs.admin_order_field = 'logs__count' - num_logs.allow_tags = True + return '%d' % (url, instance.retries) + display_retries.short_description = _("Retries") + display_retries.admin_order_field = 'retries' + display_retries.allow_tags = True def display_content(self, instance): part = email.message_from_string(instance.content) diff --git a/orchestra/contrib/mailer/engine.py b/orchestra/contrib/mailer/engine.py index bbcbd3ed..ddf06c2f 100644 --- a/orchestra/contrib/mailer/engine.py +++ b/orchestra/contrib/mailer/engine.py @@ -23,6 +23,12 @@ def send_message(message, num=0, connection=None, bulk=settings.MAILER_BULK_MESS connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend') connection.open() error = None + message.last_try = timezone.now() + update_fields = ['last_try'] + if message.state != message.QUEUED: + message.retries += 1 + update_fields.append('retries') + message.save(update_fields=update_fields) try: connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content)) except (SocketError, @@ -49,7 +55,7 @@ def send_pending(bulk=settings.MAILER_BULK_MESSAGES): qs = Q() for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS): delta = timedelta(seconds=seconds) - qs = qs | Q(retries=retries, last_retry__lte=now-delta) + qs = qs | Q(retries=retries, last_try__lte=now-delta) for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'): connection = send_message(message, num, connection, bulk) num += 1 diff --git a/orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py b/orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py new file mode 100644 index 00000000..d224585b --- /dev/null +++ b/orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('mailer', '0003_auto_20150617_1024'), + ] + + operations = [ + migrations.RenameField( + model_name='message', + old_name='last_retry', + new_name='last_try', + ), + migrations.AlterField( + model_name='message', + name='state', + field=models.CharField(verbose_name='State', max_length=16, choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failed')], default='QUEUED'), + ), + ] diff --git a/orchestra/contrib/mailer/models.py b/orchestra/contrib/mailer/models.py index 4d396c13..519ecfa5 100644 --- a/orchestra/contrib/mailer/models.py +++ b/orchestra/contrib/mailer/models.py @@ -1,5 +1,4 @@ from django.db import models -from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from . import settings @@ -14,7 +13,7 @@ class Message(models.Model): (QUEUED, _("Queued")), (SENT, _("Sent")), (DEFERRED, _("Deferred")), - (FAILED, _("Failes")), + (FAILED, _("Failed")), ) CRITICAL = 0 @@ -36,8 +35,7 @@ class Message(models.Model): content = models.TextField(_("content")) created_at = models.DateTimeField(_("created at"), auto_now_add=True) retries = models.PositiveIntegerField(_("retries"), default=0) - # TODO rename to last_try - last_retry = models.DateTimeField(_("last try"), null=True) + last_try = models.DateTimeField(_("last try"), null=True) def __str__(self): return '%s to %s' % (self.subject, self.to_address) @@ -47,9 +45,7 @@ class Message(models.Model): # Max tries if self.retries >= len(settings.MAILER_DEFERE_SECONDS): self.state = self.FAILED - self.retries += 1 - self.last_retry = timezone.now() - self.save(update_fields=('state', 'retries', 'last_retry')) + self.save(update_fields=('state',)) def sent(self): self.state = self.SENT diff --git a/orchestra/contrib/resources/admin.py b/orchestra/contrib/resources/admin.py index 66062f31..56f8108f 100644 --- a/orchestra/contrib/resources/admin.py +++ b/orchestra/contrib/resources/admin.py @@ -198,7 +198,7 @@ class MonitorDataAdmin(ExtendedModelAdmin): list_display = ('id', 'monitor', content_object_link, 'display_created', 'value') list_filter = ('monitor', ResourceDataListFilter) add_fields = ('monitor', 'content_type', 'object_id', 'created_at', 'value') - fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value') + fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value', 'state') change_readonly_fields = fields list_select_related = ('content_type',) search_fields = ('content_object_repr',) @@ -258,6 +258,7 @@ def resource_inline_factory(resources): if resource not in queryset_resources: kwargs = { 'content_object': self.instance, + 'content_object_repr': str(self.instance), } if resource.default_allocation: kwargs['allocated'] = resource.default_allocation @@ -323,7 +324,9 @@ def resource_inline_factory(resources): used_url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,)) used = '%s %s' % (used_url, rdata.used, rdata.unit) return ' '.join(map(str, (used, update, history))) - return _("Unknonw %s %s") % (update, history) + if rdata.resource.monitors: + return _("Unknonw %s %s") % (update, history) + return _("No monitor") display_used.short_description = _("Used") display_used.allow_tags = True diff --git a/orchestra/contrib/resources/backends.py b/orchestra/contrib/resources/backends.py index 2b6ca374..e12e6135 100644 --- a/orchestra/contrib/resources/backends.py +++ b/orchestra/contrib/resources/backends.py @@ -57,8 +57,14 @@ class ServiceMonitor(ServiceBackend): return data.created_at def process(self, line): - """ line -> object_id, value """ - return line.split() + """ line -> object_id, value, state""" + result = line.split() + if len(result) != 2: + cls_name = self.__class__.__name__ + raise ValueError("%s expected ' ' got '%s'" % (cls_name, line)) + # State is None, unless your monitor needs to keep track of it + result.append(None) + return result def store(self, log): """ stores monitored values from stdout """ @@ -68,16 +74,14 @@ class ServiceMonitor(ServiceBackend): ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower()) for line in log.stdout.splitlines(): line = line.strip() - try: - object_id, value = self.process(line) - except ValueError: - cls_name = self.__class__.__name__ - raise ValueError("%s expected ' ' got '%s'" % (cls_name, line)) + object_id, value, state = self.process(line) if isinstance(value, bytes): value = value.decode('ascii') + if isinstance(state, bytes): + state = state.decode('ascii') content_object = ct.get_object_for_this_type(pk=object_id) MonitorData.objects.create( - monitor=name, object_id=object_id, content_type=ct, value=value, + monitor=name, object_id=object_id, content_type=ct, value=value, state=state, created_at=self.current_date, content_object_repr=str(content_object), ) diff --git a/orchestra/contrib/resources/migrations/0008_monitordata_state.py b/orchestra/contrib/resources/migrations/0008_monitordata_state.py new file mode 100644 index 00000000..65475d4e --- /dev/null +++ b/orchestra/contrib/resources/migrations/0008_monitordata_state.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('resources', '0007_auto_20150723_1251'), + ] + + operations = [ + migrations.AddField( + model_name='monitordata', + name='state', + field=models.DecimalField(verbose_name='state', null=True, max_digits=16, help_text='Optional field used to store current state needed for diff-based monitoring.', decimal_places=2), + ), + ] diff --git a/orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py b/orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py new file mode 100644 index 00000000..f93fb2c3 --- /dev/null +++ b/orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('resources', '0008_monitordata_state'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcedata', + name='allocated', + field=models.PositiveIntegerField(verbose_name='allocated', null=True, blank=True), + ), + ] diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index b49497c7..24da00b5 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -172,8 +172,7 @@ class ResourceData(models.Model): used = models.DecimalField(_("used"), max_digits=16, decimal_places=3, null=True, editable=False) updated_at = models.DateTimeField(_("updated"), null=True, editable=False) - allocated = models.DecimalField(_("allocated"), max_digits=8, decimal_places=2, - null=True, blank=True) + allocated = models.PositiveIntegerField(_("allocated"), null=True, blank=True) content_object_repr = models.CharField(_("content object representation"), max_length=256, editable=False) @@ -268,6 +267,8 @@ class MonitorData(models.Model): object_id = models.PositiveIntegerField(_("object id"), db_index=True) created_at = models.DateTimeField(_("created"), default=timezone.now, db_index=True) value = models.DecimalField(_("value"), max_digits=16, decimal_places=2) + state = models.DecimalField(_("state"), max_digits=16, decimal_places=2, null=True, + help_text=_("Optional field used to store current state needed for diff-based monitoring.")) content_object_repr = models.CharField(_("content object representation"), max_length=256, editable=False) @@ -308,6 +309,7 @@ def create_resource_relation(): ) rdata = ResourceData( content_object=self.obj, + content_object_repr=str(self.obj), resource=resource, allocated=resource.default_allocation ) diff --git a/orchestra/contrib/resources/serializers.py b/orchestra/contrib/resources/serializers.py index 0ffadf2f..140674ed 100644 --- a/orchestra/contrib/resources/serializers.py +++ b/orchestra/contrib/resources/serializers.py @@ -33,13 +33,18 @@ class ResourceSerializer(serializers.ModelSerializer): def insert_resource_serializers(): # clean previous state for related in Resource._related: - viewset = router.get_viewset(related) - fields = list(viewset.serializer_class.Meta.fields) try: - fields.remove('resources') - except ValueError: + viewset = router.get_viewset(related) + except KeyError: + # API viewset not registered pass - viewset.serializer_class.Meta.fields = fields + else: + fields = list(viewset.serializer_class.Meta.fields) + try: + fields.remove('resources') + except ValueError: + pass + viewset.serializer_class.Meta.fields = fields # Create nested serializers on target models for ct, resources in Resource.objects.group_by('content_type').items(): model = ct.model_class() diff --git a/orchestra/contrib/resources/tasks.py b/orchestra/contrib/resources/tasks.py index 38ab8220..b802e876 100644 --- a/orchestra/contrib/resources/tasks.py +++ b/orchestra/contrib/resources/tasks.py @@ -63,7 +63,7 @@ def monitor(resource_id, ids=None): def cleanup_old_monitors(queryset=None): if queryset is None: from .models import MonitorData - queryset = MonitorData.objects.filter() + queryset = MonitorData.objects.all() delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS) threshold = timezone.now() - delta queryset = queryset.filter(created_at__lt=threshold) diff --git a/orchestra/contrib/saas/backends/phplist.py b/orchestra/contrib/saas/backends/phplist.py index 667498a0..03bd5c0b 100644 --- a/orchestra/contrib/saas/backends/phplist.py +++ b/orchestra/contrib/saas/backends/phplist.py @@ -34,7 +34,8 @@ class PhpListSaaSBackend(ServiceController): sys.stderr.write(msg + '\n') raise RuntimeError(msg) - def _save(self, saas, server): + def _install_or_change_password(self, saas, server): + """ configures the database for the new site through HTTP to /admin/ """ admin_link = 'https://%s/admin/' % saas.get_site_domain() sys.stdout.write('admin_link: %s\n' % admin_link) admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL) @@ -75,7 +76,7 @@ class PhpListSaaSBackend(ServiceController): def save(self, saas): if hasattr(saas, 'password'): - self.append(self._save, saas) + self.append(self._install_or_change_password, saas) context = self.get_context(saas) if context['crontab']: context['escaped_crontab'] = context['crontab'].replace('$', '\\$') diff --git a/orchestra/contrib/services/templates/admin/services/service/change_form.html b/orchestra/contrib/services/templates/admin/services/service/change_form.html index 07642046..44f7c1ae 100644 --- a/orchestra/contrib/services/templates/admin/services/service/change_form.html +++ b/orchestra/contrib/services/templates/admin/services/service/change_form.html @@ -37,7 +37,7 @@ is_fee=& order_description=& ignore_period=TEN_DAYS& - metric=max(logsteps(mailbox.resources.disk.allocated%20or%200)%20-2,%200)& + metric=max((mailbox.resources.disk.allocated%20or%200)%20-2,%200)& nominal_price=20.00& tax=21& pricing_period=& diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index c02b169c..673c42eb 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -1,3 +1,4 @@ +import fnmatch import os import textwrap @@ -97,6 +98,51 @@ class UNIXUserBackend(ServiceController): else: self.append("rm -fr %(base_home)s" % context) + def grant_permissions(self, user, context): + context['perms'] = user.set_perm_perms + # Capital X adds execution permissions for directories, not files + context['perms_X'] = context['perms'] + 'X' + self.append(textwrap.dedent("""\ + # Grant execution permissions to every parent directory + for access_path in %(access_paths)s; do + # Preserve existing ACLs + acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && { + perms=$(echo "$acl" | cut -d':' -f3) + perms=$(echo "$perms" | cut -c 1,2)x + setfacl -m u:%(user)s:$perms "$access_path" + } || setfacl -m u:%(user)s:--x "$access_path" + done + # Grant perms to existing files, excluding execution + find '%(perm_to)s' -type f %(exclude_acl)s \\ + -exec setfacl -m u:%(user)s:%(perms)s {} \\; + # Grant perms to extisting directories and set defaults for future content + find '%(perm_to)s' -type d %(exclude_acl)s \\ + -exec setfacl -m u:%(user)s:%(perms_X)s -m d:u:%(user)s:%(perms_X)s {} \\; + # Account group as the owner of new files + chmod g+s '%(perm_to)s'""") % context + ) + if not user.is_main: + self.append(textwrap.dedent("""\ + # Grant access to main user + find '%(perm_to)s' -type d %(exclude_acl)s \\ + -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\ + """) % context + ) + + def revoke_permissions(self, context): + revoke_perms = { + 'rw': '', + 'r': 'w', + 'w': 'r', + } + context['perms'] = revoke_perms[user.set_perm_perms] + self.append(textwrap.dedent("""\ + # Revoke permissions + find '%(perm_to)s' %(exclude_acl)s \\ + -exec setfacl -m u:%(user)s:%(perms)s {} \\;\ + """) % context + ) + def set_permission(self, user): context = self.get_context(user) context.update({ @@ -108,17 +154,10 @@ class UNIXUserBackend(ServiceController): context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude) exclude_acl.append('-not -path "%(exclude_acl)s"' % context) context['exclude_acl'] = ' \\\n -a '.join(exclude_acl) if exclude_acl else '' - if user.set_perm_perms == 'rw': - context['perm_perms'] = 'rwx' if user.set_perm_action == 'grant' else '---' - elif user.set_perm_perms == 'r': - context['perm_perms'] = 'r-x' if user.set_perm_action == 'grant' else '-wx' - elif user.set_perm_perms == 'w': - context['perm_perms'] = '-wx' if user.set_perm_action == 'grant' else 'r-x' # Access paths head = user.set_perm_base_home relative = '' access_paths = ["'%s'" % head] - import fnmatch for tail in user.set_perm_home_extension.split(os.sep)[:-1]: relative = os.path.join(relative, tail) for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS: @@ -129,39 +168,11 @@ class UNIXUserBackend(ServiceController): head = os.path.join(head, tail) access_paths.append("'%s'" % head) context['access_paths'] = ' '.join(access_paths) + if user.set_perm_action == 'grant': - self.append(textwrap.dedent("""\ - # Grant execution permissions to every parent directory - for access_path in %(access_paths)s; do - # Preserve existing ACLs - acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && { - perms=$(echo "$acl" | cut -d':' -f3) - perms=$(echo "$perms" | cut -c 1,2)x - setfacl -m u:%(user)s:$perms "$access_path" - } || setfacl -m u:%(user)s:--x "$access_path" - done - # Grant perms to existing and future files - find '%(perm_to)s' %(exclude_acl)s \\ - -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\; - find '%(perm_to)s' -type d %(exclude_acl)s \\ - -exec setfacl -m d:u:%(user)s:%(perm_perms)s {} \\; - # Account group as the owner of new files - chmod g+s '%(perm_to)s'""") % context - ) - if not user.is_main: - self.append(textwrap.dedent("""\ - # Grant access to main user - find '%(perm_to)s' -type d %(exclude_acl)s \\ - -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\ - """) % context - ) + self.grant_permissions(user, context) elif user.set_perm_action == 'revoke': - self.append(textwrap.dedent("""\ - # Revoke permissions - find '%(perm_to)s' %(exclude_acl)s \\ - -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;\ - """) % context - ) + self.revoke_permissions(user, context) else: raise NotImplementedError() diff --git a/orchestra/contrib/systemusers/models.py b/orchestra/contrib/systemusers/models.py index 62798d2c..06cb775d 100644 --- a/orchestra/contrib/systemusers/models.py +++ b/orchestra/contrib/systemusers/models.py @@ -45,7 +45,7 @@ class SystemUser(models.Model): help_text=_("Optional directory relative to user's home.")) shell = models.CharField(_("shell"), max_length=32, choices=settings.SYSTEMUSERS_SHELLS, default=settings.SYSTEMUSERS_DEFAULT_SHELL) - groups = models.ManyToManyField('self', blank=True, symmetrical=False, + groups = models.ManyToManyField('self', blank=True, symmetrical=False, help_text=_("A new group will be created for the user. " "Which additional groups would you like them to be a member of?")) is_active = models.BooleanField(_("active"), default=True, diff --git a/orchestra/contrib/vps/admin.py b/orchestra/contrib/vps/admin.py index d81c85d0..14624ba0 100644 --- a/orchestra/contrib/vps/admin.py +++ b/orchestra/contrib/vps/admin.py @@ -6,18 +6,18 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin from orchestra.contrib.accounts.actions import list_accounts from orchestra.contrib.accounts.admin import AccountAdminMixin +from orchestra.forms import UserCreationForm, NonStoredUserChangeForm -from .forms import VPSChangeForm, VPSCreationForm from .models import VPS class VPSAdmin(AccountAdminMixin, ExtendedModelAdmin): list_display = ('hostname', 'type', 'template', 'account_link') list_filter = ('type', 'template') - form = VPSChangeForm - add_form = VPSCreationForm + form = NonStoredUserChangeForm + add_form = UserCreationForm readonly_fields = ('account_link',) - change_readonly_fields = ('account', 'name', 'type', 'template') + change_readonly_fields = ('account', 'hostname', 'type', 'template') fieldsets = ( (None, { 'classes': ('wide',), diff --git a/orchestra/contrib/vps/backends.py b/orchestra/contrib/vps/backends.py index 1dd12dbb..f4fed982 100644 --- a/orchestra/contrib/vps/backends.py +++ b/orchestra/contrib/vps/backends.py @@ -1,38 +1,114 @@ -from orchestra.contrib.orchestration import replace +import decimal +import textwrap + +from orchestra.contrib.orchestration import ServiceController from orchestra.contrib.resources import ServiceMonitor +class ProxmoxOVZ(ServiceController): + model = 'vps.VPS' + + RESOURCES = ( + ('memory', 'mem'), + ('swap', 'swap'), + ('disk', 'disk') + ) + GET_PROXMOX_INFO = textwrap.dedent(""" + function get_vz_info () { + hostname=$1 + version=$(pveversion | cut -d '/' -f2 | cut -d'.' -f1) + if [[ $version -lt 2 ]]; then + conf=$(grep "CID\\|:$hostname:" /var/lib/pve-manager/vzlist | grep -B1 ":$hostname:") + CID=$(echo "$conf" | head -n1 | cut -d':' -f2) + CTID=$(echo "$conf" | tail -n1 | cut -d':' -f1) + node=$(pveca -l | grep "^\\s*$CID\\s*:" | awk {'print $3'}) + else + conf=$(grep -r "HOSTNAME=\\"$hostname\\"" /etc/pve/nodes/*/openvz/*.conf) + node=$(echo "${conf}" | cut -d"/" -f5) + CTID=$(echo "${conf}" | cut -d"/" -f7 | cut -d"\\." -f1) + fi + echo $CTID $node + }""") + + def prepare(self): + super(ProxmoxOVZ, self).prepare() + self.append(self.GET_PROXMOX_INFO) + + def get_vzset_args(self, vps): + args = [] + for resource, arg_name in self.RESOURCES: + try: + allocation = getattr(vps.resources, resource).allocated + except AttributeError: + pass + else: + args.append('--%s %i' % (arg_name, allocation)) + return ' '.join(args) + + def save(self, vps): + context = self.get_context(vps) + self.append('info=( $(get_vz_info %(hostname)s) )' % context) + vzset_args = self.get_vzset_args(vps) + if vzset_args: + context['vzset_args'] = vzset_args + self.append(textwrap.dedent("""\ + cat << EOF | ssh root@${info[1]} + pvectl vzset ${info[0]} %(vzset_args)s + EOF""") % context + ) + if hasattr(vps, 'password'): + context['password'] = vps.password.replace('$', '\\$') + self.append(textwrap.dedent("""\ + cat << EOF | ssh root@${info[1]} + echo 'root:%(password)s' | vzctl exec ${info[0]} chpasswd -e + EOF""") % context + ) + def get_context(self, vps): + return { + 'hostname': vps.hostname, + } + + +# TODO rename to proxmox class OpenVZTraffic(ServiceMonitor): - """ - WARNING: Not fully implemeted - """ model = 'vps.VPS' resource = ServiceMonitor.TRAFFIC monthly_sum_old_values = True + def prepare(self): + super(OpenVZTraffic, self).prepare() + self.append(ProxmoxOVZ.GET_PROXMOX_INFO) + self.append(textwrap.dedent(""" + function monitor () { + object_id=$1 + hostname=$2 + info=( $(get_vz_info $hostname) ) + cat << EOF | ssh root@${info[1]} + vzctl exec ${info[0]} cat /proc/net/dev \\ + | grep venet0 \\ + | tr ':' ' ' \\ + | awk '{sum=\\$2+\\$10} END {printf ("$object_id %0.0f\\n", sum)}' + EOF + } + """) + ) + def process(self, line): - """ diff with last stored value """ - object_id, value = line.split() + """ diff with last stored state """ + object_id, value, state = super(OpenVZTraffic, self).process(line) + value = decimal.Decimal(value) last = self.get_last_data(object_id) - if not last or last.value > value: - return object_id, value - return object_id, value-last.value + if not last or last.state > value: + return object_id, value, value + return object_id, value-last.state, value - def monitor(self, container): - """ Get OpenVZ container traffic on a Proxmox +2.0 cluster """ - context = self.get_context(container) - self.append( - "CONF=$(grep -r 'HOSTNAME=\"%(hostname)s\"' /etc/pve/nodes/*/openvz/*.conf)" % context) - self.append('NODE=$(echo "${CONF}" | cut -d"/" -f5)') - self.append('CTID=$(echo "${CONF}" | cut -d"/" -f7 | cur -d"\." -f1)') - self.append( - "ssh root@${NODE} vzctl exec ${CTID} cat /proc/net/dev \\\n" - " | grep venet0 \\\n" - " | awk -F: '{print $2}' \\\n" - " | awk '{print $1+$9}'") + def monitor(self, vps): + """ Get OpenVZ container traffic on a Proxmox cluster """ + context = self.get_context(vps) + self.append('monitor %(object_id)s %(hostname)s' % context) - def get_context(self, container): - context = { - 'hostname': container.hostname, + def get_context(self, vps): + return { + 'object_id': vps.id, + 'hostname': vps.hostname, } - return replace(context, "'", '"') diff --git a/orchestra/contrib/vps/forms.py b/orchestra/contrib/vps/forms.py deleted file mode 100644 index a6f12aab..00000000 --- a/orchestra/contrib/vps/forms.py +++ /dev/null @@ -1,39 +0,0 @@ -from django import forms -from django.contrib.auth.forms import ReadOnlyPasswordHashField -from django.utils.translation import ugettext_lazy as _ - - -class VPSCreationForm(forms.ModelForm): - password1 = forms.CharField(label=_("Password"), - widget=forms.PasswordInput) - password2 = forms.CharField(label=_("Password confirmation"), - widget=forms.PasswordInput, - help_text=_("Enter the same password as above, for verification.")) - - class Meta: - fields = ('username', 'account', 'type', 'template') - - def clean_password2(self): - password1 = self.cleaned_data.get("password1") - password2 = self.cleaned_data.get("password2") - if password1 and password2 and password1 != password2: - msg = _("The two password fields didn't match.") - raise forms.ValidationError(msg) - return password2 - - def save(self, commit=True): - vps = super(VPSCreationForm, self).save(commit=False) - vps.set_password(self.cleaned_data["password1"]) - if commit: - vps.save() - return vps - - -class VPSChangeForm(forms.ModelForm): - password = ReadOnlyPasswordHashField(label=_("Password"), - help_text=_("Raw passwords are not stored, so there is no way to see " - "this user's password, but you can change the password " - "using this form.")) - - def clean_password(self): - return self.initial["password"] diff --git a/orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py b/orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py new file mode 100644 index 00000000..c3569416 --- /dev/null +++ b/orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('vps', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='vps', + name='password', + ), + migrations.AlterField( + model_name='vps', + name='template', + field=models.CharField(default='debian7', verbose_name='template', choices=[('debian7', 'Debian 7 - Wheezy')], max_length=64, help_text='Initial template.'), + ), + ] diff --git a/orchestra/contrib/vps/models.py b/orchestra/contrib/vps/models.py index 27691f3c..93441fac 100644 --- a/orchestra/contrib/vps/models.py +++ b/orchestra/contrib/vps/models.py @@ -13,9 +13,8 @@ class VPS(models.Model): type = models.CharField(_("type"), max_length=64, choices=settings.VPS_TYPES, default=settings.VPS_DEFAULT_TYPE) template = models.CharField(_("template"), max_length=64, - choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE) - password = models.CharField(_('password'), max_length=128, - help_text=_("root password of this virtual machine")) + choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE, + help_text=_("Initial template.")) account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), related_name='vpss') diff --git a/orchestra/forms/options.py b/orchestra/forms/options.py index e81e68da..afa0f6ec 100644 --- a/orchestra/forms/options.py +++ b/orchestra/forms/options.py @@ -60,7 +60,7 @@ class UserCreationForm(forms.ModelForm): class UserChangeForm(forms.ModelForm): password = auth_forms.ReadOnlyPasswordHashField(label=_("Password"), help_text=_("Raw passwords are not stored, so there is no way to see " - "this user's password, but you can change the password " + "this user's password, but you can change it by " "using this form.")) def clean_password(self): @@ -70,6 +70,13 @@ class UserChangeForm(forms.ModelForm): return self.initial["password"] +class NonStoredUserChangeForm(forms.ModelForm): + password = forms.CharField(label=_("Password"), required=False, + widget=SpanWidget(display='Unknown password'), + help_text=_("This service's password is not stored, so there is no way to see it, " + "but you can change it using this form.")) + + class ReadOnlyFormMixin(object): """ Mixin class for ModelForm or Form that provides support for SpanField on readonly fields diff --git a/orchestra/utils/sys.py b/orchestra/utils/sys.py index ba631535..734ee474 100644 --- a/orchestra/utils/sys.py +++ b/orchestra/utils/sys.py @@ -142,7 +142,11 @@ def run(command, display=False, valid_codes=(0,), silent=False, stdin=b'', async def sshrun(addr, command, *args, executable='bash', persist=False, **kwargs): from .. import settings - options = ['stricthostkeychecking=no'] + options = [ + 'stricthostkeychecking=no', + 'BatchMode=yes', + 'EscapeChar=none', + ] if persist: options.extend(( 'ControlMaster=auto',