Fixes on resources, mailer and vps0

TODO.md

@@ -36,8 +36,6 @@
 
 * What fields we really need on contacts? name email phone and what more?
 
-* Redirect junk emails and delete every 30 days?
-
 * DOC: Complitely decouples scripts execution, billing, service definition
 
 * init.d celery scripts
@@ -58,21 +56,11 @@
 * env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python3 manage.py test orchestra.contrib.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture --keepdb
 
 * ForeignKey.swappable
-* Field.editable
-* ManyToManyField.symmetrical = False (user group)
 
 * REST PERMISSIONS
 
-* caching based on "def text2int(textnum, numwords={}):"
-
-* sync() ServiceController method that synchronizes orchestra and servers (delete or import)
-
-* consider removing mailbox support on forward (user@pangea.org instead)
-
 * Databases.User add reverse M2M databases widget (like mailbox.addresses)
 
-* Grant permissions to systemusers
-
 * Make one dedicated CGI user for each account only for CGI execution (fpm/fcgid). Different from the files owner, and without W permissions, so attackers can not inject backdors and malware.
 
 * resource min max allocation with validation
@@ -81,24 +69,14 @@
 
 * Directory Protection on webapp and use webapp path as base path (validate)
 
-* validate systemuser.home on server-side
-
 * webapp backend option compatibility check? raise exception, missconfigured error
 
-* admin systemuser home/directory, add default home and empty directory with has_shell on admin
-
 * Resource used_list_display=True, allocated_list_displat=True, allow resources to show up on list_display
 
 * BackendLog.updated_at (tasks that run over several minutes when finished they do not appear first on the changelist) (like celery tasks.when)
 
-* Periodic task for cleaning old monitoring data
-
 * Create an admin service_view with icons (like SaaS app)
 
-* Resource graph for each related object
-
-* SaaS model splitted into SaaSUser and SaaSSite? inherit from SaaS, proxy model?
-
 * prevent @pangea.org email addresses on contacts, enforce at least one email without @pangea.org
 
 * forms autocomplete="off", doesn't work in chrome
@@ -125,8 +103,6 @@ Php binaries should have this format: /usr/bin/php5.2-cgi
 
 * contact.alternative_phone on a phone.tooltip, email:to
 
-* better validate options and directives (url locations, filesystem paths, etc..)
-
 * make sure that you understand the risks
 
 * full support for deactivation of services/accounts
@@ -295,7 +271,7 @@ https://code.djangoproject.com/ticket/24576
 # Settings.parser.changes: if setting.value == default. remove
 # reload generic admin view ?redirect=http...
 # inspecting django db connection for asserting db readines? or performing a query
-# wake up django mailer on send_mail
+* wake up django mailer on send_mail
 
         from orchestra.contrib.tasks import task
         import time, sys
@@ -308,14 +284,14 @@ https://code.djangoproject.com/ticket/24576
                 time.sleep(1)
         counter.apply_async(10, '/tmp/kakas')
 
-# Provide some fixtures with mocked data
+* Provide some fixtures with mocked data
 
 
 TODO http://wiki2.dovecot.org/HowTo/SimpleVirtualInstall
 TODO http://wiki2.dovecot.org/HowTo/VirtualUserFlatFilesPostfix
 TODO mount the filesystem with "nosuid" option
 
-# uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
+* uwse uwsgi cron: decorator or config cron = 59 2 -1 -1 -1 %(virtualenv)/bin/python manage.py runmyfunnytask
 
 # mailboxes.address settings multiple local domains, not only one?
 # backend.context = self.get_context() or save(obj, context=None) ?? more like form.cleaned_data
@@ -323,7 +299,7 @@ TODO mount the filesystem with "nosuid" option
 # smtplib.SMTPConnectError: (421, b'4.7.0 mail.pangea.org Error: too many connections from 77.246.181.209')
 
 # rename virtual_maps to virtual_alias_maps and remove virtual_alias_domains ?
-# virtdomains file is not ideal, prevent fake/error on domains there! and make sure to chekc if this file is required!
+# virtdomains file is not ideal, prevent user provided fake/error domains there! and make sure to chekc if this file is required!
 
 # Deprecate restart/start/stop services (do touch wsgi.py and fuck celery)
 orchestra-beat support for uwsgi cron
@@ -338,11 +314,9 @@ resorce monitoring more efficient, less mem an better queries for calc current d
 
 # bill this https://orchestra.pangea.org/admin/orders/order/8236/ should be already billed, <= vs <
 # Convert rating method from function to PluginClass
-# Tests can not run because django.db.utils.ProgrammingError: relation "accounts_account" does not exist
 
 # autoresponses on mailboxes, not addresses or remove them
 
-# ACL don't give exec permissions to files!
 # force save and continue on routes (and others?)
 # gevent for python3
 apt-get install cython3
@@ -384,8 +358,6 @@ uwsgi --reload /tmp/project-master.pid
 # or if uwsgi was started with touch-reload=/tmp/somefile
 touch /tmp/somefile
 
-# datetime metric storage granularity: otherwise innacurate detection of billed metric on order.billed_on
-
 # Serializers.validation migration to DRF3: grep -r 'attrs, source' *|grep -v '~'
 serailzer self.instance on create.
 
@@ -427,14 +399,14 @@ Case
 
 # round decimals on every billing operation
 
-# Add SPF record type
-
-# OVZ TRAFFIC ACCOUNTING!!
-
 # PHPlist cron, bounces and traffic (maybe specific mail script with sitename)
 
-
 # use "su $user --shell /bin/bash" on backends for security : MKDIR -p...
 
-
 # model.field.flatchoices
+
+* This is beta software, please test thoroughly before putting into production and report back any issues.
+
+# messages SMTP errors: temporary->deferre else Failed
+
+# Don't enforce one contact per account? remove account.email in favour of contacts?

orchestra/bin/orchestra-beat

@@ -191,7 +191,7 @@ def fire_pending_messages(settings, db):
         for num, seconds in enumerate(MAILER_DEFERE_SECONDS):
             delta = timedelta(seconds=seconds)
             epoch = now-delta
-            query_or.append("""(mailer_message.retries = %i AND mailer_message.last_retry <= '%s')"""
+            query_or.append("""(mailer_message.retries = %i AND mailer_message.last_try <= '%s')"""
                 % (num, epoch.isoformat().replace('T', ' ')))
         query = """\
             SELECT 1 FROM mailer_message

orchestra/contrib/lists/forms.py

@@ -1,8 +1,6 @@
-from django import forms
 from django.utils.translation import ugettext_lazy as _
 
-from orchestra.core.validators import validate_password
+from orchestra.forms import UserCreationForm, NonStoredUserChangeForm
-from orchestra.forms.widgets import SpanWidget
 
 
 class CleanAddressMixin(object):
@@ -15,25 +13,9 @@ class CleanAddressMixin(object):
         return domain
 
 
-class ListCreationForm(CleanAddressMixin, forms.ModelForm):
+class ListCreationForm(CleanAddressMixin, UserCreationForm):
-    password1 = forms.CharField(label=_("Password"), validators=[validate_password],
+    pass
-        widget=forms.PasswordInput)
-    password2 = forms.CharField(label=_("Password confirmation"),
-        widget=forms.PasswordInput,
-        help_text=_("Enter the same password as above, for verification."))
-    
-    def clean_password2(self):
-        password1 = self.cleaned_data.get("password1")
-        password2 = self.cleaned_data.get("password2")
-        if password1 and password2 and password1 != password2:
-            msg = _("The two password fields didn't match.")
-            raise forms.ValidationError(msg)
-        return password2
 
 
-class ListChangeForm(CleanAddressMixin, forms.ModelForm):
+class ListChangeForm(CleanAddressMixin, NonStoredUserChangeForm):
-    password = forms.CharField(label=_("Password"), required=False,
+    pass
-        widget=SpanWidget(display='<strong>Unknown password</strong>'),
-        help_text=_("List passwords are not stored, so there is no way to see this "
-                    "list's password, but you can change the password using "
-                    "<a href=\"password/\">this form</a>."))

orchestra/contrib/mailer/admin.py

@@ -28,13 +28,13 @@ COLORS = {
 class MessageAdmin(admin.ModelAdmin):
     list_display = (
         'display_subject', 'colored_state', 'priority', 'to_address', 'from_address',
-        'created_at_delta', 'retries', 'last_retry_delta', 'num_logs',
+        'created_at_delta', 'display_retries', 'last_try_delta',
     )
     list_filter = ('state', 'priority', 'retries')
     list_prefetch_related = ('logs__id')
     fieldsets = (
         (None, {
-            'fields': ('state', 'priority', ('retries', 'last_retry_delta', 'created_at_delta'),
+            'fields': ('state', 'priority', ('retries', 'last_try_delta', 'created_at_delta'),
                        'display_full_subject', 'display_from', 'display_to',
                        'display_content'),
         }),
@@ -44,36 +44,36 @@ class MessageAdmin(admin.ModelAdmin):
         }),
     )
     readonly_fields = (
-        'retries', 'last_retry_delta', 'created_at_delta', 'display_full_subject',
+        'retries', 'last_try_delta', 'created_at_delta', 'display_full_subject',
         'display_to', 'display_from', 'display_content',
     )
     date_hierarchy = 'created_at'
     
     colored_state = admin_colored('state', colors=COLORS)
     created_at_delta = admin_date('created_at')
-    last_retry_delta = admin_date('last_retry')
+    last_try_delta = admin_date('last_try')
     
     def display_subject(self, instance):
         subject = instance.subject
-        if len(subject) > 32:
+        if len(subject) > 64:
-            return subject[:32] + '…'
+            return subject[:64] + '…'
         return subject
     display_subject.short_description = _("Subject")
     display_subject.admin_order_field = 'subject'
     display_subject.allow_tags = True
     
-    def num_logs(self, instance):
+    def display_retries(self, instance):
-        num = instance.logs__count
+        num_logs = instance.logs__count
-        if num == 1:
+        if num_logs == 1:
             pk = instance.logs.all()[0].id
             url = reverse('admin:mailer_smtplog_change', args=(pk,))
         else:
             url = reverse('admin:mailer_smtplog_changelist')
             url += '?&message=%i' % instance.pk
-        return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, num)
+        return '<a href="%s" onclick="return showAddAnotherPopup(this);">%d</a>' % (url, instance.retries)
-    num_logs.short_description = _("Logs")
+    display_retries.short_description = _("Retries")
-    num_logs.admin_order_field = 'logs__count'
+    display_retries.admin_order_field = 'retries'
-    num_logs.allow_tags = True
+    display_retries.allow_tags = True
     
     def display_content(self, instance):
         part = email.message_from_string(instance.content)

orchestra/contrib/mailer/engine.py

@@ -23,6 +23,12 @@ def send_message(message, num=0, connection=None, bulk=settings.MAILER_BULK_MESS
         connection = get_connection(backend='django.core.mail.backends.smtp.EmailBackend')
         connection.open()
     error = None
+    message.last_try = timezone.now()
+    update_fields = ['last_try']
+    if message.state != message.QUEUED:
+        message.retries += 1
+        update_fields.append('retries')
+    message.save(update_fields=update_fields)
     try:
         connection.connection.sendmail(message.from_address, [message.to_address], smart_str(message.content))
     except (SocketError,
@@ -49,7 +55,7 @@ def send_pending(bulk=settings.MAILER_BULK_MESSAGES):
             qs = Q()
             for retries, seconds in enumerate(settings.MAILER_DEFERE_SECONDS):
                 delta = timedelta(seconds=seconds)
-                qs = qs | Q(retries=retries, last_retry__lte=now-delta)
+                qs = qs | Q(retries=retries, last_try__lte=now-delta)
             for message in Message.objects.filter(state=Message.DEFERRED).filter(qs).order_by('priority'):
                 connection = send_message(message, num, connection, bulk)
                 num += 1

orchestra/contrib/mailer/migrations/0004_auto_20150805_1328.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mailer', '0003_auto_20150617_1024'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='message',
+            old_name='last_retry',
+            new_name='last_try',
+        ),
+        migrations.AlterField(
+            model_name='message',
+            name='state',
+            field=models.CharField(verbose_name='State', max_length=16, choices=[('QUEUED', 'Queued'), ('SENT', 'Sent'), ('DEFERRED', 'Deferred'), ('FAILED', 'Failed')], default='QUEUED'),
+        ),
+    ]

orchestra/contrib/mailer/models.py

@@ -1,5 +1,4 @@
 from django.db import models
-from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
 
 from . import settings
@@ -14,7 +13,7 @@ class Message(models.Model):
         (QUEUED, _("Queued")),
         (SENT, _("Sent")),
         (DEFERRED, _("Deferred")),
-        (FAILED, _("Failes")),
+        (FAILED, _("Failed")),
     )
     
     CRITICAL = 0
@@ -36,8 +35,7 @@ class Message(models.Model):
     content = models.TextField(_("content"))
     created_at = models.DateTimeField(_("created at"), auto_now_add=True)
     retries = models.PositiveIntegerField(_("retries"), default=0)
-    # TODO rename to last_try
+    last_try = models.DateTimeField(_("last try"), null=True)
-    last_retry = models.DateTimeField(_("last try"), null=True)
     
     def __str__(self):
         return '%s to %s' % (self.subject, self.to_address)
@@ -47,9 +45,7 @@ class Message(models.Model):
         # Max tries
         if self.retries >= len(settings.MAILER_DEFERE_SECONDS):
             self.state = self.FAILED
-        self.retries += 1
+        self.save(update_fields=('state',))
-        self.last_retry = timezone.now()
-        self.save(update_fields=('state', 'retries', 'last_retry'))
     
     def sent(self):
         self.state = self.SENT

orchestra/contrib/resources/admin.py

@@ -198,7 +198,7 @@ class MonitorDataAdmin(ExtendedModelAdmin):
     list_display = ('id', 'monitor', content_object_link, 'display_created', 'value')
     list_filter = ('monitor', ResourceDataListFilter)
     add_fields = ('monitor', 'content_type', 'object_id', 'created_at', 'value')
-    fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value')
+    fields = ('monitor', 'content_type', content_object_link, 'display_created', 'value', 'state')
     change_readonly_fields = fields
     list_select_related = ('content_type',)
     search_fields = ('content_object_repr',)
@@ -258,6 +258,7 @@ def resource_inline_factory(resources):
                     if resource not in queryset_resources:
                         kwargs = {
                             'content_object': self.instance,
+                            'content_object_repr': str(self.instance),
                         }
                         if resource.default_allocation:
                             kwargs['allocated'] = resource.default_allocation
@@ -323,7 +324,9 @@ def resource_inline_factory(resources):
                 used_url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,))
                 used =  '<a href="%s">%s %s</a>' % (used_url, rdata.used, rdata.unit)
                 return ' '.join(map(str, (used, update, history)))
-            return _("Unknonw %s %s") % (update, history)
+            if rdata.resource.monitors:
+                return _("Unknonw %s %s") % (update, history)
+            return _("No monitor")
         display_used.short_description = _("Used")
         display_used.allow_tags = True
         

orchestra/contrib/resources/backends.py

@@ -57,8 +57,14 @@ class ServiceMonitor(ServiceBackend):
         return data.created_at
     
     def process(self, line):
-        """ line -> object_id, value """
+        """ line -> object_id, value, state"""
-        return line.split()
+        result = line.split()
+        if len(result) != 2:
+            cls_name = self.__class__.__name__
+            raise ValueError("%s expected '<id> <value>' got '%s'" % (cls_name, line))
+        # State is None, unless your monitor needs to keep track of it
+        result.append(None)
+        return result
     
     def store(self, log):
         """ stores monitored values from stdout """
@@ -68,16 +74,14 @@ class ServiceMonitor(ServiceBackend):
         ct = ContentType.objects.get_by_natural_key(app_label, model_name.lower())
         for line in log.stdout.splitlines():
             line = line.strip()
-            try:
+            object_id, value, state = self.process(line)
-                object_id, value = self.process(line)
-            except ValueError:
-                cls_name = self.__class__.__name__
-                raise ValueError("%s expected '<id> <value>' got '%s'" % (cls_name, line))
             if isinstance(value, bytes):
                 value = value.decode('ascii')
+            if isinstance(state, bytes):
+                state = state.decode('ascii')
             content_object = ct.get_object_for_this_type(pk=object_id)
             MonitorData.objects.create(
-                monitor=name, object_id=object_id, content_type=ct, value=value,
+                monitor=name, object_id=object_id, content_type=ct, value=value, state=state,
                 created_at=self.current_date, content_object_repr=str(content_object),
             )
     

orchestra/contrib/resources/migrations/0008_monitordata_state.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('resources', '0007_auto_20150723_1251'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='monitordata',
+            name='state',
+            field=models.DecimalField(verbose_name='state', null=True, max_digits=16, help_text='Optional field used to store current state needed for diff-based monitoring.', decimal_places=2),
+        ),
+    ]

orchestra/contrib/resources/migrations/0009_auto_20150804_1450.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('resources', '0008_monitordata_state'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='resourcedata',
+            name='allocated',
+            field=models.PositiveIntegerField(verbose_name='allocated', null=True, blank=True),
+        ),
+    ]

orchestra/contrib/resources/models.py

@@ -172,8 +172,7 @@ class ResourceData(models.Model):
     used = models.DecimalField(_("used"), max_digits=16, decimal_places=3, null=True,
         editable=False)
     updated_at = models.DateTimeField(_("updated"), null=True, editable=False)
-    allocated = models.DecimalField(_("allocated"), max_digits=8, decimal_places=2,
+    allocated = models.PositiveIntegerField(_("allocated"), null=True, blank=True)
-        null=True, blank=True)
     content_object_repr = models.CharField(_("content object representation"), max_length=256,
         editable=False)
     
@@ -268,6 +267,8 @@ class MonitorData(models.Model):
     object_id = models.PositiveIntegerField(_("object id"), db_index=True)
     created_at = models.DateTimeField(_("created"), default=timezone.now, db_index=True)
     value = models.DecimalField(_("value"), max_digits=16, decimal_places=2)
+    state = models.DecimalField(_("state"), max_digits=16, decimal_places=2, null=True,
+        help_text=_("Optional field used to store current state needed for diff-based monitoring."))
     content_object_repr = models.CharField(_("content object representation"), max_length=256,
         editable=False)
     
@@ -308,6 +309,7 @@ def create_resource_relation():
                 )
                 rdata = ResourceData(
                     content_object=self.obj,
+                    content_object_repr=str(self.obj),
                     resource=resource,
                     allocated=resource.default_allocation
                 )

orchestra/contrib/resources/serializers.py

@@ -33,13 +33,18 @@ class ResourceSerializer(serializers.ModelSerializer):
 def insert_resource_serializers():
     # clean previous state
     for related in Resource._related:
-        viewset = router.get_viewset(related)
-        fields = list(viewset.serializer_class.Meta.fields)
         try:
-            fields.remove('resources')
+            viewset = router.get_viewset(related)
-        except ValueError:
+        except KeyError:
+            # API viewset not registered
             pass
-        viewset.serializer_class.Meta.fields = fields
+        else:
+            fields = list(viewset.serializer_class.Meta.fields)
+            try:
+                fields.remove('resources')
+            except ValueError:
+                pass
+            viewset.serializer_class.Meta.fields = fields
     # Create nested serializers on target models
     for ct, resources in Resource.objects.group_by('content_type').items():
         model = ct.model_class()

orchestra/contrib/resources/tasks.py

@@ -63,7 +63,7 @@ def monitor(resource_id, ids=None):
 def cleanup_old_monitors(queryset=None):
     if queryset is None:
         from .models import MonitorData
-        queryset = MonitorData.objects.filter()
+        queryset = MonitorData.objects.all()
     delta = datetime.timedelta(days=settings.RESOURCES_OLD_MONITOR_DATA_DAYS)
     threshold = timezone.now() - delta
     queryset = queryset.filter(created_at__lt=threshold)

orchestra/contrib/saas/backends/phplist.py

@@ -34,7 +34,8 @@ class PhpListSaaSBackend(ServiceController):
         sys.stderr.write(msg + '\n')
         raise RuntimeError(msg)
     
-    def _save(self, saas, server):
+    def _install_or_change_password(self, saas, server):
+        """ configures the database for the new site through HTTP to /admin/ """
         admin_link = 'https://%s/admin/' % saas.get_site_domain()
         sys.stdout.write('admin_link: %s\n' % admin_link)
         admin_content = requests.get(admin_link, verify=settings.SAAS_PHPLIST_VERIFY_SSL)
@@ -75,7 +76,7 @@ class PhpListSaaSBackend(ServiceController):
     
     def save(self, saas):
         if hasattr(saas, 'password'):
-            self.append(self._save, saas)
+            self.append(self._install_or_change_password, saas)
         context = self.get_context(saas)
         if context['crontab']:
             context['escaped_crontab'] = context['crontab'].replace('$', '\\$')

orchestra/contrib/services/templates/admin/services/service/change_form.html

@@ -37,7 +37,7 @@
                         is_fee=&
                         order_description=&
                         ignore_period=TEN_DAYS&
-                        metric=max(logsteps(mailbox.resources.disk.allocated%20or%200)%20-2,%200)&
+                        metric=max((mailbox.resources.disk.allocated%20or%200)%20-2,%200)&
                         nominal_price=20.00&
                         tax=21&
                         pricing_period=&

orchestra/contrib/systemusers/backends.py

@@ -1,3 +1,4 @@
+import fnmatch
 import os
 import textwrap
 
@@ -97,6 +98,51 @@ class UNIXUserBackend(ServiceController):
         else:
             self.append("rm -fr %(base_home)s" % context)
     
+    def grant_permissions(self, user, context):
+        context['perms'] = user.set_perm_perms
+        # Capital X adds execution permissions for directories, not files
+        context['perms_X'] = context['perms'] + 'X'
+        self.append(textwrap.dedent("""\
+            # Grant execution permissions to every parent directory
+            for access_path in %(access_paths)s; do
+                # Preserve existing ACLs
+                acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
+                    perms=$(echo "$acl" | cut -d':' -f3)
+                    perms=$(echo "$perms" | cut -c 1,2)x
+                    setfacl -m u:%(user)s:$perms "$access_path"
+                } || setfacl -m u:%(user)s:--x "$access_path"
+            done
+            # Grant perms to existing files, excluding execution
+            find '%(perm_to)s' -type f %(exclude_acl)s \\
+                -exec setfacl -m u:%(user)s:%(perms)s {} \\;
+            # Grant perms to extisting directories and set defaults for future content
+            find '%(perm_to)s' -type d %(exclude_acl)s \\
+                -exec setfacl -m u:%(user)s:%(perms_X)s -m d:u:%(user)s:%(perms_X)s {} \\;
+            # Account group as the owner of new files
+            chmod g+s '%(perm_to)s'""") % context
+        )
+        if not user.is_main:
+            self.append(textwrap.dedent("""\
+                # Grant access to main user
+                find '%(perm_to)s' -type d %(exclude_acl)s \\
+                    -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
+                """) % context
+            )
+    
+    def revoke_permissions(self, context):
+        revoke_perms = {
+            'rw': '',
+            'r': 'w',
+            'w': 'r',
+        }
+        context['perms'] = revoke_perms[user.set_perm_perms]
+        self.append(textwrap.dedent("""\
+            # Revoke permissions
+            find '%(perm_to)s' %(exclude_acl)s \\
+                -exec setfacl -m u:%(user)s:%(perms)s {} \\;\
+            """) % context
+        )
+    
     def set_permission(self, user):
         context = self.get_context(user)
         context.update({
@@ -108,17 +154,10 @@ class UNIXUserBackend(ServiceController):
             context['exclude_acl'] = os.path.join(user.set_perm_base_home, exclude)
             exclude_acl.append('-not -path "%(exclude_acl)s"' % context)
         context['exclude_acl'] = ' \\\n    -a '.join(exclude_acl) if exclude_acl else ''
-        if user.set_perm_perms == 'rw':
-            context['perm_perms'] = 'rwx' if user.set_perm_action == 'grant' else '---'
-        elif user.set_perm_perms == 'r':
-            context['perm_perms'] = 'r-x' if user.set_perm_action == 'grant' else '-wx'
-        elif user.set_perm_perms == 'w':
-            context['perm_perms'] = '-wx' if user.set_perm_action == 'grant' else 'r-x'
         # Access paths
         head = user.set_perm_base_home
         relative = ''
         access_paths = ["'%s'" % head]
-        import fnmatch
         for tail in user.set_perm_home_extension.split(os.sep)[:-1]:
             relative = os.path.join(relative, tail)
             for exclude in settings.SYSTEMUSERS_FORBIDDEN_PATHS:
@@ -129,39 +168,11 @@ class UNIXUserBackend(ServiceController):
                 head = os.path.join(head, tail)
                 access_paths.append("'%s'" % head)
         context['access_paths'] = ' '.join(access_paths)
+        
         if user.set_perm_action == 'grant':
-            self.append(textwrap.dedent("""\
+            self.grant_permissions(user, context)
-                # Grant execution permissions to every parent directory
-                for access_path in %(access_paths)s; do
-                    # Preserve existing ACLs
-                    acl=$(getfacl -a "$access_path" | grep '^user:%(user)s:') && {
-                        perms=$(echo "$acl" | cut -d':' -f3)
-                        perms=$(echo "$perms" | cut -c 1,2)x
-                        setfacl -m u:%(user)s:$perms "$access_path"
-                    } || setfacl -m u:%(user)s:--x "$access_path"
-                done
-                # Grant perms to existing and future files
-                find '%(perm_to)s' %(exclude_acl)s \\
-                    -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;
-                find '%(perm_to)s' -type d %(exclude_acl)s \\
-                    -exec setfacl -m d:u:%(user)s:%(perm_perms)s {} \\;
-                # Account group as the owner of new files
-                chmod g+s '%(perm_to)s'""") % context
-            )
-            if not user.is_main:
-                self.append(textwrap.dedent("""\
-                    # Grant access to main user
-                    find '%(perm_to)s' -type d %(exclude_acl)s \\
-                        -exec setfacl -m d:u:%(mainuser)s:rwx {} \\;\
-                    """) % context
-                )
         elif user.set_perm_action == 'revoke':
-            self.append(textwrap.dedent("""\
+            self.revoke_permissions(user, context)
-                # Revoke permissions
-                find '%(perm_to)s' %(exclude_acl)s \\
-                    -exec setfacl -m u:%(user)s:%(perm_perms)s {} \\;\
-                """) % context
-            )
         else:
             raise NotImplementedError()
     

orchestra/contrib/systemusers/models.py

@@ -45,7 +45,7 @@ class SystemUser(models.Model):
         help_text=_("Optional directory relative to user's home."))
     shell = models.CharField(_("shell"), max_length=32, choices=settings.SYSTEMUSERS_SHELLS,
         default=settings.SYSTEMUSERS_DEFAULT_SHELL)
-    groups = models.ManyToManyField('self', blank=True,  symmetrical=False,
+    groups = models.ManyToManyField('self', blank=True, symmetrical=False,
         help_text=_("A new group will be created for the user. "
                     "Which additional groups would you like them to be a member of?"))
     is_active = models.BooleanField(_("active"), default=True,

orchestra/contrib/vps/admin.py

@@ -6,18 +6,18 @@ from django.utils.translation import ugettext_lazy as _
 from orchestra.admin import ExtendedModelAdmin
 from orchestra.contrib.accounts.actions import list_accounts
 from orchestra.contrib.accounts.admin import AccountAdminMixin
+from orchestra.forms import UserCreationForm, NonStoredUserChangeForm
 
-from .forms import VPSChangeForm, VPSCreationForm
 from .models import VPS
 
 
 class VPSAdmin(AccountAdminMixin, ExtendedModelAdmin):
     list_display = ('hostname', 'type', 'template', 'account_link')
     list_filter = ('type', 'template')
-    form = VPSChangeForm
+    form = NonStoredUserChangeForm
-    add_form = VPSCreationForm
+    add_form = UserCreationForm
     readonly_fields = ('account_link',)
-    change_readonly_fields = ('account', 'name', 'type', 'template')
+    change_readonly_fields = ('account', 'hostname', 'type', 'template')
     fieldsets = (
         (None, {
             'classes': ('wide',),

orchestra/contrib/vps/backends.py

@@ -1,38 +1,114 @@
-from orchestra.contrib.orchestration import replace
+import decimal
+import textwrap
+
+from orchestra.contrib.orchestration import ServiceController
 from orchestra.contrib.resources import ServiceMonitor
 
 
+class ProxmoxOVZ(ServiceController):
+    model = 'vps.VPS'
+    
+    RESOURCES = (
+        ('memory', 'mem'),
+        ('swap', 'swap'),
+        ('disk', 'disk')
+    )
+    GET_PROXMOX_INFO = textwrap.dedent("""
+        function get_vz_info () {
+            hostname=$1
+            version=$(pveversion | cut -d '/' -f2 | cut -d'.' -f1)
+            if [[ $version -lt 2 ]]; then
+                conf=$(grep "CID\\|:$hostname:" /var/lib/pve-manager/vzlist | grep -B1 ":$hostname:")
+                CID=$(echo "$conf" | head -n1 | cut -d':' -f2)
+                CTID=$(echo "$conf" | tail -n1 | cut -d':' -f1)
+                node=$(pveca -l | grep "^\\s*$CID\\s*:" | awk {'print $3'})
+            else
+                conf=$(grep -r "HOSTNAME=\\"$hostname\\"" /etc/pve/nodes/*/openvz/*.conf)
+                node=$(echo "${conf}" | cut -d"/" -f5)
+                CTID=$(echo "${conf}" | cut -d"/" -f7 | cut -d"\\." -f1)
+            fi
+            echo $CTID $node
+        }""")
+    
+    def prepare(self):
+        super(ProxmoxOVZ, self).prepare()
+        self.append(self.GET_PROXMOX_INFO)
+    
+    def get_vzset_args(self, vps):
+        args = []
+        for resource, arg_name in self.RESOURCES:
+            try:
+                allocation = getattr(vps.resources, resource).allocated
+            except AttributeError:
+                pass
+            else:
+                args.append('--%s %i' % (arg_name, allocation))
+        return ' '.join(args)
+    
+    def save(self, vps):
+        context = self.get_context(vps)
+        self.append('info=( $(get_vz_info %(hostname)s) )' % context)
+        vzset_args = self.get_vzset_args(vps)
+        if vzset_args:
+            context['vzset_args'] = vzset_args
+            self.append(textwrap.dedent("""\
+                cat << EOF | ssh root@${info[1]}
+                    pvectl vzset ${info[0]} %(vzset_args)s
+                EOF""") % context
+            )
+        if hasattr(vps, 'password'):
+            context['password'] = vps.password.replace('$', '\\$')
+            self.append(textwrap.dedent("""\
+                cat << EOF | ssh root@${info[1]}
+                    echo 'root:%(password)s' | vzctl exec ${info[0]} chpasswd -e
+                EOF""") % context
+            )
+    def get_context(self, vps):
+        return {
+            'hostname': vps.hostname,
+        }
+
+
+# TODO rename to proxmox
 class OpenVZTraffic(ServiceMonitor):
-    """
-    WARNING: Not fully implemeted
-    """
     model = 'vps.VPS'
     resource = ServiceMonitor.TRAFFIC
     monthly_sum_old_values = True
     
+    def prepare(self):
+        super(OpenVZTraffic, self).prepare()
+        self.append(ProxmoxOVZ.GET_PROXMOX_INFO)
+        self.append(textwrap.dedent("""
+            function monitor () {
+                object_id=$1
+                hostname=$2
+                info=( $(get_vz_info $hostname) )
+                cat << EOF | ssh root@${info[1]}
+                    vzctl exec ${info[0]} cat /proc/net/dev \\
+                        | grep venet0 \\
+                        | tr ':' ' ' \\
+                        | awk '{sum=\\$2+\\$10} END {printf ("$object_id %0.0f\\n", sum)}'
+            EOF
+            }
+            """)
+        )
+    
     def process(self, line):
-        """ diff with last stored value """
+        """ diff with last stored state """
-        object_id, value = line.split()
+        object_id, value, state = super(OpenVZTraffic, self).process(line)
+        value = decimal.Decimal(value)
         last = self.get_last_data(object_id)
-        if not last or last.value > value:
+        if not last or last.state > value:
-            return object_id, value
+            return object_id, value, value
-        return object_id, value-last.value
+        return object_id, value-last.state, value
     
-    def monitor(self, container):
+    def monitor(self, vps):
-        """ Get OpenVZ container traffic on a Proxmox +2.0 cluster """
+        """ Get OpenVZ container traffic on a Proxmox cluster """
-        context = self.get_context(container)
+        context = self.get_context(vps)
-        self.append(
+        self.append('monitor %(object_id)s %(hostname)s' % context)
-            "CONF=$(grep -r 'HOSTNAME=\"%(hostname)s\"' /etc/pve/nodes/*/openvz/*.conf)" % context)
-        self.append('NODE=$(echo "${CONF}" | cut -d"/" -f5)')
-        self.append('CTID=$(echo "${CONF}" | cut -d"/" -f7 | cur -d"\." -f1)')
-        self.append(
-            "ssh root@${NODE} vzctl exec ${CTID} cat /proc/net/dev \\\n"
-            "    | grep venet0 \\\n"
-            "    | awk -F: '{print $2}' \\\n"
-            "    | awk '{print $1+$9}'")
     
-    def get_context(self, container):
+    def get_context(self, vps):
-        context = {
+        return {
-            'hostname': container.hostname,
+            'object_id': vps.id,
+            'hostname': vps.hostname,
         }
-        return replace(context, "'", '"')

orchestra/contrib/vps/forms.py

@@ -1,39 +0,0 @@
-from django import forms
-from django.contrib.auth.forms import ReadOnlyPasswordHashField
-from django.utils.translation import ugettext_lazy as _
-
-
-class VPSCreationForm(forms.ModelForm):
-    password1 = forms.CharField(label=_("Password"),
-        widget=forms.PasswordInput)
-    password2 = forms.CharField(label=_("Password confirmation"),
-        widget=forms.PasswordInput,
-        help_text=_("Enter the same password as above, for verification."))
-    
-    class Meta:
-        fields = ('username', 'account', 'type', 'template')
-    
-    def clean_password2(self):
-        password1 = self.cleaned_data.get("password1")
-        password2 = self.cleaned_data.get("password2")
-        if password1 and password2 and password1 != password2:
-            msg = _("The two password fields didn't match.")
-            raise forms.ValidationError(msg)
-        return password2
-    
-    def save(self, commit=True):
-        vps = super(VPSCreationForm, self).save(commit=False)
-        vps.set_password(self.cleaned_data["password1"])
-        if commit:
-            vps.save()
-        return vps
-
-
-class VPSChangeForm(forms.ModelForm):
-    password = ReadOnlyPasswordHashField(label=_("Password"),
-        help_text=_("Raw passwords are not stored, so there is no way to see "
-                    "this user's password, but you can change the password "
-                    "using <a href=\"password/\">this form</a>."))
-    
-    def clean_password(self):
-        return self.initial["password"]

orchestra/contrib/vps/migrations/0002_auto_20150804_1524.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('vps', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='vps',
+            name='password',
+        ),
+        migrations.AlterField(
+            model_name='vps',
+            name='template',
+            field=models.CharField(default='debian7', verbose_name='template', choices=[('debian7', 'Debian 7 - Wheezy')], max_length=64, help_text='Initial template.'),
+        ),
+    ]

orchestra/contrib/vps/models.py

@@ -13,9 +13,8 @@ class VPS(models.Model):
     type = models.CharField(_("type"), max_length=64, choices=settings.VPS_TYPES,
         default=settings.VPS_DEFAULT_TYPE)
     template = models.CharField(_("template"), max_length=64,
-        choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE)
+        choices=settings.VPS_TEMPLATES, default=settings.VPS_DEFAULT_TEMPLATE,
-    password = models.CharField(_('password'), max_length=128,
+        help_text=_("Initial template."))
-        help_text=_("<TT>root</TT> password of this virtual machine"))
     account = models.ForeignKey('accounts.Account', verbose_name=_("Account"),
         related_name='vpss')
     

orchestra/forms/options.py

@@ -60,7 +60,7 @@ class UserCreationForm(forms.ModelForm):
 class UserChangeForm(forms.ModelForm):
     password = auth_forms.ReadOnlyPasswordHashField(label=_("Password"),
         help_text=_("Raw passwords are not stored, so there is no way to see "
-                    "this user's password, but you can change the password "
+                    "this user's password, but you can change it by "
                     "using <a href=\"password/\">this form</a>."))
     
     def clean_password(self):
@@ -70,6 +70,13 @@ class UserChangeForm(forms.ModelForm):
         return self.initial["password"]
 
 
+class NonStoredUserChangeForm(forms.ModelForm):
+    password = forms.CharField(label=_("Password"), required=False,
+        widget=SpanWidget(display='<strong>Unknown password</strong>'),
+        help_text=_("This service's password is not stored, so there is no way to see it, "
+                    "but you can change it using <a href=\"password/\">this form</a>."))
+
+
 class ReadOnlyFormMixin(object):
     """
     Mixin class for ModelForm or Form that provides support for SpanField on readonly fields

orchestra/utils/sys.py

@@ -142,7 +142,11 @@ def run(command, display=False, valid_codes=(0,), silent=False, stdin=b'', async
 
 def sshrun(addr, command, *args, executable='bash', persist=False, **kwargs):
     from .. import settings
-    options = ['stricthostkeychecking=no']
+    options = [
+        'stricthostkeychecking=no',
+        'BatchMode=yes',
+        'EscapeChar=none',
+    ]
     if persist:
         options.extend((
             'ControlMaster=auto',