Lots of improvements on webapps and saas

TODO.md

@@ -241,12 +241,25 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl
 
 * WPMU blog traffic
 
-* normurlpath '' returns '/'
+* normurlpath '' return '/'
 
 * rename webapps.type to something more generic
 
 * initial configuration of multisite sas apps with password stored in DATA
 
-* websites links on webpaps ans saas
+* webapps installation complete, passowrd protected
+* saas.initial_password autogenerated (ok because its random and not user provided) vs saas.password /change_Form provided + send email with initial_password
+
+* disable saas apps
+
+* more robust backend error handling, continue executing but exit code > 0 if failure, replace exit_code=0; do_sometging || exit_code=1
+
+* saas require unique emails? connect to backend server to find out because they change
+
+* automaitcally set passwords and email users?
+
+* website directives  uniquenes validation on serializers
+
+* gitlab store id, username changes
+
 
-* /var/lib/fcgid/wrappers/ rm write permissions

orchestra/apps/databases/backends.py

@@ -38,10 +38,11 @@ class MySQLBackend(ServiceController):
         if database.type != database.MYSQL:
             return
         context = self.get_context(database)
-        self.append("mysql -e 'DROP DATABASE `%(database)s`;'" % context)
+        self.append("mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=1" % context)
         self.append("mysql mysql -e 'DELETE FROM db WHERE db = \"%(database)s\";'" % context)
         
     def commit(self):
+        super(MySQLBackend, self).commit()
         self.append("mysql -e 'FLUSH PRIVILEGES;'")
     
     def get_context(self, database):

orchestra/apps/domains/settings.py

@@ -36,8 +36,6 @@ DOMAINS_SLAVES_PATH = getattr(settings, 'DOMAINS_SLAVES_PATH', '/etc/bind/named.
 DOMAINS_CHECKZONE_BIN_PATH = getattr(settings, 'DOMAINS_CHECKZONE_BIN_PATH',
     '/usr/sbin/named-checkzone -i local -k fail -n fail')
 
-DOMAINS_CHECKZONE_PATH = getattr(settings, 'DOMAINS_CHECKZONE_PATH', '/dev/shm')
-
 
 DOMAINS_DEFAULT_A = getattr(settings, 'DOMAINS_DEFAULT_A', '10.0.3.13')
 

orchestra/apps/domains/validators.py

@@ -108,11 +108,10 @@ def validate_soa_record(value):
 def validate_zone(zone):
     """ Ultimate zone file validation using named-checkzone """
     zone_name = zone.split()[0][:-1]
-    path = os.path.join(settings.DOMAINS_CHECKZONE_PATH, zone_name)
-    with open(path, 'wb') as f:
-        f.write(zone)
     checkzone = settings.DOMAINS_CHECKZONE_BIN_PATH
-    check = run(' '.join([checkzone, zone_name, path]), error_codes=[0,1], display=False)
+    cmd = ' '.join(["echo -e '%s'" % zone, '|', checkzone, zone_name, '/dev/stdin'])
+    print cmd
+    check = run(cmd, error_codes=[0, 1], display=False)
     if check.return_code == 1:
         errors = re.compile(r'zone.*: (.*)').findall(check.stdout)[:-1]
         raise ValidationError(', '.join(errors))

orchestra/apps/orchestration/backends.py

@@ -177,7 +177,8 @@ class ServiceBackend(plugins.Plugin):
         """
         self.append(
             'set -e\n'
-            'set -o pipefail'
+            'set -o pipefail\n'
+            'exit_code=0;'
         )
     
     def commit(self):
@@ -187,7 +188,7 @@ class ServiceBackend(plugins.Plugin):
         reloading a service is done in a separated method in order to reload 
         the service once in bulk operations
         """
-        self.append('exit 0')
+        self.append('exit $exit_code')
 
 
 class ServiceController(ServiceBackend):

orchestra/apps/orchestration/manager.py

@@ -22,18 +22,10 @@ def as_task(execute):
     def wrapper(*args, **kwargs):
         """ send report """
         # Tasks run on a separate transaction pool (thread), no need to temper with the transaction
-        log = execute(*args, **kwargs)
-        if log.state != log.SUCCESS:
-            send_report(execute, args, log)
-        return log
-    return wrapper
-
-
-def close_connection(execute):
-    """ Threads have their own connection pool, closing it when finishing """
-    def wrapper(*args, **kwargs):
         try:
             log = execute(*args, **kwargs)
+            if log.state != log.SUCCESS:
+                send_report(execute, args, log)
         except Exception as e:
             subject = 'EXCEPTION executing backend(s) %s %s' % (str(args), str(kwargs))
             message = traceback.format_exc()
@@ -45,6 +37,19 @@ def close_connection(execute):
             # Using the wrapper function as threader messenger for the execute output
             # Absense of it will indicate a failure at this stage
             wrapper.log = log
+            return log
+    return wrapper
+
+
+def close_connection(execute):
+    """ Threads have their own connection pool, closing it when finishing """
+    def wrapper(*args, **kwargs):
+        try:
+            log = execute(*args, **kwargs)
+        except:
+            pass
+        else:
+            wrapper.log = log
         finally:
             db.connection.close()
     return wrapper
@@ -89,15 +94,15 @@ def execute(operations, async=False):
         backend, operations = value
         backend.commit()
         execute = as_task(backend.execute)
-        execute = close_connection(execute)
-        # DEBUG: substitute all thread related stuff for this function
-        #execute(server, async=async)
         logger.debug('%s is going to be executed on %s' % (backend, server))
-        thread = threading.Thread(target=execute, args=(server,), kwargs={'async': async})
-        thread.start()
         if block:
-            thread.join()
+            # Execute one bakend at a time, no need for threads
-        threads.append(thread)
+            execute(server, async=async)
+        else:
+            execute = close_connection(execute)
+            thread = threading.Thread(target=execute, args=(server,), kwargs={'async': async})
+            thread.start()
+            threads.append(thread)
         executions.append((execute, operations))
     [ thread.join() for thread in threads ]
     logs = []
@@ -108,7 +113,9 @@ def execute(operations, async=False):
             for operation in operations:
                 logger.info("Executed %s" % str(operation))
                 operation.log = execution.log
-                operation.save()
+                if operation.object_id:
+                    # Not all backends are call with objects saved on the database
+                    operation.save()
             stdout = execution.log.stdout.strip()
             stdout and logger.debug('STDOUT %s', stdout)
             stderr = execution.log.stderr.strip()

orchestra/apps/resources/helpers.py

@@ -6,6 +6,7 @@ def compute_resource_usage(data):
     resource = data.resource
     result = 0
     has_result = False
+    today = datetime.date.today()
     for dataset in data.get_monitor_datasets():
         if resource.period == resource.MONTHLY_AVG:
             last = dataset.latest()

orchestra/apps/saas/admin.py

@@ -11,18 +11,19 @@ from .services import SoftwareService
 
 
 class SaaSAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin):
-    list_display = ('username', 'service', 'display_site_name', 'account_link')
+    list_display = ('name', 'service', 'display_site_domain', 'account_link')
     list_filter = ('service',)
+    change_readonly_fields = ('service',)
     plugin = SoftwareService
     plugin_field = 'service'
     plugin_title = 'Software as a Service'
     
-    def display_site_name(self, saas):
+    def display_site_domain(self, saas):
-        site_name = saas.get_site_name()
+        site_domain = saas.get_site_domain()
-        return '<a href="http://%s">%s</a>' % (site_name, site_name)
+        return '<a href="http://%s">%s</a>' % (site_domain, site_domain)
-    display_site_name.short_description = _("Site name")
+    display_site_domain.short_description = _("Site domain")
-    display_site_name.allow_tags = True
+    display_site_domain.allow_tags = True
-    display_site_name.admin_order_field = 'site_name'
+    display_site_domain.admin_order_field = 'name'
 
 
 admin.site.register(SaaS, SaaSAdmin)

orchestra/apps/saas/backends/gitlab.py

@@ -0,0 +1,101 @@
+import json
+
+import requests
+from django.utils.translation import ugettext_lazy as _
+
+from orchestra.apps.orchestration import ServiceController
+
+from .. import settings
+
+
+class GitLabSaaSBackend(ServiceController):
+    verbose_name = _("GitLab SaaS")
+    model = 'saas.SaaS'
+    default_route_match = "saas.service == 'gitlab'"
+    block = True
+    actions = ('save', 'delete', 'validate_creation')
+    
+    def get_base_url(self):
+        return 'https://%s/api/v3' %  settings.SAAS_GITLAB_DOMAIN
+    
+    def get_user_url(self, saas):
+        user_id = saas.data['user_id']
+        return self.get_base_url() + '/users/%i' % user_id
+    
+    def validate_response(self, response, status_codes):
+        if response.status_code not in status_codes:
+            raise RuntimeError("[%i] %s" % (response.status_code, response.content))
+    
+    def authenticate(self):
+        login_url = self.get_base_url() + '/session'
+        data = {
+            'login': 'root',
+            'password': settings.SAAS_GITLAB_ROOT_PASSWORD,
+        }
+        response = requests.post(login_url, data=data)
+        self.validate_response(response, [201])
+        token = json.loads(response.content)['private_token']
+        self.headers = {
+            'PRIVATE-TOKEN': token,
+        }
+    
+    def create_user(self, saas, server):
+        self.authenticate()
+        user_url = self.get_base_url() + '/users'
+        data = {
+            'email': saas.data['email'],
+            'password': saas.password,
+            'username': saas.name,
+            'name': saas.account.get_full_name(),
+        }
+        response = requests.post(user_url, data=data, headers=self.headers)
+        self.validate_response(response, [201])
+        print response.content
+        user = json.loads(response.content)
+        saas.data['user_id'] = user['id']
+        # Using queryset update to avoid triggering backends with the post_save signal
+        type(saas).objects.filter(pk=saas.pk).update(data=saas.data)
+        print json.dumps(user, indent=4)
+    
+    def change_password(self, saas, server):
+        self.authenticate()
+        user_url = self.get_user_url(saas)
+        data = {
+            'password': saas.password,
+        }
+        response = requests.patch(user_url, data=data, headers=self.headers)
+        self.validate_response(response, [200])
+        print json.dumps(json.loads(response.content), indent=4)
+    
+    def delete_user(self, saas, server):
+        self.authenticate()
+        user_url = self.get_user_url(saas)
+        response = requests.delete(user_url, headers=self.headers)
+        self.validate_response(response, [200, 404])
+        print json.dumps(json.loads(response.content), indent=4)
+    
+    def _validate_creation(self, saas, server):
+        """ checks if a saas object is valid for creation on the server side """
+        self.authenticate()
+        username = saas.name
+        email = saas.data['email']
+        users_url = self.get_base_url() + '/users/'
+        users = json.loads(requests.get(users_url, headers=self.headers).content)
+        for user in users:
+            if user['username'] == username:
+                print 'user-exists'
+            if user['email'] == email:
+                print 'email-exists'
+    
+    def validate_creation(self, saas):
+        self.append(self._validate_creation, saas)
+    
+    def save(self, saas):
+        if hasattr(saas, 'password'):
+            if saas.data.get('user_id', None):
+                self.append(self.change_password, saas)
+            else:
+                self.append(self.create_user, saas)
+    
+    def delete(self, saas):
+        self.append(self.delete_user, saas)

orchestra/apps/saas/backends/phplist.py

@@ -17,7 +17,7 @@ class PhpListSaaSBackend(ServiceController):
     
     def initialize_database(self, saas, server):
         base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN
-        admin_link = 'http://%s.%s/admin/' % (saas.get_site_name(), base_domain)
+        admin_link = 'http://%s/admin/' % saas.get_site_domain()
         admin_content = requests.get(admin_link).content
         if admin_content.startswith('Cannot connect to Database'):
             raise RuntimeError("Database is not yet configured")
@@ -28,7 +28,7 @@ class PhpListSaaSBackend(ServiceController):
             install = install.groups()[0]
             install_link = admin_link + install[1:]
             post = {
-                'adminname': saas.username,
+                'adminname': saas.name,
                 'orgname': saas.account.username,
                 'adminemail': saas.account.username,
                 'adminpassword': saas.password,

orchestra/apps/saas/models.py

@@ -14,10 +14,9 @@ from .services import SoftwareService
 class SaaS(models.Model):
     service = models.CharField(_("service"), max_length=32,
             choices=SoftwareService.get_plugin_choices())
-    username = models.CharField(_("name"), max_length=64,
+    name = models.CharField(_("Name"), max_length=64,
             help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."),
             validators=[validators.validate_username])
-#    site_name = NullableCharField(_("site name"), max_length=32, null=True)
     account = models.ForeignKey('accounts.Account', verbose_name=_("account"),
             related_name='saas')
     data = JSONField(_("data"), default={},
@@ -27,12 +26,11 @@ class SaaS(models.Model):
         verbose_name = "SaaS"
         verbose_name_plural = "SaaS"
         unique_together = (
-            ('username', 'service'),
+            ('name', 'service'),
-#            ('site_name', 'service'),
         )
     
     def __unicode__(self):
-        return "%s@%s" % (self.username, self.service)
+        return "%s@%s" % (self.name, self.service)
     
     @cached_property
     def service_class(self):
@@ -43,12 +41,12 @@ class SaaS(models.Model):
         """ Per request lived service_instance """
         return self.service_class(self)
     
-    def get_site_name(self):
-        return self.service_instance.get_site_name()
-    
     def clean(self):
         self.data = self.service_instance.clean_data()
     
+    def get_site_domain(self):
+        return self.service_instance.get_site_domain()
+    
     def set_password(self, password):
         self.password = password
 

orchestra/apps/saas/services/bscw.py

@@ -11,12 +11,14 @@ from .options import SoftwareService, SoftwareServiceForm
 
 class BSCWForm(SoftwareServiceForm):
     email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'}))
-    quota = forms.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB."))
+    quota = forms.IntegerField(label=_("Quota"), initial=settings.SAAS_BSCW_DEFAULT_QUOTA,
+            help_text=_("Disk quota in MB."))
 
 
 class BSCWDataSerializer(serializers.Serializer):
     email = serializers.EmailField(label=_("Email"))
-    quota = serializers.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB."))
+    quota = serializers.IntegerField(label=_("Quota"), default=settings.SAAS_BSCW_DEFAULT_QUOTA,
+            help_text=_("Disk quota in MB."))
 
 
 class BSCWService(SoftwareService):
@@ -26,5 +28,5 @@ class BSCWService(SoftwareService):
     serializer = BSCWDataSerializer
     icon = 'orchestra/icons/apps/BSCW.png'
     # TODO override from settings
-    site_name = settings.SAAS_BSCW_DOMAIN
+    site_domain = settings.SAAS_BSCW_DOMAIN
     change_readonly_fileds = ('email',)

orchestra/apps/saas/services/gitlab.py

@@ -1,6 +1,50 @@
-from .options import SoftwareService
+from django import forms
+from django.core.exceptions import ValidationError
+from django.utils.translation import ugettext_lazy as _
+from rest_framework import serializers
+
+from orchestra.apps.orchestration.models import BackendOperation as Operation
+from orchestra.forms import widgets
+
+from .options import SoftwareService, SoftwareServiceForm
+
+from .. import settings
+
+
+class GitLabForm(SoftwareServiceForm):
+    email = forms.EmailField(label=_("Email"),
+            help_text=_("Initial email address, changes on the GitLab server are not reflected here."))
+
+
+class GitLaChangebForm(GitLabForm):
+    user_id = forms.IntegerField(label=("User ID"), widget=widgets.ShowTextWidget,
+            help_text=_("ID of this user on the GitLab server, the only attribute that not changes."))
+
+
+class GitLabSerializer(serializers.Serializer):
+    email = serializers.EmailField(label=_("Email"))
+    user_id = serializers.IntegerField(label=_("User ID"), required=False)
 
 
 class GitLabService(SoftwareService):
+    name = 'gitlab'
+    form = GitLabForm
+    change_form = GitLaChangebForm
+    serializer = GitLabSerializer
+    site_domain = settings.SAAS_GITLAB_DOMAIN
+    change_readonly_fileds = ('email', 'user_id',)
     verbose_name = "GitLab"
     icon = 'orchestra/icons/apps/gitlab.png'
+    
+    def clean_data(self):
+        data = super(GitLabService, self).clean_data()
+        if not self.instance.pk:
+            log = Operation.execute_action(self.instance, 'validate_creation')[0]
+            errors = {}
+            if 'user-exists' in log.stdout:
+                errors['name'] = _("User with this username already exists.")
+            elif 'email-exists' in log.stdout:
+                errors['email'] = _("User with this email address already exists.")
+            if errors:
+                raise ValidationError(errors)
+        return data

orchestra/apps/saas/services/options.py

@@ -8,13 +8,13 @@ from orchestra.plugins.forms import PluginDataForm
 from orchestra.core import validators
 from orchestra.forms import widgets
 from orchestra.utils.functional import cached
-from orchestra.utils.python import import_class
+from orchestra.utils.python import import_class, random_ascii
 
 from .. import settings
 
 
 class SoftwareServiceForm(PluginDataForm):
-    site_name = forms.CharField(widget=widgets.ShowTextWidget, required=False)
+    site_url = forms.CharField(label=_("Site URL"), widget=widgets.ShowTextWidget, required=False)
     password = forms.CharField(label=_("Password"), required=False,
             widget=widgets.ReadOnlyWidget('<strong>Unknown password</strong>'),
             help_text=_("Passwords are not stored, so there is no way to see this "
@@ -30,25 +30,21 @@ class SoftwareServiceForm(PluginDataForm):
         super(SoftwareServiceForm, self).__init__(*args, **kwargs)
         self.is_change = bool(self.instance and self.instance.pk)
         if self.is_change:
-            site_name = self.instance.get_site_name()
+            site_domain = self.instance.get_site_domain()
             self.fields['password1'].required = False
             self.fields['password1'].widget = forms.HiddenInput()
             self.fields['password2'].required = False
             self.fields['password2'].widget = forms.HiddenInput()
         else:
             self.fields['password'].widget = forms.HiddenInput()
-            site_name = self.plugin.site_name
+            self.fields['password1'].help_text = _("Suggestion: %s") % random_ascii(10)
-        if site_name:
+            site_domain = self.plugin.site_domain
-            site_name_link = '<a href="http://%s">%s</a>' % (site_name, site_name)
+        if site_domain:
+            site_link = '<a href="http://%s">%s</a>' % (site_domain, site_domain)
         else:
-            site_name_link = '&lt;name&gt;.%s' % self.plugin.site_name_base_domain
+            site_link = '&lt;site_name&gt;.%s' % self.plugin.site_base_domain
-        self.fields['site_name'].initial = site_name_link
+        self.fields['site_url'].initial = site_link
-##            self.fields['site_name'].widget = widgets.ReadOnlyWidget(site_name, mark_safe(link))
+        self.fields['name'].label = _("Username")
-##            self.fields['site_name'].required = False
-#        else:
-#            base_name = self.plugin.site_name_base_domain
-#            help_text = _("The final URL would be &lt;site_name&gt;.%s") % base_name
-#            self.fields['site_name'].help_text = help_text
     
     def clean_password2(self):
         if not self.is_change:
@@ -59,11 +55,6 @@ class SoftwareServiceForm(PluginDataForm):
                 raise forms.ValidationError(msg)
             return password2
     
-    def clean_site_name(self):
-        if self.plugin.site_name:
-            return None
-        return self.cleaned_data['site_name']
-    
     def save(self, commit=True):
         obj = super(SoftwareServiceForm, self).save(commit=commit)
         if not self.is_change:
@@ -73,11 +64,10 @@ class SoftwareServiceForm(PluginDataForm):
 
 class SoftwareService(plugins.Plugin):
     form = SoftwareServiceForm
-    site_name = None
+    site_domain = None
-    site_name_base_domain = 'orchestra.lan'
+    site_base_domain = None
     has_custom_domain = False
     icon = 'orchestra/icons/apps.png'
-    change_readonly_fileds = ('site_name',)
     class_verbose_name = _("Software as a Service")
     plugin_field = 'service'
     
@@ -89,14 +79,13 @@ class SoftwareService(plugins.Plugin):
             plugins.append(import_class(cls))
         return plugins
     
-    @classmethod
     def get_change_readonly_fileds(cls):
         fields = super(SoftwareService, cls).get_change_readonly_fileds()
-        return fields + ('username',)
+        return fields + ('name',)
     
-    def get_site_name(self):
+    def get_site_domain(self):
-        return self.site_name or '.'.join(
+        return self.site_domain or '.'.join(
-            (self.instance.username, self.site_name_base_domain)
+            (self.instance.name, self.site_base_domain)
         )
     
     def save(self):

orchestra/apps/saas/services/phplist.py

@@ -17,23 +17,22 @@ class PHPListForm(SoftwareServiceForm):
     
     def __init__(self, *args, **kwargs):
         super(PHPListForm, self).__init__(*args, **kwargs)
-        self.fields['username'].label = _("Name")
+        self.fields['name'].label = _("Site name")
-        base_domain = self.plugin.site_name_base_domain
+        base_domain = self.plugin.site_base_domain
-        help_text = _("Admin URL http://<name>.{}/admin/").format(base_domain)
+        help_text = _("Admin URL http://<site_name>.{}/admin/").format(base_domain)
-        self.fields['site_name'].help_text = help_text
+        self.fields['site_url'].help_text = help_text
 
 
 class PHPListChangeForm(PHPListForm):
-#    site_name = forms.CharField(widget=widgets.ShowTextWidget, required=False)
     db_name = forms.CharField(label=_("Database name"),
             help_text=_("Database used for this webapp."))
     
     def __init__(self, *args, **kwargs):
         super(PHPListChangeForm, self).__init__(*args, **kwargs)
-        site_name = self.instance.get_site_name()
+        site_domain = self.instance.get_site_domain()
-        admin_url = "http://%s/admin/" % site_name
+        admin_url = "http://%s/admin/" % site_domain
         help_text = _("Admin URL <a href={0}>{0}</a>").format(admin_url)
-        self.fields['site_name'].help_text = help_text
+        self.fields['site_url'].help_text = help_text
 
 
 class PHPListSerializer(serializers.Serializer):
@@ -48,21 +47,25 @@ class PHPListService(SoftwareService):
     change_readonly_fileds = ('db_name',)
     serializer = PHPListSerializer
     icon = 'orchestra/icons/apps/Phplist.png'
-    site_name_base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN
+    site_base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN
     
     def get_db_name(self):
-        db_name = 'phplist_mu_%s' % self.instance.username
+        db_name = 'phplist_mu_%s' % self.instance.name
         # Limit for mysql database names
         return db_name[:65]
     
     def get_db_user(self):
         return settings.SAAS_PHPLIST_DB_NAME
     
+    def get_account(self):
+        return type(self.instance.account).get_main()
+    
     def validate(self):
         super(PHPListService, self).validate()
         create = not self.instance.pk
         if create:
-            db = Database(name=self.get_db_name(), account=self.instance.account)
+            account = self.get_account()
+            db = Database(name=self.get_db_name(), account=account)
             try:
                 db.full_clean()
             except ValidationError as e:
@@ -73,7 +76,8 @@ class PHPListService(SoftwareService):
     def save(self):
         db_name = self.get_db_name()
         db_user = self.get_db_user()
-        db, db_created = Database.objects.get_or_create(name=db_name, account=self.instance.account)
+        account = self.get_account()
+        db, db_created = account.databases.get_or_create(name=db_name)
         user = DatabaseUser.objects.get(username=db_user)
         db.users.add(user)
         self.instance.data = {
@@ -90,9 +94,10 @@ class PHPListService(SoftwareService):
     
     def get_related(self):
         related = []
-        account = self.instance.account
+        account = self.get_account()
+        db_name = self.instance.data.get('db_name')
         try:
-            db = account.databases.get(name=self.instance.data.get('db_name'))
+            db = account.databases.get(name=db_name)
         except Database.DoesNotExist:
             pass
         else:

orchestra/apps/saas/settings.py

@@ -47,3 +47,16 @@ SAAS_BSCW_DOMAIN = getattr(settings, 'SAAS_BSCW_DOMAIN',
 )
 
 
+SAAS_BSCW_DEFAULT_QUOTA = getattr(settings, 'SAAS_BSCW_DEFAULT_QUOTA',
+    50
+)
+
+
+SAAS_GITLAB_ROOT_PASSWORD = getattr(settings, 'SAAS_GITLAB_ROOT_PASSWORD',
+    'secret'
+)
+
+SAAS_GITLAB_DOMAIN = getattr(settings, 'SAAS_GITLAB_DOMAIN',
+    'gitlab.orchestra.lan'
+)
+

orchestra/apps/webapps/backends/php.py

@@ -28,10 +28,11 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         self.create_webapp_dir(context)
         self.set_under_construction(context)
         self.append(textwrap.dedent("""\
+            fpm_config='%(fpm_config)s'
             {
-                echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s -
+                echo -e "${fpm_config}" | diff -N -I'^\s*;;' %(fpm_path)s -
             } || {
-                echo -e '%(fpm_config)s' > %(fpm_path)s
+                echo -e "${fpm_config}" > %(fpm_path)s
                 UPDATEDFPM=1
             }""") % context
         )
@@ -41,20 +42,23 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
         self.set_under_construction(context)
         self.append("mkdir -p %(wrapper_dir)s" % context)
         self.append(textwrap.dedent("""\
+            wrapper='%(wrapper)s'
             {
-                echo -e '%(wrapper)s' | diff -N -I'^\s*#' %(wrapper_path)s -
+                echo -e "${wrapper}" | diff -N -I'^\s*#' %(wrapper_path)s -
             } || {
-                echo -e '%(wrapper)s' > %(wrapper_path)s; UPDATED_APACHE=1
+                echo -e "${wrapper}" > %(wrapper_path)s; UPDATED_APACHE=1
             }""") % context
         )
-        self.append("chmod +x %(wrapper_path)s" % context)
+        self.append("chmod 550 %(wrapper_dir)s" % context)
+        self.append("chmod 550 %(wrapper_path)s" % context)
         self.append("chown -R %(user)s:%(group)s %(wrapper_dir)s" % context)
         if context['cmd_options']:
             self.append(textwrap.dedent("""
+                cmd_options='%(cmd_options)s'
                 {
-                    echo -e '%(cmd_options)s' | diff -N -I'^\s*#' %(cmd_options_path)s -
+                    echo -e "${cmd_options}" | diff -N -I'^\s*#' %(cmd_options_path)s -
                 } || {
-                    echo -e '%(cmd_options)s' > %(cmd_options_path)s; UPDATED_APACHE=1
+                    echo -e "${cmd_options}" > %(cmd_options_path)s; UPDATED_APACHE=1
                 }""" ) % context
             )
         else:

orchestra/apps/webapps/settings.py

@@ -16,6 +16,7 @@ WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
 
 
 WEBAPPS_FCGID_WRAPPER_PATH = getattr(settings, 'WEBAPPS_FCGID_WRAPPER_PATH',
+    # Inside SuExec Document root
     '/home/httpd/fcgi-bin.d/%(user)s/%(app_name)s-wrapper')
 
 

orchestra/apps/webapps/types/__init__.py

@@ -89,5 +89,6 @@ class AppType(plugins.Plugin):
             'app_id': self.instance.id,
             'app_name': self.instance.name,
             'user': self.instance.account.username,
+            'home': self.instance.account.main_systemuser.get_home(),
         }
 

orchestra/apps/webapps/types/misc.py

@@ -33,8 +33,8 @@ class WebalizerApp(AppType):
     icon = 'orchestra/icons/apps/Stats.png'
     option_groups = ()
     
-    def get_directive(self, webapp):
+    def get_directive(self):
-        webalizer_path = os.path.join(webapp.get_path(), '%(site_name)s')
+        webalizer_path = os.path.join(self.instance.get_path(), '%(site_name)s')
         webalizer_path = os.path.normpath(webalizer_path)
         return ('static', webalizer_path)
 

orchestra/apps/webapps/types/php.py

@@ -57,15 +57,6 @@ class PHPApp(AppType):
     def get_detail(self):
         return self.instance.data.get('php_version', '')
     
-    def get_context(self):
-        """ context used to format settings """
-        return {
-            'home': self.instance.account.main_systemuser.get_home(),
-            'account': self.instance.account.username,
-            'user': self.instance.account.username,
-            'app_name': self.instance.name,
-        }
-    
     def get_php_init_vars(self, merge=False):
         """
         process php options for inclusion on php.ini
@@ -77,17 +68,17 @@ class PHPApp(AppType):
             # Get options from the same account and php_version webapps
             options = []
             php_version = self.get_php_version()
-            webapps = self.instance.account.webapps.filter(webapp_type=self.instance.type)
+            webapps = self.instance.account.webapps.filter(type=self.instance.type)
             for webapp in webapps:
                 if webapp.type_instance.get_php_version == php_version:
                     options += list(webapp.options.all())
         php_options = [option.name for option in type(self).get_php_options()]
+        enabled_functions = set()
         for opt in options:
             if opt.name in php_options:
                 init_vars[opt.name] = opt.value
-        enabled_functions = []
+            elif opt.name == 'enabled_functions':
-        for value in options.filter(name='enabled_functions').values_list('value', flat=True):
+                enabled_functions.union(set(opt.value.split(',')))
-            enabled_functions += enabled_functions.get().value.split(',')
         if enabled_functions:
             disabled_functions = []
             for function in self.PHP_DISABLED_FUNCTIONS:
@@ -95,11 +86,18 @@ class PHPApp(AppType):
                     disabled_functions.append(function)
             init_vars['dissabled_functions'] = ','.join(disabled_functions)
         if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars:
-            context = self.get_context()
+            context = self.get_directive_context()
             error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context)
             init_vars['error_log'] = error_log_path
         return init_vars
     
+    def get_directive_context(self):
+        context = super(PHPApp, self).get_directive_context()
+        context.update({
+            'php_version': self.get_php_version(),
+        })
+        return context
+    
     def get_directive(self):
         context = self.get_directive_context()
         if self.is_fpm:

orchestra/apps/websites/admin.py

@@ -12,12 +12,13 @@ from orchestra.forms.widgets import DynamicHelpTextSelect
 
 from . import settings
 from .directives import SiteDirective
-from .forms import WebsiteAdminForm
+from .forms import WebsiteAdminForm, WebsiteDirectiveInlineFormSet
 from .models import Content, Website, WebsiteDirective
 
 
 class WebsiteDirectiveInline(admin.TabularInline):
     model = WebsiteDirective
+    formset = WebsiteDirectiveInlineFormSet
     extra = 1
     
     DIRECTIVES_HELP_TEXT = {

orchestra/apps/websites/backends/apache.py

@@ -31,6 +31,7 @@ class Apache2Backend(ServiceController):
         extra_conf += self.get_security(directives)
         extra_conf += self.get_redirects(directives)
         extra_conf += self.get_proxies(directives)
+        extra_conf += self.get_saas(directives)
         # Order extra conf directives based on directives (longer first)
         extra_conf = sorted(extra_conf, key=lambda a: len(a[0]), reverse=True)
         context['extra_conf'] = '\n'.join([conf for location, conf in extra_conf])
@@ -46,7 +47,7 @@ class Apache2Backend(ServiceController):
                 SuexecUserGroup {{ user }} {{ group }}\
             {% for line in extra_conf.splitlines %}
                 {{ line | safe }}{% endfor %}
-                #IncludeOptional /etc/apache2/extra-vhos[t]/{{ site_unique_name }}.con[f]
+                IncludeOptional /etc/apache2/extra-vhos[t]/{{ site_unique_name }}.con[f]
             </VirtualHost>
             """)
         ).render(Context(context))
@@ -80,10 +81,11 @@ class Apache2Backend(ServiceController):
             apache_conf += self.render_redirect_https(context)
         context['apache_conf'] = apache_conf
         self.append(textwrap.dedent("""\
+            apache_conf='%(apache_conf)s'
             {
-                echo -e '%(apache_conf)s' | diff -N -I'^\s*#' %(sites_available)s -
+                echo -e "${apache_conf}" | diff -N -I'^\s*#' %(sites_available)s -
             } || {
-                echo -e '%(apache_conf)s' > %(sites_available)s
+                echo -e "${apache_conf}" > %(sites_available)s
                 UPDATED=1
             }""") % context
         )
@@ -116,7 +118,7 @@ class Apache2Backend(ServiceController):
         return directives
     
     def get_static_directives(self, context, app_path):
-        context['app_path'] = app_path % context
+        context['app_path'] = os.path.normpath(app_path % context)
         location = "%(location)s/" % context
         directive = "Alias %(location)s/ %(app_path)s/" % context
         return [(location, directive)]
@@ -128,10 +130,10 @@ class Apache2Backend(ServiceController):
         else:
             # UNIX socket
             target = 'unix:%(socket)s|fcgi://127.0.0.1%(app_path)s/'
-            if context['location'] != '/':
+            if context['location']:
                 target = 'unix:%(socket)s|fcgi://127.0.0.1%(app_path)s/$1'
         context.update({
-            'app_path': app_path,
+            'app_path': os.path.normpath(app_path),
             'socket': socket,
         })
         location = "%(location)s/" % context
@@ -143,7 +145,7 @@ class Apache2Backend(ServiceController):
     
     def get_fcgid_directives(self, context, app_path, wrapper_path):
         context.update({
-            'app_path': app_path,
+            'app_path': os.path.normpath(app_path),
             'wrapper_path': wrapper_path,
         })
         location = "%(location)s/" % context
@@ -158,16 +160,20 @@ class Apache2Backend(ServiceController):
         return [(location, directives)]
     
     def get_ssl(self, directives):
-        config = ''
-        ca = directives.get('ssl_ca')
-        if ca:
-            config += "SSLCACertificateFile %s\n" % ca[0]
         cert = directives.get('ssl_cert')
-        if cert:
-            config += "SSLCertificateFile %\n" % cert[0]
         key = directives.get('ssl_key')
-        if key:
+        ca = directives.get('ssl_ca')
-            config += "SSLCertificateKeyFile %s\n" % key[0]
+        if not (cert and key):
+            cert = [settings.WEBSITES_DEFAULT_SSL_CERT]
+            key = [settings.WEBSITES_DEFAULT_SSL_KEY]
+            ca = [settings.WEBSITES_DEFAULT_SSL_CA]
+            if not (cert and key):
+                return []
+        config = 'SSLEngine on\n'
+        config += "SSLCertificateFile %s\n" % cert[0]
+        config += "SSLCertificateKeyFile %s\n" % key[0]
+        if ca:
+           config += "SSLCACertificateFile %s\n" % ca[0]
         return [('', config)]
         
     def get_security(self, directives):
@@ -210,13 +216,14 @@ class Apache2Backend(ServiceController):
     
     def get_saas(self, directives):
         saas = []
-        for name, value in directives.iteritems():
+        for name, values in directives.iteritems():
             if name.endswith('-saas'):
-                context = {
+                for value in values:
-                    'location': normurlpath(value),
+                    context = {
-                }
+                        'location': normurlpath(value),
-                directive = settings.WEBSITES_SAAS_DIRECTIVES[name]
+                    }
-                saas += self.get_directive(context, directive)
+                    directive = settings.WEBSITES_SAAS_DIRECTIVES[name]
+                    saas += self.get_directives(directive, context)
         return saas
 #    def get_protections(self, site):
 #        protections = ''
@@ -280,7 +287,8 @@ class Apache2Backend(ServiceController):
             'site_unique_name': site.unique_name,
             'user': self.get_username(site),
             'group': self.get_groupname(site),
-            'sites_enabled': "%s.conf" % os.path.join(sites_enabled, site.unique_name),
+            # TODO remove '0-'
+            'sites_enabled': "%s.conf" % os.path.join(sites_enabled, '0-'+site.unique_name),
             'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name),
             'access_log': site.get_www_access_log_path(),
             'error_log': site.get_www_error_log_path(),

orchestra/apps/websites/directives.py

@@ -18,7 +18,8 @@ class SiteDirective(Plugin):
     SAAS = 'SaaS'
     
     help_text = ""
-    unique = True
+    unique_name = False
+    unique_value = False
     
     @classmethod
     @cached
@@ -67,6 +68,7 @@ class Redirect(SiteDirective):
     help_text = _("<tt>&lt;website path&gt; &lt;destination URL&gt;</tt>")
     regex = r'^[^ ]+\s[^ ]+$'
     group = SiteDirective.HTTPD
+    unique_value = True
 
 
 class Proxy(SiteDirective):
@@ -75,6 +77,7 @@ class Proxy(SiteDirective):
     help_text = _("<tt>&lt;website path&gt; &lt;target URL&gt;</tt>")
     regex = r'^[^ ]+\shttp[^ ]+(timeout=[0-9]{1,3}|retry=[0-9]|\s)*$'
     group = SiteDirective.HTTPD
+    unique_value = True
 
 
 class ErrorDocument(SiteDirective):
@@ -87,6 +90,7 @@ class ErrorDocument(SiteDirective):
                   "<tt>&nbsp;403 \"Sorry can't allow you access today\"</tt>")
     regex = r'[45]0[0-9]\s.*'
     group = SiteDirective.HTTPD
+    unique_value = True
 
 
 class SSLCA(SiteDirective):
@@ -95,6 +99,7 @@ class SSLCA(SiteDirective):
     help_text = _("Filesystem path of the CA certificate file.")
     regex = r'^[^ ]+$'
     group = SiteDirective.SSL
+    unique_name = True
 
 
 class SSLCert(SiteDirective):
@@ -103,6 +108,7 @@ class SSLCert(SiteDirective):
     help_text = _("Filesystem path of the certificate file.")
     regex = r'^[^ ]+$'
     group = SiteDirective.SSL
+    unique_name = True
 
 
 class SSLKey(SiteDirective):
@@ -111,6 +117,7 @@ class SSLKey(SiteDirective):
     help_text = _("Filesystem path of the key file.")
     regex = r'^[^ ]+$'
     group = SiteDirective.SSL
+    unique_name = True
 
 
 class SecRuleRemove(SiteDirective):
@@ -123,34 +130,38 @@ class SecRuleRemove(SiteDirective):
 
 class SecEngine(SiteDirective):
     name = 'sec_engine'
-    verbose_name = _("Modsecurity engine")
+    verbose_name = _("SecRuleEngine Off")
-    help_text = _("URL location for disabling modsecurity engine.")
+    help_text = _("URL path with disabled modsecurity engine.")
     regex = r'^/[^ ]*$'
     group = SiteDirective.SEC
+    unique_value = True
 
 
 class WordPressSaaS(SiteDirective):
     name = 'wordpress-saas'
-    verbose_name = "WordPress"
+    verbose_name = "WordPress SaaS"
-    help_text = _("URL location for mounting wordpress multisite.")
+    help_text = _("URL path for mounting wordpress multisite.")
 #    fpm_listen = settings.WEBAPPS_WORDPRESSMU_LISTEN
     group = SiteDirective.SAAS
     regex = r'^/[^ ]*$'
+    unique_value = True
 
 
 class DokuWikiSaaS(SiteDirective):
     name = 'dokuwiki-saas'
-    verbose_name = "DokuWiki"
+    verbose_name = "DokuWiki SaaS"
-    help_text = _("URL location for mounting wordpress multisite.")
+    help_text = _("URL path for mounting wordpress multisite.")
 #    fpm_listen = settings.WEBAPPS_DOKUWIKIMU_LISTEN
     group = SiteDirective.SAAS
     regex = r'^/[^ ]*$'
+    unique_value = True
 
 
 class DrupalSaaS(SiteDirective):
     name = 'drupal-saas'
-    verbose_name = "Drupdal"
+    verbose_name = "Drupdal SaaS"
-    help_text = _("URL location for mounting wordpress multisite.")
+    help_text = _("URL path for mounting wordpress multisite.")
 #    fpm_listen = settings.WEBAPPS_DRUPALMU_LISTEN
     group = SiteDirective.SAAS
     regex = r'^/[^ ]*$'
+    unique_value = True

orchestra/apps/websites/forms.py

@@ -19,3 +19,26 @@ class WebsiteAdminForm(forms.ModelForm):
                 self.add_error(None, e)
         return self.cleaned_data
 
+
+class WebsiteDirectiveInlineFormSet(forms.models.BaseInlineFormSet):
+    """ Validate uniqueness """
+    def clean(self):
+        values = {}
+        for form in self.forms:
+            name = form.cleaned_data.get('name', None)
+            if name is not None:
+                directive = form.instance.directive_class
+                if directive.unique_name and name in values:
+                    form.add_error(None, ValidationError(
+                        _("Only one %s can be defined.") % directive.get_verbose_name()
+                    ))
+                value = form.cleaned_data.get('value', None)
+                if value is not None:
+                    if directive.unique_value and value in values.get(name, []):
+                        form.add_error('value', ValidationError(
+                            _("This value is already used by other %s.") % unicode(directive.get_verbose_name())
+                        ))
+                try:
+                    values[name].append(value)
+                except KeyError:
+                    values[name] = [value]

orchestra/apps/websites/settings.py

@@ -76,13 +76,21 @@ WEBSITES_TRAFFIC_IGNORE_HOSTS = getattr(settings, 'WEBSITES_TRAFFIC_IGNORE_HOSTS
 #    '')
 
 
-WEBAPPS_SAAS_DIRECTIVES = getattr(settings, 'WEBAPPS_SAAS_DIRECTIVES', {
+WEBSITES_SAAS_DIRECTIVES = getattr(settings, 'WEBSITES_SAAS_DIRECTIVES', {
-    'wordpress-saas': ('fpm', '/home/httpd/wordpress-mu/', '/opt/php/5.4/socks/wordpress-mu.sock'),
+    'wordpress-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock', '/home/httpd/wordpress-mu/'),
-    'drupal-saas': ('fpm', '/home/httpd/drupal-mu/', '/opt/php/5.4/socks/drupal-mu.sock'),
+    'drupal-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock','/home/httpd/drupal-mu/'),
-    'dokuwiki-saas': ('fpm', '/home/httpd/moodle-mu/', '/opt/php/5.4/socks/moodle-mu.sock'),
+    'dokuwiki-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock','/home/httpd/moodle-mu/'),
-#    'moodle-saas': ('fpm', '/home/httpd/moodle-mu/', '/opt/php/5.4/socks/moodle-mu.sock'),
 })
 
 
+WEBSITES_DEFAULT_SSL_CERT = getattr(settings, 'WEBSITES_DEFAULT_SSL_CERT',
+    ''
+)
 
+WEBSITES_DEFAULT_SSL_KEY = getattr(settings, 'WEBSITES_DEFAULT_SSL_KEY',
+    ''
+)
 
+WEBSITES_DEFAULT_SSL_CA = getattr(settings, 'WEBSITES_DEFAULT_SSL_CA',
+    ''
+)

orchestra/forms/options.py

@@ -2,6 +2,8 @@ from django import forms
 from django.contrib.auth import forms as auth_forms
 from django.utils.translation import ugettext, ugettext_lazy as _
 
+from orchestra.utils.python import random_ascii
+
 from ..core.validators import validate_password
 
 
@@ -20,6 +22,10 @@ class UserCreationForm(forms.ModelForm):
         widget=forms.PasswordInput,
         help_text=_("Enter the same password as above, for verification."))
     
+    def __init__(self, *args, **kwargs):
+        super(UserCreationForm, self).__init__(*args, **kwargs)
+        self.fields['password1'].help_text = _("Suggestion: %s") % random_ascii(10)
+    
     def clean_password2(self):
         password1 = self.cleaned_data.get('password1')
         password2 = self.cleaned_data.get('password2')

orchestra/plugins/options.py

@@ -1,3 +1,5 @@
+from django.core.exceptions import ValidationError
+
 from orchestra.utils.functional import cached
 
 
@@ -53,7 +55,7 @@ class Plugin(object):
     
     @classmethod
     def get_change_readonly_fileds(cls):
-        return (cls.plugin_field,) + cls.change_readonly_fileds
+        return cls.change_readonly_fileds
     
     def clean_data(self):
         """ model clean, uses cls.serizlier by default """

orchestra/utils/python.py

@@ -13,7 +13,7 @@ def import_class(cls):
 
 
 def random_ascii(length):
-    return ''.join([random.choice(string.hexdigits) for i in range(0, length)]).lower()
+    return ''.join([random.SystemRandom().choice(string.hexdigits) for i in range(0, length)]).lower()
 
 
 class OrderedSet(collections.MutableSet):