From dd84217320f94aed1387402037d64a27f7e34053 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Wed, 25 Mar 2015 15:45:04 +0000 Subject: [PATCH] Lots of improvements on webapps and saas --- TODO.md | 19 +++- orchestra/apps/databases/backends.py | 3 +- orchestra/apps/domains/settings.py | 2 - orchestra/apps/domains/validators.py | 7 +- orchestra/apps/orchestration/backends.py | 5 +- orchestra/apps/orchestration/manager.py | 43 +++++---- orchestra/apps/resources/helpers.py | 1 + orchestra/apps/saas/admin.py | 15 +-- orchestra/apps/saas/backends/gitlab.py | 101 +++++++++++++++++++++ orchestra/apps/saas/backends/phplist.py | 4 +- orchestra/apps/saas/models.py | 14 ++- orchestra/apps/saas/services/bscw.py | 8 +- orchestra/apps/saas/services/gitlab.py | 46 +++++++++- orchestra/apps/saas/services/options.py | 43 ++++----- orchestra/apps/saas/services/phplist.py | 33 ++++--- orchestra/apps/saas/settings.py | 13 +++ orchestra/apps/webapps/backends/php.py | 18 ++-- orchestra/apps/webapps/settings.py | 1 + orchestra/apps/webapps/types/__init__.py | 1 + orchestra/apps/webapps/types/misc.py | 4 +- orchestra/apps/webapps/types/php.py | 26 +++--- orchestra/apps/websites/admin.py | 3 +- orchestra/apps/websites/backends/apache.py | 52 ++++++----- orchestra/apps/websites/directives.py | 29 ++++-- orchestra/apps/websites/forms.py | 23 +++++ orchestra/apps/websites/settings.py | 18 +++- orchestra/forms/options.py | 6 ++ orchestra/plugins/options.py | 4 +- orchestra/utils/python.py | 2 +- 29 files changed, 390 insertions(+), 154 deletions(-) create mode 100644 orchestra/apps/saas/backends/gitlab.py diff --git a/TODO.md b/TODO.md index f1fc95e5..6dbe10f0 100644 --- a/TODO.md +++ b/TODO.md @@ -241,12 +241,25 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * WPMU blog traffic -* normurlpath '' returns '/' +* normurlpath '' return '/' * rename webapps.type to something more generic * initial configuration of multisite sas apps with password stored in DATA -* websites links on webpaps ans saas +* webapps installation complete, passowrd protected +* saas.initial_password autogenerated (ok because its random and not user provided) vs saas.password /change_Form provided + send email with initial_password + +* disable saas apps + +* more robust backend error handling, continue executing but exit code > 0 if failure, replace exit_code=0; do_sometging || exit_code=1 + +* saas require unique emails? connect to backend server to find out because they change + +* automaitcally set passwords and email users? + +* website directives uniquenes validation on serializers + +* gitlab store id, username changes + -* /var/lib/fcgid/wrappers/ rm write permissions diff --git a/orchestra/apps/databases/backends.py b/orchestra/apps/databases/backends.py index eca453ae..171701bd 100644 --- a/orchestra/apps/databases/backends.py +++ b/orchestra/apps/databases/backends.py @@ -38,10 +38,11 @@ class MySQLBackend(ServiceController): if database.type != database.MYSQL: return context = self.get_context(database) - self.append("mysql -e 'DROP DATABASE `%(database)s`;'" % context) + self.append("mysql -e 'DROP DATABASE `%(database)s`;' || exit_code=1" % context) self.append("mysql mysql -e 'DELETE FROM db WHERE db = \"%(database)s\";'" % context) def commit(self): + super(MySQLBackend, self).commit() self.append("mysql -e 'FLUSH PRIVILEGES;'") def get_context(self, database): diff --git a/orchestra/apps/domains/settings.py b/orchestra/apps/domains/settings.py index 6293e6a4..766a7a1d 100644 --- a/orchestra/apps/domains/settings.py +++ b/orchestra/apps/domains/settings.py @@ -36,8 +36,6 @@ DOMAINS_SLAVES_PATH = getattr(settings, 'DOMAINS_SLAVES_PATH', '/etc/bind/named. DOMAINS_CHECKZONE_BIN_PATH = getattr(settings, 'DOMAINS_CHECKZONE_BIN_PATH', '/usr/sbin/named-checkzone -i local -k fail -n fail') -DOMAINS_CHECKZONE_PATH = getattr(settings, 'DOMAINS_CHECKZONE_PATH', '/dev/shm') - DOMAINS_DEFAULT_A = getattr(settings, 'DOMAINS_DEFAULT_A', '10.0.3.13') diff --git a/orchestra/apps/domains/validators.py b/orchestra/apps/domains/validators.py index e5d366a9..e9897d8e 100644 --- a/orchestra/apps/domains/validators.py +++ b/orchestra/apps/domains/validators.py @@ -108,11 +108,10 @@ def validate_soa_record(value): def validate_zone(zone): """ Ultimate zone file validation using named-checkzone """ zone_name = zone.split()[0][:-1] - path = os.path.join(settings.DOMAINS_CHECKZONE_PATH, zone_name) - with open(path, 'wb') as f: - f.write(zone) checkzone = settings.DOMAINS_CHECKZONE_BIN_PATH - check = run(' '.join([checkzone, zone_name, path]), error_codes=[0,1], display=False) + cmd = ' '.join(["echo -e '%s'" % zone, '|', checkzone, zone_name, '/dev/stdin']) + print cmd + check = run(cmd, error_codes=[0, 1], display=False) if check.return_code == 1: errors = re.compile(r'zone.*: (.*)').findall(check.stdout)[:-1] raise ValidationError(', '.join(errors)) diff --git a/orchestra/apps/orchestration/backends.py b/orchestra/apps/orchestration/backends.py index 46a070a6..7797e397 100644 --- a/orchestra/apps/orchestration/backends.py +++ b/orchestra/apps/orchestration/backends.py @@ -177,7 +177,8 @@ class ServiceBackend(plugins.Plugin): """ self.append( 'set -e\n' - 'set -o pipefail' + 'set -o pipefail\n' + 'exit_code=0;' ) def commit(self): @@ -187,7 +188,7 @@ class ServiceBackend(plugins.Plugin): reloading a service is done in a separated method in order to reload the service once in bulk operations """ - self.append('exit 0') + self.append('exit $exit_code') class ServiceController(ServiceBackend): diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index 66694792..bb53da4e 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -22,18 +22,10 @@ def as_task(execute): def wrapper(*args, **kwargs): """ send report """ # Tasks run on a separate transaction pool (thread), no need to temper with the transaction - log = execute(*args, **kwargs) - if log.state != log.SUCCESS: - send_report(execute, args, log) - return log - return wrapper - - -def close_connection(execute): - """ Threads have their own connection pool, closing it when finishing """ - def wrapper(*args, **kwargs): try: log = execute(*args, **kwargs) + if log.state != log.SUCCESS: + send_report(execute, args, log) except Exception as e: subject = 'EXCEPTION executing backend(s) %s %s' % (str(args), str(kwargs)) message = traceback.format_exc() @@ -45,6 +37,19 @@ def close_connection(execute): # Using the wrapper function as threader messenger for the execute output # Absense of it will indicate a failure at this stage wrapper.log = log + return log + return wrapper + + +def close_connection(execute): + """ Threads have their own connection pool, closing it when finishing """ + def wrapper(*args, **kwargs): + try: + log = execute(*args, **kwargs) + except: + pass + else: + wrapper.log = log finally: db.connection.close() return wrapper @@ -89,15 +94,15 @@ def execute(operations, async=False): backend, operations = value backend.commit() execute = as_task(backend.execute) - execute = close_connection(execute) - # DEBUG: substitute all thread related stuff for this function - #execute(server, async=async) logger.debug('%s is going to be executed on %s' % (backend, server)) - thread = threading.Thread(target=execute, args=(server,), kwargs={'async': async}) - thread.start() if block: - thread.join() - threads.append(thread) + # Execute one bakend at a time, no need for threads + execute(server, async=async) + else: + execute = close_connection(execute) + thread = threading.Thread(target=execute, args=(server,), kwargs={'async': async}) + thread.start() + threads.append(thread) executions.append((execute, operations)) [ thread.join() for thread in threads ] logs = [] @@ -108,7 +113,9 @@ def execute(operations, async=False): for operation in operations: logger.info("Executed %s" % str(operation)) operation.log = execution.log - operation.save() + if operation.object_id: + # Not all backends are call with objects saved on the database + operation.save() stdout = execution.log.stdout.strip() stdout and logger.debug('STDOUT %s', stdout) stderr = execution.log.stderr.strip() diff --git a/orchestra/apps/resources/helpers.py b/orchestra/apps/resources/helpers.py index 6cd67d10..59523821 100644 --- a/orchestra/apps/resources/helpers.py +++ b/orchestra/apps/resources/helpers.py @@ -6,6 +6,7 @@ def compute_resource_usage(data): resource = data.resource result = 0 has_result = False + today = datetime.date.today() for dataset in data.get_monitor_datasets(): if resource.period == resource.MONTHLY_AVG: last = dataset.latest() diff --git a/orchestra/apps/saas/admin.py b/orchestra/apps/saas/admin.py index 197fd39e..ff09b933 100644 --- a/orchestra/apps/saas/admin.py +++ b/orchestra/apps/saas/admin.py @@ -11,18 +11,19 @@ from .services import SoftwareService class SaaSAdmin(SelectPluginAdminMixin, AccountAdminMixin, ExtendedModelAdmin): - list_display = ('username', 'service', 'display_site_name', 'account_link') + list_display = ('name', 'service', 'display_site_domain', 'account_link') list_filter = ('service',) + change_readonly_fields = ('service',) plugin = SoftwareService plugin_field = 'service' plugin_title = 'Software as a Service' - def display_site_name(self, saas): - site_name = saas.get_site_name() - return '%s' % (site_name, site_name) - display_site_name.short_description = _("Site name") - display_site_name.allow_tags = True - display_site_name.admin_order_field = 'site_name' + def display_site_domain(self, saas): + site_domain = saas.get_site_domain() + return '%s' % (site_domain, site_domain) + display_site_domain.short_description = _("Site domain") + display_site_domain.allow_tags = True + display_site_domain.admin_order_field = 'name' admin.site.register(SaaS, SaaSAdmin) diff --git a/orchestra/apps/saas/backends/gitlab.py b/orchestra/apps/saas/backends/gitlab.py new file mode 100644 index 00000000..ee7232b7 --- /dev/null +++ b/orchestra/apps/saas/backends/gitlab.py @@ -0,0 +1,101 @@ +import json + +import requests +from django.utils.translation import ugettext_lazy as _ + +from orchestra.apps.orchestration import ServiceController + +from .. import settings + + +class GitLabSaaSBackend(ServiceController): + verbose_name = _("GitLab SaaS") + model = 'saas.SaaS' + default_route_match = "saas.service == 'gitlab'" + block = True + actions = ('save', 'delete', 'validate_creation') + + def get_base_url(self): + return 'https://%s/api/v3' % settings.SAAS_GITLAB_DOMAIN + + def get_user_url(self, saas): + user_id = saas.data['user_id'] + return self.get_base_url() + '/users/%i' % user_id + + def validate_response(self, response, status_codes): + if response.status_code not in status_codes: + raise RuntimeError("[%i] %s" % (response.status_code, response.content)) + + def authenticate(self): + login_url = self.get_base_url() + '/session' + data = { + 'login': 'root', + 'password': settings.SAAS_GITLAB_ROOT_PASSWORD, + } + response = requests.post(login_url, data=data) + self.validate_response(response, [201]) + token = json.loads(response.content)['private_token'] + self.headers = { + 'PRIVATE-TOKEN': token, + } + + def create_user(self, saas, server): + self.authenticate() + user_url = self.get_base_url() + '/users' + data = { + 'email': saas.data['email'], + 'password': saas.password, + 'username': saas.name, + 'name': saas.account.get_full_name(), + } + response = requests.post(user_url, data=data, headers=self.headers) + self.validate_response(response, [201]) + print response.content + user = json.loads(response.content) + saas.data['user_id'] = user['id'] + # Using queryset update to avoid triggering backends with the post_save signal + type(saas).objects.filter(pk=saas.pk).update(data=saas.data) + print json.dumps(user, indent=4) + + def change_password(self, saas, server): + self.authenticate() + user_url = self.get_user_url(saas) + data = { + 'password': saas.password, + } + response = requests.patch(user_url, data=data, headers=self.headers) + self.validate_response(response, [200]) + print json.dumps(json.loads(response.content), indent=4) + + def delete_user(self, saas, server): + self.authenticate() + user_url = self.get_user_url(saas) + response = requests.delete(user_url, headers=self.headers) + self.validate_response(response, [200, 404]) + print json.dumps(json.loads(response.content), indent=4) + + def _validate_creation(self, saas, server): + """ checks if a saas object is valid for creation on the server side """ + self.authenticate() + username = saas.name + email = saas.data['email'] + users_url = self.get_base_url() + '/users/' + users = json.loads(requests.get(users_url, headers=self.headers).content) + for user in users: + if user['username'] == username: + print 'user-exists' + if user['email'] == email: + print 'email-exists' + + def validate_creation(self, saas): + self.append(self._validate_creation, saas) + + def save(self, saas): + if hasattr(saas, 'password'): + if saas.data.get('user_id', None): + self.append(self.change_password, saas) + else: + self.append(self.create_user, saas) + + def delete(self, saas): + self.append(self.delete_user, saas) diff --git a/orchestra/apps/saas/backends/phplist.py b/orchestra/apps/saas/backends/phplist.py index 49bc6d49..bfc242db 100644 --- a/orchestra/apps/saas/backends/phplist.py +++ b/orchestra/apps/saas/backends/phplist.py @@ -17,7 +17,7 @@ class PhpListSaaSBackend(ServiceController): def initialize_database(self, saas, server): base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN - admin_link = 'http://%s.%s/admin/' % (saas.get_site_name(), base_domain) + admin_link = 'http://%s/admin/' % saas.get_site_domain() admin_content = requests.get(admin_link).content if admin_content.startswith('Cannot connect to Database'): raise RuntimeError("Database is not yet configured") @@ -28,7 +28,7 @@ class PhpListSaaSBackend(ServiceController): install = install.groups()[0] install_link = admin_link + install[1:] post = { - 'adminname': saas.username, + 'adminname': saas.name, 'orgname': saas.account.username, 'adminemail': saas.account.username, 'adminpassword': saas.password, diff --git a/orchestra/apps/saas/models.py b/orchestra/apps/saas/models.py index d764bdef..6b33b767 100644 --- a/orchestra/apps/saas/models.py +++ b/orchestra/apps/saas/models.py @@ -14,10 +14,9 @@ from .services import SoftwareService class SaaS(models.Model): service = models.CharField(_("service"), max_length=32, choices=SoftwareService.get_plugin_choices()) - username = models.CharField(_("name"), max_length=64, + name = models.CharField(_("Name"), max_length=64, help_text=_("Required. 64 characters or fewer. Letters, digits and ./-/_ only."), validators=[validators.validate_username]) -# site_name = NullableCharField(_("site name"), max_length=32, null=True) account = models.ForeignKey('accounts.Account', verbose_name=_("account"), related_name='saas') data = JSONField(_("data"), default={}, @@ -27,12 +26,11 @@ class SaaS(models.Model): verbose_name = "SaaS" verbose_name_plural = "SaaS" unique_together = ( - ('username', 'service'), -# ('site_name', 'service'), + ('name', 'service'), ) def __unicode__(self): - return "%s@%s" % (self.username, self.service) + return "%s@%s" % (self.name, self.service) @cached_property def service_class(self): @@ -43,12 +41,12 @@ class SaaS(models.Model): """ Per request lived service_instance """ return self.service_class(self) - def get_site_name(self): - return self.service_instance.get_site_name() - def clean(self): self.data = self.service_instance.clean_data() + def get_site_domain(self): + return self.service_instance.get_site_domain() + def set_password(self, password): self.password = password diff --git a/orchestra/apps/saas/services/bscw.py b/orchestra/apps/saas/services/bscw.py index cc462aae..ecf6abf9 100644 --- a/orchestra/apps/saas/services/bscw.py +++ b/orchestra/apps/saas/services/bscw.py @@ -11,12 +11,14 @@ from .options import SoftwareService, SoftwareServiceForm class BSCWForm(SoftwareServiceForm): email = forms.EmailField(label=_("Email"), widget=forms.TextInput(attrs={'size':'40'})) - quota = forms.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB.")) + quota = forms.IntegerField(label=_("Quota"), initial=settings.SAAS_BSCW_DEFAULT_QUOTA, + help_text=_("Disk quota in MB.")) class BSCWDataSerializer(serializers.Serializer): email = serializers.EmailField(label=_("Email")) - quota = serializers.IntegerField(label=_("Quota"), help_text=_("Disk quota in MB.")) + quota = serializers.IntegerField(label=_("Quota"), default=settings.SAAS_BSCW_DEFAULT_QUOTA, + help_text=_("Disk quota in MB.")) class BSCWService(SoftwareService): @@ -26,5 +28,5 @@ class BSCWService(SoftwareService): serializer = BSCWDataSerializer icon = 'orchestra/icons/apps/BSCW.png' # TODO override from settings - site_name = settings.SAAS_BSCW_DOMAIN + site_domain = settings.SAAS_BSCW_DOMAIN change_readonly_fileds = ('email',) diff --git a/orchestra/apps/saas/services/gitlab.py b/orchestra/apps/saas/services/gitlab.py index 250b4cff..a3bf6a54 100644 --- a/orchestra/apps/saas/services/gitlab.py +++ b/orchestra/apps/saas/services/gitlab.py @@ -1,6 +1,50 @@ -from .options import SoftwareService +from django import forms +from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from orchestra.apps.orchestration.models import BackendOperation as Operation +from orchestra.forms import widgets + +from .options import SoftwareService, SoftwareServiceForm + +from .. import settings + + +class GitLabForm(SoftwareServiceForm): + email = forms.EmailField(label=_("Email"), + help_text=_("Initial email address, changes on the GitLab server are not reflected here.")) + + +class GitLaChangebForm(GitLabForm): + user_id = forms.IntegerField(label=("User ID"), widget=widgets.ShowTextWidget, + help_text=_("ID of this user on the GitLab server, the only attribute that not changes.")) + + +class GitLabSerializer(serializers.Serializer): + email = serializers.EmailField(label=_("Email")) + user_id = serializers.IntegerField(label=_("User ID"), required=False) class GitLabService(SoftwareService): + name = 'gitlab' + form = GitLabForm + change_form = GitLaChangebForm + serializer = GitLabSerializer + site_domain = settings.SAAS_GITLAB_DOMAIN + change_readonly_fileds = ('email', 'user_id',) verbose_name = "GitLab" icon = 'orchestra/icons/apps/gitlab.png' + + def clean_data(self): + data = super(GitLabService, self).clean_data() + if not self.instance.pk: + log = Operation.execute_action(self.instance, 'validate_creation')[0] + errors = {} + if 'user-exists' in log.stdout: + errors['name'] = _("User with this username already exists.") + elif 'email-exists' in log.stdout: + errors['email'] = _("User with this email address already exists.") + if errors: + raise ValidationError(errors) + return data diff --git a/orchestra/apps/saas/services/options.py b/orchestra/apps/saas/services/options.py index a192b83c..0cce80a5 100644 --- a/orchestra/apps/saas/services/options.py +++ b/orchestra/apps/saas/services/options.py @@ -8,13 +8,13 @@ from orchestra.plugins.forms import PluginDataForm from orchestra.core import validators from orchestra.forms import widgets from orchestra.utils.functional import cached -from orchestra.utils.python import import_class +from orchestra.utils.python import import_class, random_ascii from .. import settings class SoftwareServiceForm(PluginDataForm): - site_name = forms.CharField(widget=widgets.ShowTextWidget, required=False) + site_url = forms.CharField(label=_("Site URL"), widget=widgets.ShowTextWidget, required=False) password = forms.CharField(label=_("Password"), required=False, widget=widgets.ReadOnlyWidget('Unknown password'), help_text=_("Passwords are not stored, so there is no way to see this " @@ -30,25 +30,21 @@ class SoftwareServiceForm(PluginDataForm): super(SoftwareServiceForm, self).__init__(*args, **kwargs) self.is_change = bool(self.instance and self.instance.pk) if self.is_change: - site_name = self.instance.get_site_name() + site_domain = self.instance.get_site_domain() self.fields['password1'].required = False self.fields['password1'].widget = forms.HiddenInput() self.fields['password2'].required = False self.fields['password2'].widget = forms.HiddenInput() else: self.fields['password'].widget = forms.HiddenInput() - site_name = self.plugin.site_name - if site_name: - site_name_link = '%s' % (site_name, site_name) + self.fields['password1'].help_text = _("Suggestion: %s") % random_ascii(10) + site_domain = self.plugin.site_domain + if site_domain: + site_link = '%s' % (site_domain, site_domain) else: - site_name_link = '<name>.%s' % self.plugin.site_name_base_domain - self.fields['site_name'].initial = site_name_link -## self.fields['site_name'].widget = widgets.ReadOnlyWidget(site_name, mark_safe(link)) -## self.fields['site_name'].required = False -# else: -# base_name = self.plugin.site_name_base_domain -# help_text = _("The final URL would be <site_name>.%s") % base_name -# self.fields['site_name'].help_text = help_text + site_link = '<site_name>.%s' % self.plugin.site_base_domain + self.fields['site_url'].initial = site_link + self.fields['name'].label = _("Username") def clean_password2(self): if not self.is_change: @@ -59,11 +55,6 @@ class SoftwareServiceForm(PluginDataForm): raise forms.ValidationError(msg) return password2 - def clean_site_name(self): - if self.plugin.site_name: - return None - return self.cleaned_data['site_name'] - def save(self, commit=True): obj = super(SoftwareServiceForm, self).save(commit=commit) if not self.is_change: @@ -73,11 +64,10 @@ class SoftwareServiceForm(PluginDataForm): class SoftwareService(plugins.Plugin): form = SoftwareServiceForm - site_name = None - site_name_base_domain = 'orchestra.lan' + site_domain = None + site_base_domain = None has_custom_domain = False icon = 'orchestra/icons/apps.png' - change_readonly_fileds = ('site_name',) class_verbose_name = _("Software as a Service") plugin_field = 'service' @@ -89,14 +79,13 @@ class SoftwareService(plugins.Plugin): plugins.append(import_class(cls)) return plugins - @classmethod def get_change_readonly_fileds(cls): fields = super(SoftwareService, cls).get_change_readonly_fileds() - return fields + ('username',) + return fields + ('name',) - def get_site_name(self): - return self.site_name or '.'.join( - (self.instance.username, self.site_name_base_domain) + def get_site_domain(self): + return self.site_domain or '.'.join( + (self.instance.name, self.site_base_domain) ) def save(self): diff --git a/orchestra/apps/saas/services/phplist.py b/orchestra/apps/saas/services/phplist.py index 9ed80994..03e71ef4 100644 --- a/orchestra/apps/saas/services/phplist.py +++ b/orchestra/apps/saas/services/phplist.py @@ -17,23 +17,22 @@ class PHPListForm(SoftwareServiceForm): def __init__(self, *args, **kwargs): super(PHPListForm, self).__init__(*args, **kwargs) - self.fields['username'].label = _("Name") - base_domain = self.plugin.site_name_base_domain - help_text = _("Admin URL http://<name>.{}/admin/").format(base_domain) - self.fields['site_name'].help_text = help_text + self.fields['name'].label = _("Site name") + base_domain = self.plugin.site_base_domain + help_text = _("Admin URL http://<site_name>.{}/admin/").format(base_domain) + self.fields['site_url'].help_text = help_text class PHPListChangeForm(PHPListForm): -# site_name = forms.CharField(widget=widgets.ShowTextWidget, required=False) db_name = forms.CharField(label=_("Database name"), help_text=_("Database used for this webapp.")) def __init__(self, *args, **kwargs): super(PHPListChangeForm, self).__init__(*args, **kwargs) - site_name = self.instance.get_site_name() - admin_url = "http://%s/admin/" % site_name + site_domain = self.instance.get_site_domain() + admin_url = "http://%s/admin/" % site_domain help_text = _("Admin URL {0}").format(admin_url) - self.fields['site_name'].help_text = help_text + self.fields['site_url'].help_text = help_text class PHPListSerializer(serializers.Serializer): @@ -48,21 +47,25 @@ class PHPListService(SoftwareService): change_readonly_fileds = ('db_name',) serializer = PHPListSerializer icon = 'orchestra/icons/apps/Phplist.png' - site_name_base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN + site_base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN def get_db_name(self): - db_name = 'phplist_mu_%s' % self.instance.username + db_name = 'phplist_mu_%s' % self.instance.name # Limit for mysql database names return db_name[:65] def get_db_user(self): return settings.SAAS_PHPLIST_DB_NAME + def get_account(self): + return type(self.instance.account).get_main() + def validate(self): super(PHPListService, self).validate() create = not self.instance.pk if create: - db = Database(name=self.get_db_name(), account=self.instance.account) + account = self.get_account() + db = Database(name=self.get_db_name(), account=account) try: db.full_clean() except ValidationError as e: @@ -73,7 +76,8 @@ class PHPListService(SoftwareService): def save(self): db_name = self.get_db_name() db_user = self.get_db_user() - db, db_created = Database.objects.get_or_create(name=db_name, account=self.instance.account) + account = self.get_account() + db, db_created = account.databases.get_or_create(name=db_name) user = DatabaseUser.objects.get(username=db_user) db.users.add(user) self.instance.data = { @@ -90,9 +94,10 @@ class PHPListService(SoftwareService): def get_related(self): related = [] - account = self.instance.account + account = self.get_account() + db_name = self.instance.data.get('db_name') try: - db = account.databases.get(name=self.instance.data.get('db_name')) + db = account.databases.get(name=db_name) except Database.DoesNotExist: pass else: diff --git a/orchestra/apps/saas/settings.py b/orchestra/apps/saas/settings.py index 971ff0f0..02081b0d 100644 --- a/orchestra/apps/saas/settings.py +++ b/orchestra/apps/saas/settings.py @@ -47,3 +47,16 @@ SAAS_BSCW_DOMAIN = getattr(settings, 'SAAS_BSCW_DOMAIN', ) +SAAS_BSCW_DEFAULT_QUOTA = getattr(settings, 'SAAS_BSCW_DEFAULT_QUOTA', + 50 +) + + +SAAS_GITLAB_ROOT_PASSWORD = getattr(settings, 'SAAS_GITLAB_ROOT_PASSWORD', + 'secret' +) + +SAAS_GITLAB_DOMAIN = getattr(settings, 'SAAS_GITLAB_DOMAIN', + 'gitlab.orchestra.lan' +) + diff --git a/orchestra/apps/webapps/backends/php.py b/orchestra/apps/webapps/backends/php.py index 33ede752..9cdbe5e6 100644 --- a/orchestra/apps/webapps/backends/php.py +++ b/orchestra/apps/webapps/backends/php.py @@ -28,10 +28,11 @@ class PHPBackend(WebAppServiceMixin, ServiceController): self.create_webapp_dir(context) self.set_under_construction(context) self.append(textwrap.dedent("""\ + fpm_config='%(fpm_config)s' { - echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - + echo -e "${fpm_config}" | diff -N -I'^\s*;;' %(fpm_path)s - } || { - echo -e '%(fpm_config)s' > %(fpm_path)s + echo -e "${fpm_config}" > %(fpm_path)s UPDATEDFPM=1 }""") % context ) @@ -41,20 +42,23 @@ class PHPBackend(WebAppServiceMixin, ServiceController): self.set_under_construction(context) self.append("mkdir -p %(wrapper_dir)s" % context) self.append(textwrap.dedent("""\ + wrapper='%(wrapper)s' { - echo -e '%(wrapper)s' | diff -N -I'^\s*#' %(wrapper_path)s - + echo -e "${wrapper}" | diff -N -I'^\s*#' %(wrapper_path)s - } || { - echo -e '%(wrapper)s' > %(wrapper_path)s; UPDATED_APACHE=1 + echo -e "${wrapper}" > %(wrapper_path)s; UPDATED_APACHE=1 }""") % context ) - self.append("chmod +x %(wrapper_path)s" % context) + self.append("chmod 550 %(wrapper_dir)s" % context) + self.append("chmod 550 %(wrapper_path)s" % context) self.append("chown -R %(user)s:%(group)s %(wrapper_dir)s" % context) if context['cmd_options']: self.append(textwrap.dedent(""" + cmd_options='%(cmd_options)s' { - echo -e '%(cmd_options)s' | diff -N -I'^\s*#' %(cmd_options_path)s - + echo -e "${cmd_options}" | diff -N -I'^\s*#' %(cmd_options_path)s - } || { - echo -e '%(cmd_options)s' > %(cmd_options_path)s; UPDATED_APACHE=1 + echo -e "${cmd_options}" > %(cmd_options_path)s; UPDATED_APACHE=1 }""" ) % context ) else: diff --git a/orchestra/apps/webapps/settings.py b/orchestra/apps/webapps/settings.py index f7dc8d72..daeb101b 100644 --- a/orchestra/apps/webapps/settings.py +++ b/orchestra/apps/webapps/settings.py @@ -16,6 +16,7 @@ WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH', WEBAPPS_FCGID_WRAPPER_PATH = getattr(settings, 'WEBAPPS_FCGID_WRAPPER_PATH', + # Inside SuExec Document root '/home/httpd/fcgi-bin.d/%(user)s/%(app_name)s-wrapper') diff --git a/orchestra/apps/webapps/types/__init__.py b/orchestra/apps/webapps/types/__init__.py index cd78848b..3e179b7e 100644 --- a/orchestra/apps/webapps/types/__init__.py +++ b/orchestra/apps/webapps/types/__init__.py @@ -89,5 +89,6 @@ class AppType(plugins.Plugin): 'app_id': self.instance.id, 'app_name': self.instance.name, 'user': self.instance.account.username, + 'home': self.instance.account.main_systemuser.get_home(), } diff --git a/orchestra/apps/webapps/types/misc.py b/orchestra/apps/webapps/types/misc.py index 93b555be..edff2685 100644 --- a/orchestra/apps/webapps/types/misc.py +++ b/orchestra/apps/webapps/types/misc.py @@ -33,8 +33,8 @@ class WebalizerApp(AppType): icon = 'orchestra/icons/apps/Stats.png' option_groups = () - def get_directive(self, webapp): - webalizer_path = os.path.join(webapp.get_path(), '%(site_name)s') + def get_directive(self): + webalizer_path = os.path.join(self.instance.get_path(), '%(site_name)s') webalizer_path = os.path.normpath(webalizer_path) return ('static', webalizer_path) diff --git a/orchestra/apps/webapps/types/php.py b/orchestra/apps/webapps/types/php.py index 9edd647b..aed56a80 100644 --- a/orchestra/apps/webapps/types/php.py +++ b/orchestra/apps/webapps/types/php.py @@ -57,15 +57,6 @@ class PHPApp(AppType): def get_detail(self): return self.instance.data.get('php_version', '') - def get_context(self): - """ context used to format settings """ - return { - 'home': self.instance.account.main_systemuser.get_home(), - 'account': self.instance.account.username, - 'user': self.instance.account.username, - 'app_name': self.instance.name, - } - def get_php_init_vars(self, merge=False): """ process php options for inclusion on php.ini @@ -77,17 +68,17 @@ class PHPApp(AppType): # Get options from the same account and php_version webapps options = [] php_version = self.get_php_version() - webapps = self.instance.account.webapps.filter(webapp_type=self.instance.type) + webapps = self.instance.account.webapps.filter(type=self.instance.type) for webapp in webapps: if webapp.type_instance.get_php_version == php_version: options += list(webapp.options.all()) php_options = [option.name for option in type(self).get_php_options()] + enabled_functions = set() for opt in options: if opt.name in php_options: init_vars[opt.name] = opt.value - enabled_functions = [] - for value in options.filter(name='enabled_functions').values_list('value', flat=True): - enabled_functions += enabled_functions.get().value.split(',') + elif opt.name == 'enabled_functions': + enabled_functions.union(set(opt.value.split(','))) if enabled_functions: disabled_functions = [] for function in self.PHP_DISABLED_FUNCTIONS: @@ -95,11 +86,18 @@ class PHPApp(AppType): disabled_functions.append(function) init_vars['dissabled_functions'] = ','.join(disabled_functions) if self.PHP_ERROR_LOG_PATH and 'error_log' not in init_vars: - context = self.get_context() + context = self.get_directive_context() error_log_path = os.path.normpath(self.PHP_ERROR_LOG_PATH % context) init_vars['error_log'] = error_log_path return init_vars + def get_directive_context(self): + context = super(PHPApp, self).get_directive_context() + context.update({ + 'php_version': self.get_php_version(), + }) + return context + def get_directive(self): context = self.get_directive_context() if self.is_fpm: diff --git a/orchestra/apps/websites/admin.py b/orchestra/apps/websites/admin.py index 3a2874e8..a4500553 100644 --- a/orchestra/apps/websites/admin.py +++ b/orchestra/apps/websites/admin.py @@ -12,12 +12,13 @@ from orchestra.forms.widgets import DynamicHelpTextSelect from . import settings from .directives import SiteDirective -from .forms import WebsiteAdminForm +from .forms import WebsiteAdminForm, WebsiteDirectiveInlineFormSet from .models import Content, Website, WebsiteDirective class WebsiteDirectiveInline(admin.TabularInline): model = WebsiteDirective + formset = WebsiteDirectiveInlineFormSet extra = 1 DIRECTIVES_HELP_TEXT = { diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index af4a48d4..ce43301e 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -31,6 +31,7 @@ class Apache2Backend(ServiceController): extra_conf += self.get_security(directives) extra_conf += self.get_redirects(directives) extra_conf += self.get_proxies(directives) + extra_conf += self.get_saas(directives) # Order extra conf directives based on directives (longer first) extra_conf = sorted(extra_conf, key=lambda a: len(a[0]), reverse=True) context['extra_conf'] = '\n'.join([conf for location, conf in extra_conf]) @@ -46,7 +47,7 @@ class Apache2Backend(ServiceController): SuexecUserGroup {{ user }} {{ group }}\ {% for line in extra_conf.splitlines %} {{ line | safe }}{% endfor %} - #IncludeOptional /etc/apache2/extra-vhos[t]/{{ site_unique_name }}.con[f] + IncludeOptional /etc/apache2/extra-vhos[t]/{{ site_unique_name }}.con[f] """) ).render(Context(context)) @@ -80,10 +81,11 @@ class Apache2Backend(ServiceController): apache_conf += self.render_redirect_https(context) context['apache_conf'] = apache_conf self.append(textwrap.dedent("""\ + apache_conf='%(apache_conf)s' { - echo -e '%(apache_conf)s' | diff -N -I'^\s*#' %(sites_available)s - + echo -e "${apache_conf}" | diff -N -I'^\s*#' %(sites_available)s - } || { - echo -e '%(apache_conf)s' > %(sites_available)s + echo -e "${apache_conf}" > %(sites_available)s UPDATED=1 }""") % context ) @@ -116,7 +118,7 @@ class Apache2Backend(ServiceController): return directives def get_static_directives(self, context, app_path): - context['app_path'] = app_path % context + context['app_path'] = os.path.normpath(app_path % context) location = "%(location)s/" % context directive = "Alias %(location)s/ %(app_path)s/" % context return [(location, directive)] @@ -128,10 +130,10 @@ class Apache2Backend(ServiceController): else: # UNIX socket target = 'unix:%(socket)s|fcgi://127.0.0.1%(app_path)s/' - if context['location'] != '/': + if context['location']: target = 'unix:%(socket)s|fcgi://127.0.0.1%(app_path)s/$1' context.update({ - 'app_path': app_path, + 'app_path': os.path.normpath(app_path), 'socket': socket, }) location = "%(location)s/" % context @@ -143,7 +145,7 @@ class Apache2Backend(ServiceController): def get_fcgid_directives(self, context, app_path, wrapper_path): context.update({ - 'app_path': app_path, + 'app_path': os.path.normpath(app_path), 'wrapper_path': wrapper_path, }) location = "%(location)s/" % context @@ -158,16 +160,20 @@ class Apache2Backend(ServiceController): return [(location, directives)] def get_ssl(self, directives): - config = '' - ca = directives.get('ssl_ca') - if ca: - config += "SSLCACertificateFile %s\n" % ca[0] cert = directives.get('ssl_cert') - if cert: - config += "SSLCertificateFile %\n" % cert[0] key = directives.get('ssl_key') - if key: - config += "SSLCertificateKeyFile %s\n" % key[0] + ca = directives.get('ssl_ca') + if not (cert and key): + cert = [settings.WEBSITES_DEFAULT_SSL_CERT] + key = [settings.WEBSITES_DEFAULT_SSL_KEY] + ca = [settings.WEBSITES_DEFAULT_SSL_CA] + if not (cert and key): + return [] + config = 'SSLEngine on\n' + config += "SSLCertificateFile %s\n" % cert[0] + config += "SSLCertificateKeyFile %s\n" % key[0] + if ca: + config += "SSLCACertificateFile %s\n" % ca[0] return [('', config)] def get_security(self, directives): @@ -210,13 +216,14 @@ class Apache2Backend(ServiceController): def get_saas(self, directives): saas = [] - for name, value in directives.iteritems(): + for name, values in directives.iteritems(): if name.endswith('-saas'): - context = { - 'location': normurlpath(value), - } - directive = settings.WEBSITES_SAAS_DIRECTIVES[name] - saas += self.get_directive(context, directive) + for value in values: + context = { + 'location': normurlpath(value), + } + directive = settings.WEBSITES_SAAS_DIRECTIVES[name] + saas += self.get_directives(directive, context) return saas # def get_protections(self, site): # protections = '' @@ -280,7 +287,8 @@ class Apache2Backend(ServiceController): 'site_unique_name': site.unique_name, 'user': self.get_username(site), 'group': self.get_groupname(site), - 'sites_enabled': "%s.conf" % os.path.join(sites_enabled, site.unique_name), + # TODO remove '0-' + 'sites_enabled': "%s.conf" % os.path.join(sites_enabled, '0-'+site.unique_name), 'sites_available': "%s.conf" % os.path.join(sites_available, site.unique_name), 'access_log': site.get_www_access_log_path(), 'error_log': site.get_www_error_log_path(), diff --git a/orchestra/apps/websites/directives.py b/orchestra/apps/websites/directives.py index 2c9cf9b4..da6273e6 100644 --- a/orchestra/apps/websites/directives.py +++ b/orchestra/apps/websites/directives.py @@ -18,7 +18,8 @@ class SiteDirective(Plugin): SAAS = 'SaaS' help_text = "" - unique = True + unique_name = False + unique_value = False @classmethod @cached @@ -67,6 +68,7 @@ class Redirect(SiteDirective): help_text = _("<website path> <destination URL>") regex = r'^[^ ]+\s[^ ]+$' group = SiteDirective.HTTPD + unique_value = True class Proxy(SiteDirective): @@ -75,6 +77,7 @@ class Proxy(SiteDirective): help_text = _("<website path> <target URL>") regex = r'^[^ ]+\shttp[^ ]+(timeout=[0-9]{1,3}|retry=[0-9]|\s)*$' group = SiteDirective.HTTPD + unique_value = True class ErrorDocument(SiteDirective): @@ -87,6 +90,7 @@ class ErrorDocument(SiteDirective): " 403 \"Sorry can't allow you access today\"") regex = r'[45]0[0-9]\s.*' group = SiteDirective.HTTPD + unique_value = True class SSLCA(SiteDirective): @@ -95,6 +99,7 @@ class SSLCA(SiteDirective): help_text = _("Filesystem path of the CA certificate file.") regex = r'^[^ ]+$' group = SiteDirective.SSL + unique_name = True class SSLCert(SiteDirective): @@ -103,6 +108,7 @@ class SSLCert(SiteDirective): help_text = _("Filesystem path of the certificate file.") regex = r'^[^ ]+$' group = SiteDirective.SSL + unique_name = True class SSLKey(SiteDirective): @@ -111,6 +117,7 @@ class SSLKey(SiteDirective): help_text = _("Filesystem path of the key file.") regex = r'^[^ ]+$' group = SiteDirective.SSL + unique_name = True class SecRuleRemove(SiteDirective): @@ -123,34 +130,38 @@ class SecRuleRemove(SiteDirective): class SecEngine(SiteDirective): name = 'sec_engine' - verbose_name = _("Modsecurity engine") - help_text = _("URL location for disabling modsecurity engine.") + verbose_name = _("SecRuleEngine Off") + help_text = _("URL path with disabled modsecurity engine.") regex = r'^/[^ ]*$' group = SiteDirective.SEC + unique_value = True class WordPressSaaS(SiteDirective): name = 'wordpress-saas' - verbose_name = "WordPress" - help_text = _("URL location for mounting wordpress multisite.") + verbose_name = "WordPress SaaS" + help_text = _("URL path for mounting wordpress multisite.") # fpm_listen = settings.WEBAPPS_WORDPRESSMU_LISTEN group = SiteDirective.SAAS regex = r'^/[^ ]*$' + unique_value = True class DokuWikiSaaS(SiteDirective): name = 'dokuwiki-saas' - verbose_name = "DokuWiki" - help_text = _("URL location for mounting wordpress multisite.") + verbose_name = "DokuWiki SaaS" + help_text = _("URL path for mounting wordpress multisite.") # fpm_listen = settings.WEBAPPS_DOKUWIKIMU_LISTEN group = SiteDirective.SAAS regex = r'^/[^ ]*$' + unique_value = True class DrupalSaaS(SiteDirective): name = 'drupal-saas' - verbose_name = "Drupdal" - help_text = _("URL location for mounting wordpress multisite.") + verbose_name = "Drupdal SaaS" + help_text = _("URL path for mounting wordpress multisite.") # fpm_listen = settings.WEBAPPS_DRUPALMU_LISTEN group = SiteDirective.SAAS regex = r'^/[^ ]*$' + unique_value = True diff --git a/orchestra/apps/websites/forms.py b/orchestra/apps/websites/forms.py index 9ee3f3fd..ee0599c8 100644 --- a/orchestra/apps/websites/forms.py +++ b/orchestra/apps/websites/forms.py @@ -19,3 +19,26 @@ class WebsiteAdminForm(forms.ModelForm): self.add_error(None, e) return self.cleaned_data + +class WebsiteDirectiveInlineFormSet(forms.models.BaseInlineFormSet): + """ Validate uniqueness """ + def clean(self): + values = {} + for form in self.forms: + name = form.cleaned_data.get('name', None) + if name is not None: + directive = form.instance.directive_class + if directive.unique_name and name in values: + form.add_error(None, ValidationError( + _("Only one %s can be defined.") % directive.get_verbose_name() + )) + value = form.cleaned_data.get('value', None) + if value is not None: + if directive.unique_value and value in values.get(name, []): + form.add_error('value', ValidationError( + _("This value is already used by other %s.") % unicode(directive.get_verbose_name()) + )) + try: + values[name].append(value) + except KeyError: + values[name] = [value] diff --git a/orchestra/apps/websites/settings.py b/orchestra/apps/websites/settings.py index 6af8cf79..88afb601 100644 --- a/orchestra/apps/websites/settings.py +++ b/orchestra/apps/websites/settings.py @@ -76,13 +76,21 @@ WEBSITES_TRAFFIC_IGNORE_HOSTS = getattr(settings, 'WEBSITES_TRAFFIC_IGNORE_HOSTS # '') -WEBAPPS_SAAS_DIRECTIVES = getattr(settings, 'WEBAPPS_SAAS_DIRECTIVES', { - 'wordpress-saas': ('fpm', '/home/httpd/wordpress-mu/', '/opt/php/5.4/socks/wordpress-mu.sock'), - 'drupal-saas': ('fpm', '/home/httpd/drupal-mu/', '/opt/php/5.4/socks/drupal-mu.sock'), - 'dokuwiki-saas': ('fpm', '/home/httpd/moodle-mu/', '/opt/php/5.4/socks/moodle-mu.sock'), -# 'moodle-saas': ('fpm', '/home/httpd/moodle-mu/', '/opt/php/5.4/socks/moodle-mu.sock'), +WEBSITES_SAAS_DIRECTIVES = getattr(settings, 'WEBSITES_SAAS_DIRECTIVES', { + 'wordpress-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock', '/home/httpd/wordpress-mu/'), + 'drupal-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock','/home/httpd/drupal-mu/'), + 'dokuwiki-saas': ('fpm', '/opt/php/5.4/socks/pangea.sock','/home/httpd/moodle-mu/'), }) +WEBSITES_DEFAULT_SSL_CERT = getattr(settings, 'WEBSITES_DEFAULT_SSL_CERT', + '' +) +WEBSITES_DEFAULT_SSL_KEY = getattr(settings, 'WEBSITES_DEFAULT_SSL_KEY', + '' +) +WEBSITES_DEFAULT_SSL_CA = getattr(settings, 'WEBSITES_DEFAULT_SSL_CA', + '' +) diff --git a/orchestra/forms/options.py b/orchestra/forms/options.py index b1b707d0..feb880e2 100644 --- a/orchestra/forms/options.py +++ b/orchestra/forms/options.py @@ -2,6 +2,8 @@ from django import forms from django.contrib.auth import forms as auth_forms from django.utils.translation import ugettext, ugettext_lazy as _ +from orchestra.utils.python import random_ascii + from ..core.validators import validate_password @@ -20,6 +22,10 @@ class UserCreationForm(forms.ModelForm): widget=forms.PasswordInput, help_text=_("Enter the same password as above, for verification.")) + def __init__(self, *args, **kwargs): + super(UserCreationForm, self).__init__(*args, **kwargs) + self.fields['password1'].help_text = _("Suggestion: %s") % random_ascii(10) + def clean_password2(self): password1 = self.cleaned_data.get('password1') password2 = self.cleaned_data.get('password2') diff --git a/orchestra/plugins/options.py b/orchestra/plugins/options.py index ef0b0cd2..82cf4d1f 100644 --- a/orchestra/plugins/options.py +++ b/orchestra/plugins/options.py @@ -1,3 +1,5 @@ +from django.core.exceptions import ValidationError + from orchestra.utils.functional import cached @@ -53,7 +55,7 @@ class Plugin(object): @classmethod def get_change_readonly_fileds(cls): - return (cls.plugin_field,) + cls.change_readonly_fileds + return cls.change_readonly_fileds def clean_data(self): """ model clean, uses cls.serizlier by default """ diff --git a/orchestra/utils/python.py b/orchestra/utils/python.py index c5dc153b..fa5e37eb 100644 --- a/orchestra/utils/python.py +++ b/orchestra/utils/python.py @@ -13,7 +13,7 @@ def import_class(cls): def random_ascii(length): - return ''.join([random.choice(string.hexdigits) for i in range(0, length)]).lower() + return ''.join([random.SystemRandom().choice(string.hexdigits) for i in range(0, length)]).lower() class OrderedSet(collections.MutableSet):