From 4c7c5b550562c91a6ad69fc205ec3f0541e67080 Mon Sep 17 00:00:00 2001 From: Marc Date: Tue, 14 Oct 2014 13:50:19 +0000 Subject: [PATCH] Improvements on databases, webapps and websites --- TODO.md | 9 ++ orchestra/apps/accounts/models.py | 3 +- orchestra/apps/databases/admin.py | 60 ++-------- orchestra/apps/databases/backends.py | 68 ++++++----- orchestra/apps/databases/forms.py | 23 +--- orchestra/apps/databases/models.py | 55 +++++---- orchestra/apps/databases/serializers.py | 41 ++++--- .../databases/tests/functional_tests/tests.py | 44 +++++++- orchestra/apps/orchestration/methods.py | 3 +- orchestra/apps/webapps/backends/__init__.py | 27 +++++ orchestra/apps/webapps/backends/phpfcgid.py | 4 +- orchestra/apps/webapps/backends/phpfpm.py | 53 +++++---- orchestra/apps/webapps/models.py | 17 --- orchestra/apps/webapps/settings.py | 38 +++++-- .../webapps/tests/functional_tests/tests.py | 106 ++++++------------ orchestra/apps/websites/backends/apache.py | 10 +- orchestra/apps/websites/models.py | 13 ++- orchestra/apps/websites/serializers.py | 3 + .../websites/tests/functional_tests/tests.py | 89 ++++++++++++--- orchestra/templates/admin/base_site.html | 1 + scripts/services/apache_full_stack.md | 12 +- 21 files changed, 380 insertions(+), 299 deletions(-) diff --git a/TODO.md b/TODO.md index 64a38af9..ab03b9ad 100644 --- a/TODO.md +++ b/TODO.md @@ -138,3 +138,12 @@ Remember that, as always with QuerySets, any subsequent chained methods which im * DN: Transaction atomicity and backend failure * SaaS Icons + +* offer to create mailbox on account creation + +* init.d celery scripts + -# Required-Start: $network $local_fs $remote_fs postgresql celeryd + -# Required-Stop: $network $local_fs $remote_fs postgresql celeryd + + +* POST only fields (account, username, name) etc diff --git a/orchestra/apps/accounts/models.py b/orchestra/apps/accounts/models.py index c9606b02..446135aa 100644 --- a/orchestra/apps/accounts/models.py +++ b/orchestra/apps/accounts/models.py @@ -72,7 +72,8 @@ class Account(auth.AbstractBaseUser): def disable(self): self.is_active = False -# self.save(update_fields=['is_active']) + self.save(update_fields=['is_active']) + # Trigger save() on related objects that depend on this account for rel in self._meta.get_all_related_objects(): if not rel.model in services: continue diff --git a/orchestra/apps/databases/admin.py b/orchestra/apps/databases/admin.py index 1b11e0cf..0d1eb187 100644 --- a/orchestra/apps/databases/admin.py +++ b/orchestra/apps/databases/admin.py @@ -4,68 +4,24 @@ from django.contrib.auth.admin import UserAdmin from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ -from orchestra.admin import ExtendedModelAdmin +from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin from orchestra.admin.utils import admin_link from orchestra.apps.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin -from .forms import (DatabaseUserChangeForm, DatabaseUserCreationForm, - DatabaseCreationForm) -from .models import Database, Role, DatabaseUser - - -class UserInline(admin.TabularInline): - model = Role - verbose_name_plural = _("Users") - readonly_fields = ('user_link',) - extra = 0 - - user_link = admin_link('user') - - def formfield_for_dbfield(self, db_field, **kwargs): - """ Make value input widget bigger """ - if db_field.name == 'user': - users = db_field.rel.to.objects.filter(type=self.parent_object.type) - kwargs['queryset'] = users.filter(account=self.account) - return super(UserInline, self).formfield_for_dbfield(db_field, **kwargs) - - -class PermissionInline(AccountAdminMixin, admin.TabularInline): - model = Role - verbose_name_plural = _("Permissions") - readonly_fields = ('database_link',) - extra = 0 - filter_by_account_fields = ['database'] - - database_link = admin_link('database', popup=True) - - def formfield_for_dbfield(self, db_field, **kwargs): - """ Make value input widget bigger """ - formfield = super(PermissionInline, self).formfield_for_dbfield(db_field, **kwargs) - if db_field.name == 'database': - # Hack widget render in order to append ?type='db_type' to the add url - db_type = self.parent_object.type - old_render = formfield.widget.render - def render(*args, **kwargs): - output = old_render(*args, **kwargs) - output = output.replace('/add/?', '/add/?type=%s&' % db_type) - return mark_safe(output) - formfield.widget.render = render - formfield.queryset = formfield.queryset.filter(type=db_type) - return formfield +from .forms import DatabaseCreationForm, DatabaseUserChangeForm, DatabaseUserCreationForm +from .models import Database, DatabaseUser class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): list_display = ('name', 'type', 'account_link') list_filter = ('type',) search_fields = ['name'] - inlines = [UserInline] - add_inlines = [] change_readonly_fields = ('name', 'type') extra = 1 fieldsets = ( (None, { 'classes': ('extrapretty',), - 'fields': ('account_link', 'name', 'type'), + 'fields': ('account_link', 'name', 'type', 'users'), }), ) add_fieldsets = ( @@ -92,22 +48,20 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): user = DatabaseUser( username=form.cleaned_data['username'], type=obj.type, - account_id = obj.account.pk, + account_id=obj.account.pk, ) user.set_password(form.cleaned_data["password1"]) user.save() - Role.objects.create(database=obj, user=user, is_owner=True) + obj.users.add(user) -class DatabaseUserAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): +class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, ExtendedModelAdmin): list_display = ('username', 'type', 'account_link') list_filter = ('type',) search_fields = ['username'] form = DatabaseUserChangeForm add_form = DatabaseUserCreationForm change_readonly_fields = ('username', 'type') - inlines = [PermissionInline] - add_inlines = [] fieldsets = ( (None, { 'classes': ('extrapretty',), diff --git a/orchestra/apps/databases/backends.py b/orchestra/apps/databases/backends.py index ee3fe7a7..486c9887 100644 --- a/orchestra/apps/databases/backends.py +++ b/orchestra/apps/databases/backends.py @@ -1,3 +1,5 @@ +import textwrap + from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController @@ -14,12 +16,12 @@ class MySQLBackend(ServiceController): if database.type == database.MYSQL: context = self.get_context(database) self.append( - "mysql -e 'CREATE DATABASE `%(database)s`;'" % context - ) - self.append( - "mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* " - " TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context + "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context ) + self.append(textwrap.dedent("""\ + mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(owner)s"@"%(host)s" WITH GRANT OPTION;' \ + """ % context + )) def delete(self, database): if database.type == database.MYSQL: @@ -44,20 +46,25 @@ class MySQLUserBackend(ServiceController): def save(self, user): if user.type == user.MYSQL: context = self.get_context(user) - self.append( - "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context - ) - self.append( - "mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" " - " WHERE User=\"%(username)s\";'" % context - ) + self.append(textwrap.dedent("""\ + mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \ + """ % context + )) + self.append(textwrap.dedent("""\ + mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \ + """ % context + )) def delete(self, user): if user.type == user.MYSQL: context = self.get_context(database) - self.append( - "mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context - ) + self.append(textwrap.dedent("""\ + mysql -e 'DROP USER "%(username)s"@"%(host)s";' \ + """ % context + )) + + def commit(self): + self.append("mysql -e 'FLUSH PRIVILEGES;'") def get_context(self, user): return { @@ -78,27 +85,28 @@ class MysqlDisk(ServiceMonitor): def exceeded(self, db): context = self.get_context(db) - self.append("mysql -e '" - "UPDATE db SET Insert_priv=\"N\", Create_priv=\"N\"" - " WHERE Db=\"%(db_name)s\";'" % context - ) + self.append(textwrap.dedent("""\ + mysql -e 'UPDATE db SET Insert_priv="N", Create_priv="N" WHERE Db="%(db_name)s";' \ + """ % context + )) def recovery(self, db): context = self.get_context(db) - self.append("mysql -e '" - "UPDATE db SET Insert_priv=\"Y\", Create_priv=\"Y\"" - " WHERE Db=\"%(db_name)s\";'" % context - ) + self.append(textwrap.dedent("""\ + mysql -e 'UPDATE db SET Insert_priv="Y", Create_priv="Y" WHERE Db="%(db_name)s";' \ + """ % context + )) def monitor(self, db): context = self.get_context(db) - self.append( - "echo %(db_id)s $(mysql -B -e '" - " SELECT sum( data_length + index_length ) \"Size\"\n" - " FROM information_schema.TABLES\n" - " WHERE table_schema=\"gisp\"\n" - " GROUP BY table_schema;' | tail -n 1)" % context - ) + self.append(textwrap.dedent("""\ + echo %(db_id)s $(mysql -B -e '" + SELECT sum( data_length + index_length ) "Size" + FROM information_schema.TABLES + WHERE table_schema = "gisp" + GROUP BY table_schema;' | tail -n 1) \ + """ % context + )) def get_context(self, db): return { diff --git a/orchestra/apps/databases/forms.py b/orchestra/apps/databases/forms.py index a048782a..b8717bf8 100644 --- a/orchestra/apps/databases/forms.py +++ b/orchestra/apps/databases/forms.py @@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.core.validators import validate_password -from .models import DatabaseUser, Database, Role +from .models import DatabaseUser, Database class DatabaseUserCreationForm(forms.ModelForm): @@ -27,13 +27,6 @@ class DatabaseUserCreationForm(forms.ModelForm): msg = _("The two password fields didn't match.") raise forms.ValidationError(msg) return password2 - - def save(self, commit=True): - user = super(DatabaseUserCreationForm, self).save(commit=False) -# user.set_password(self.cleaned_data["password1"]) -# if commit: -# user.save() - return user class DatabaseCreationForm(DatabaseUserCreationForm): @@ -86,20 +79,6 @@ class DatabaseCreationForm(DatabaseUserCreationForm): elif not (cleaned_data['username'] or cleaned_data['user']): raise forms.ValidationError(msg) return cleaned_data - - def save(self, commit=True): - db = super(DatabaseUserCreationForm, self).save(commit=False) -# if commit: -# user = self.cleaned_data['user'] -# if not user: -# user = DatabaseUser( -# username=self.cleaned_data['username'], -# type=self.cleaned_data['type'], -# ) -# user.set_password(self.cleaned_data["password1"]) -# user.save() -# role, __ = Role.objects.get_or_create(database=db, user=user) - return db class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField): diff --git a/orchestra/apps/databases/models.py b/orchestra/apps/databases/models.py index f6bfda61..87795102 100644 --- a/orchestra/apps/databases/models.py +++ b/orchestra/apps/databases/models.py @@ -16,8 +16,8 @@ class Database(models.Model): name = models.CharField(_("name"), max_length=64, # MySQL limit validators=[validators.validate_name]) users = models.ManyToManyField('databases.DatabaseUser', - verbose_name=_("users"), - through='databases.Role', related_name='users') + verbose_name=_("users"),related_name='databases') +# through='databases.Role', type = models.CharField(_("type"), max_length=32, choices=settings.DATABASES_TYPE_CHOICES, default=settings.DATABASES_DEFAULT_TYPE) @@ -32,29 +32,38 @@ class Database(models.Model): @property def owner(self): - return self.roles.get(is_owner=True).user + """ database owner is the first user related to it """ + # Accessing intermediary model to get which is the first user + users = Database.users.through.objects.filter(database_id=self.id) + return users.order_by('-id').first().databaseuser -class Role(models.Model): - database = models.ForeignKey(Database, verbose_name=_("database"), - related_name='roles') - user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"), - related_name='roles') - is_owner = models.BooleanField(_("owner"), default=False) - - class Meta: - unique_together = ('database', 'user') - - def __unicode__(self): - return "%s@%s" % (self.user, self.database) - - def clean(self): - if self.user.type != self.database.type: - msg = _("Database and user type doesn't match") - raise validators.ValidationError(msg) - roles = self.database.roles.values('id') - if not roles or (len(roles) == 1 and roles[0].id == self.id): - self.is_owner = True +Database.users.through._meta.unique_together = (('database', 'databaseuser'),) + +#class Role(models.Model): +# database = models.ForeignKey(Database, verbose_name=_("database"), +# related_name='roles') +# user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"), +# related_name='roles') +## is_owner = models.BooleanField(_("owner"), default=False) +# +# class Meta: +# unique_together = ('database', 'user') +# +# def __unicode__(self): +# return "%s@%s" % (self.user, self.database) +# +# @property +# def is_owner(self): +# return datatase.owner == self +# +# def clean(self): +# if self.user.type != self.database.type: +# msg = _("Database and user type doesn't match") +# raise validators.ValidationError(msg) +# roles = self.database.roles.values('id') +# if not roles or (len(roles) == 1 and roles[0].id == self.id): +# self.is_owner = True class DatabaseUser(models.Model): diff --git a/orchestra/apps/databases/serializers.py b/orchestra/apps/databases/serializers.py index 51b51c7f..a9c70dc6 100644 --- a/orchestra/apps/databases/serializers.py +++ b/orchestra/apps/databases/serializers.py @@ -5,36 +5,47 @@ from rest_framework import serializers from orchestra.apps.accounts.serializers import AccountSerializerMixin from orchestra.core.validators import validate_password -from .models import Database, DatabaseUser, Role +from .models import Database, DatabaseUser -class UserRoleSerializer(serializers.HyperlinkedModelSerializer): +class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer): class Meta: - model = Role - fields = ('user', 'is_owner',) - - -class RoleSerializer(serializers.HyperlinkedModelSerializer): - class Meta: - model = Role - fields = ('database', 'is_owner',) + model = DatabaseUser + fields = ('url', 'username') + + def from_native(self, data, files=None): + return DatabaseUser.objects.get(username=data['username']) class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): - roles = UserRoleSerializer(many=True) + users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True) class Meta: model = Database - fields = ('url', 'name', 'type', 'roles') + fields = ('url', 'name', 'type', 'users') + + +class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Database + fields = ('url', 'name',) + + def from_native(self, data, files=None): + return Database.objects.get(name=data['name']) class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): password = serializers.CharField(max_length=128, label=_('Password'), validators=[validate_password], write_only=True, widget=widgets.PasswordInput) - roles = RoleSerializer(many=True, read_only=True) + databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False) class Meta: model = DatabaseUser - fields = ('url', 'username', 'password', 'type', 'roles') - write_only_fields = ('username',) + fields = ('url', 'username', 'password', 'type', 'databases') + + def save_object(self, obj, **kwargs): + # FIXME this method will be called when saving nested serializers :( + if not obj.pk: + obj.set_password(obj.password) + super(DatabaseUserSerializer, self).save_object(obj, **kwargs) diff --git a/orchestra/apps/databases/tests/functional_tests/tests.py b/orchestra/apps/databases/tests/functional_tests/tests.py index d202c3ba..e5b2d68d 100644 --- a/orchestra/apps/databases/tests/functional_tests/tests.py +++ b/orchestra/apps/databases/tests/functional_tests/tests.py @@ -1,5 +1,6 @@ import MySQLdb import os +import socket import time from functools import partial @@ -57,6 +58,17 @@ class DatabaseTestMixin(object): password = '@!?%spppP001' % random_ascii(5) self.add(dbname, username, password) self.validate_create_table(dbname, username, password) + + def test_change_password(self): + dbname = '%s_database' % random_ascii(5) + username = '%s_dbuser' % random_ascii(5) + password = '@!?%spppP001' % random_ascii(5) + self.add(dbname, username, password) + self.validate_create_table(dbname, username, password) + new_password = '@!?%spppP001' % random_ascii(5) + self.change_password(username, new_password) + self.validate_login_error(dbname, username, password) + self.validate_create_table(dbname, username, new_password) class MySQLBackendMixin(object): @@ -64,7 +76,11 @@ class MySQLBackendMixin(object): def setUp(self): super(MySQLBackendMixin, self).setUp() - settings.DATABASES_DEFAULT_HOST = '10.228.207.207' + # Get local ip address used to reach self.MASTER_SERVER + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect((self.MASTER_SERVER, 22)) + settings.DATABASES_DEFAULT_HOST = s.getsockname()[0] + s.close() def add_route(self): server = Server.objects.create(name=self.MASTER_SERVER) @@ -78,7 +94,11 @@ class MySQLBackendMixin(object): def validate_create_table(self, name, username, password): db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name) cur = db.cursor() - cur.execute('CREATE TABLE test ( id INT ) ;') + cur.execute('CREATE TABLE %s ( id INT ) ;' % random_ascii(20)) + + def validate_login_error(self, dbname, username, password): + self.assertRaises(MySQLdb.OperationalError, + self.validate_create_table, dbname, username, password) def validate_delete(self, name, username, password): self.asseRaises(MySQLdb.ConnectionError, @@ -92,9 +112,16 @@ class RESTDatabaseMixin(DatabaseTestMixin): @save_response_on_error def add(self, dbname, username, password): - user = self.rest.databaseusers.create(username=username, password=password) - # TODO fucking nested objects - self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type) + user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type) + users = [{ + 'username': user.username + }] + self.rest.databases.create(name=dbname, users=users, type=self.db_type) + + @save_response_on_error + def change_password(self, username, password): + user = self.rest.databaseusers.retrieve(username=username).get() + user.set_password(password) class AdminDatabaseMixin(DatabaseTestMixin): @@ -129,11 +156,16 @@ class AdminDatabaseMixin(DatabaseTestMixin): def delete(self, dbname): db = Database.objects.get(name=dbname) self.admin_delete(db) - + @snapshot_on_error def delete_user(self, username): user = DatabaseUser.objects.get(username=username) self.admin_delete(user) + + @snapshot_on_error + def change_password(self, username, password): + user = DatabaseUser.objects.get(username=username) + self.admin_change_password(user, password) class RESTMysqlDatabaseTest(MySQLBackendMixin, RESTDatabaseMixin, BaseLiveServerTestCase): diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index 7771a7e5..dd64aa99 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -37,7 +37,8 @@ def BashSSH(backend, log, server, cmds): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) addr = server.get_address() try: - ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH) + # TODO timeout + ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH, timeout=10) except socket.error: logger.error('%s timed out on %s' % (backend, server)) log.state = BackendLog.TIMEOUT diff --git a/orchestra/apps/webapps/backends/__init__.py b/orchestra/apps/webapps/backends/__init__.py index e1077e22..66560d8f 100644 --- a/orchestra/apps/webapps/backends/__init__.py +++ b/orchestra/apps/webapps/backends/__init__.py @@ -1,6 +1,7 @@ import pkgutil import textwrap + class WebAppServiceMixin(object): model = 'webapps.WebApp' @@ -16,6 +17,30 @@ class WebAppServiceMixin(object): done """ % context)) + def get_php_init_vars(self, webapp, per_account=False): + """ + process php options for inclusion on php.ini + per_account=True merges all (account, webapp.type) options + """ + init_vars = [] + options = webapp.options.all() + if per_account: + options = webapp.account.webapps.filter(webapp_type=webapp.type) + for opt in options: + name = opt.name.replace('PHP-', '') + value = "%s" % opt.value + init_vars.append((name, value)) + enabled_functions = [] + for value in options.filter(name='enabled_functions').values_list('value', flat=True): + enabled_functions += enabled_functions.get().value.split(',') + if enabled_functions: + disabled_functions = [] + for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS: + if function not in enabled_functions: + disabled_functions.append(function) + init_vars.append(('dissabled_functions', ','.join(disabled_functions))) + return init_vars + def delete_webapp_dir(self, context): self.append("rm -fr %(app_path)s" % context) @@ -30,5 +55,7 @@ class WebAppServiceMixin(object): } + for __, module_name, __ in pkgutil.walk_packages(__path__): + # sorry for the exec(), but Import module function fails :( exec('from . import %s' % module_name) diff --git a/orchestra/apps/webapps/backends/phpfcgid.py b/orchestra/apps/webapps/backends/phpfcgid.py index 27d45a90..ec69cee4 100644 --- a/orchestra/apps/webapps/backends/phpfcgid.py +++ b/orchestra/apps/webapps/backends/phpfcgid.py @@ -10,6 +10,7 @@ from .. import settings class PHPFcgidBackend(WebAppServiceMixin, ServiceController): + """ Per-webapp fcgid application """ verbose_name = _("PHP-Fcgid") def save(self, webapp): @@ -27,6 +28,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController): def delete(self, webapp): context = self.get_context(webapp) + self.append("rm '%(wrapper_path)s'" % context) self.delete_webapp_dir(context) def commit(self): @@ -35,7 +37,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController): def get_context(self, webapp): context = super(PHPFcgidBackend, self).get_context(webapp) - init_vars = webapp.get_php_init_vars() + init_vars = self.get_php_init_vars(webapp) if init_vars: init_vars = [ '%s="%s"' % (k,v) for v,k in init_vars.iteritems() ] init_vars = ', -d '.join(init_vars) diff --git a/orchestra/apps/webapps/backends/phpfpm.py b/orchestra/apps/webapps/backends/phpfpm.py index ee1cebd6..eb0d5ea2 100644 --- a/orchestra/apps/webapps/backends/phpfpm.py +++ b/orchestra/apps/webapps/backends/phpfpm.py @@ -1,4 +1,5 @@ import os +import textwrap from django.template import Template, Context from django.utils.translation import ugettext_lazy as _ @@ -10,49 +11,57 @@ from .. import settings class PHPFPMBackend(WebAppServiceMixin, ServiceController): + """ Per-webapp php application """ verbose_name = _("PHP-FPM") def save(self, webapp): context = self.get_context(webapp) self.create_webapp_dir(context) - self.append( - "{ echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - ; } ||" - " { echo -e '%(fpm_config)s' > %(fpm_path)s; UPDATEDFPM=1; }" % context - ) + self.append(textwrap.dedent("""\ + { + echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - + } || { + echo -e '%(fpm_config)s' > %(fpm_path)s + UPDATEDFPM=1 + }""" % context)) def delete(self, webapp): context = self.get_context(webapp) + self.append("rm '%(fpm_config)s'" % context) self.delete_webapp_dir(context) def commit(self): super(PHPFPMBackend, self).commit() - self.append('[[ $UPDATEDFPM == 1 ]] && service php5-fpm reload') + self.append(textwrap.dedent(""" + [[ $UPDATEDFPM == 1 ]] && { + service php5-fpm start + service php5-fpm reload + }""")) def get_context(self, webapp): context = super(PHPFPMBackend, self).get_context(webapp) context.update({ - 'init_vars': webapp.get_php_init_vars(), + 'init_vars': self.get_php_init_vars(webapp), 'fpm_port': webapp.get_fpm_port(), }) context['fpm_listen'] = settings.WEBAPPS_FPM_LISTEN % context - fpm_config = Template( - ";; {{ banner }}\n" - "[{{ user }}]\n" - "user = {{ user }}\n" - "group = {{ group }}\n\n" - "listen = {{ fpm_listen | safe }}\n" - "listen.owner = {{ user }}\n" - "listen.group = {{ group }}\n" - "pm = ondemand\n" - "pm.max_children = 4\n" - "{% for name,value in init_vars.iteritems %}" - "php_admin_value[{{ name | safe }}] = {{ value | safe }}\n" - "{% endfor %}" - ) - fpm_file = '%(user)s.conf' % context + fpm_config = Template(textwrap.dedent("""\ + ;; {{ banner }} + [{{ user }}] + user = {{ user }} + group = {{ group }} + + listen = {{ fpm_listen | safe }} + listen.owner = {{ user }} + listen.group = {{ group }} + pm = ondemand + pm.max_children = 4 + {% for name,value in init_vars.iteritems %} + php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}""" + )) context.update({ 'fpm_config': fpm_config.render(Context(context)), - 'fpm_path': os.path.join(settings.WEBAPPS_PHPFPM_POOL_PATH, fpm_file), + 'fpm_path': settings.WEBAPPS_PHPFPM_POOL_PATH % context, }) return context diff --git a/orchestra/apps/webapps/models.py b/orchestra/apps/webapps/models.py index 635e8c01..e8ca6518 100644 --- a/orchestra/apps/webapps/models.py +++ b/orchestra/apps/webapps/models.py @@ -39,23 +39,6 @@ class WebApp(models.Model): def get_options(self): return { opt.name: opt.value for opt in self.options.all() } - def get_php_init_vars(self): - init_vars = [] - options = WebAppOption.objects.filter(webapp__type=self.type) - for opt in options.filter(webapp__account=self.account): - name = opt.name.replace('PHP-', '') - value = "%s" % opt.value - init_vars.append((name, value)) - enabled_functions = self.options.filter(name='enabled_functions') - if enabled_functions: - enabled_functions = enabled_functions.get().value.split(',') - disabled_functions = [] - for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS: - if function not in enabled_functions: - disabled_functions.append(function) - init_vars.append(('dissabled_functions', ','.join(disabled_functions))) - return init_vars - def get_fpm_port(self): return settings.WEBAPPS_FPM_START_PORT + self.account.pk diff --git a/orchestra/apps/webapps/settings.py b/orchestra/apps/webapps/settings.py index 770a1227..25d51d12 100644 --- a/orchestra/apps/webapps/settings.py +++ b/orchestra/apps/webapps/settings.py @@ -9,10 +9,16 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN', # '/var/run/%(user)s-%(app_name)s.sock') '127.0.0.1:%(fpm_port)s') + WEBAPPS_FPM_START_PORT = getattr(settings, 'WEBAPPS_FPM_START_PORT', 10000) + +WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH', + '/etc/php5/fpm/pool.d/%(app_name)s.conf') + + WEBAPPS_FCGID_PATH = getattr(settings, 'WEBAPPS_FCGID_PATH', - '/home/httpd/fcgid/%(user)s/%(type)s-wrapper') + '/home/httpd/fcgid/%(app_name)s-wrapper') WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', { @@ -166,10 +172,26 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', { WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [ - 'exec', 'passthru', 'shell_exec', 'system', 'proc_open', 'popen', 'curl_exec', - 'curl_multi_exec', 'show_source', 'pcntl_exec', 'proc_close', - 'proc_get_status', 'proc_nice', 'proc_terminate', 'ini_alter', 'virtual', - 'openlog', 'escapeshellcmd', 'escapeshellarg', 'dl' + 'exec', + 'passthru', + 'shell_exec', + 'system', + 'proc_open', + 'popen', + 'curl_exec', + 'curl_multi_exec', + 'show_source', + 'pcntl_exec', + 'proc_close', + 'proc_get_status', + 'proc_nice', + 'proc_terminate', + 'ini_alter', + 'virtual', + 'openlog', + 'escapeshellcmd', + 'escapeshellarg', + 'dl' ]) @@ -181,9 +203,6 @@ WEBAPPS_WORDPRESSMU_ADMIN_PASSWORD = getattr(settings, 'WEBAPPS_WORDPRESSMU_ADMI 'secret') - - - WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH = setattr(settings, 'WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH', '/home/httpd/htdocs/wikifarm/template.tar.gz') @@ -195,6 +214,3 @@ WEBAPPS_DOKUWIKIMU_FARM_PATH = getattr(settings, 'WEBAPPS_DOKUWIKIMU_FARM_PATH', WEBAPPS_DRUPAL_SITES_PATH = getattr(settings, 'WEBAPPS_DRUPAL_SITES_PATH', '/home/httpd/htdocs/drupal-mu/sites/%(app_name)s') - -WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH', - '/etc/php5/fpm/pool.d') diff --git a/orchestra/apps/webapps/tests/functional_tests/tests.py b/orchestra/apps/webapps/tests/functional_tests/tests.py index e9d08fcd..f3dd45e6 100644 --- a/orchestra/apps/webapps/tests/functional_tests/tests.py +++ b/orchestra/apps/webapps/tests/functional_tests/tests.py @@ -36,39 +36,14 @@ class WebAppMixin(object): djsettings.DEBUG = True def add_route(self): -# backends = [ -# # TODO MU apps on SaaS? -# backends.awstats.AwstatsBackend, -# backends.dokuwikimu.DokuWikiMuBackend, -# backends.drupalmu.DrupalMuBackend, -# backends.phpfcgid.PHPFcgidBackend, -# backends.phpfpm.PHPFPMBackend, -# backends.static.StaticBackend, -# backends.wordpressmu.WordpressMuBackend, -# ] - server = Server.objects.create(name=self.MASTER_SERVER) - for backend in [SystemUserBackend, self.backend]: - backend = backend.get_name() - Route.objects.create(backend=backend, match=True, host=server) + server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER) + backend = SystemUserBackend.get_name() + Route.objects.get_or_create(backend=backend, match=True, host=server) + backend = self.backend.get_name() + match = 'webapp.type == "%s"' % self.type_value + Route.objects.create(backend=backend, match=match, host=server) - def test_add(self): - name = '%s_%s_webapp' % (random_ascii(10), self.type_value) - self.add_webapp(name) - self.validate_add_webapp(name) -# self.addCleanup(self.delete, username) - - -class StaticWebAppMixin(object): - backend = backends.static.StaticBackend - type_value = 'static' - token = random_ascii(100) - page = ( - 'index.html', - 'Hello World! %s \n' % token, - 'Hello World! %s \n' % token, - ) - - def validate_add_webapp(self, name): + def upload_webapp(self, name): try: ftp = ftplib.FTP(self.MASTER_SERVER) ftp.login(user=self.account.username, passwd=self.account_password) @@ -80,6 +55,23 @@ class StaticWebAppMixin(object): index.close() finally: ftp.close() + + def test_add(self): + name = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(name) + self.addCleanup(self.delete_webapp, name) + self.upload_webapp(name) + + +class StaticWebAppMixin(object): + backend = backends.static.StaticBackend + type_value = 'static' + token = random_ascii(100) + page = ( + 'index.html', + 'Hello World! %s \n' % token, + 'Hello World! %s \n' % token, + ) class PHPFcidWebAppMixin(StaticWebAppMixin): @@ -111,12 +103,11 @@ class RESTWebAppMixin(object): @save_response_on_error def add_webapp(self, name, options=[]): - self.rest.webapps.create(name=name, type=self.type_value) + self.rest.webapps.create(name=name, type=self.type_value, options=options) @save_response_on_error def delete_webapp(self, name): - list = self.rest.lists.retrieve(name=name).get() - list.delete() + self.rest.webapps.retrieve(name=name).delete() class AdminWebAppMixin(WebAppMixin): @@ -125,54 +116,25 @@ class AdminWebAppMixin(WebAppMixin): self.admin_login() # create main user self.save_systemuser() - # TODO save_account() + + @snapshot_on_error + def save_systemuser(self): + url = '' @snapshot_on_error def add(self, name, password, admin_email): - url = self.live_server_url + reverse('admin:mails_List_add') - self.selenium.get(url) - - account_input = self.selenium.find_element_by_id('id_account') - account_select = Select(account_input) - account_select.select_by_value(str(self.account.pk)) - - name_field = self.selenium.find_element_by_id('id_name') - name_field.send_keys(username) - - password_field = self.selenium.find_element_by_id('id_password1') - password_field.send_keys(password) - password_field = self.selenium.find_element_by_id('id_password2') - password_field.send_keys(password) - - if quota is not None: - quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated' - quota_field = self.selenium.find_element_by_id(quota_id) - quota_field.clear() - quota_field.send_keys(quota) - - if filtering is not None: - filtering_input = self.selenium.find_element_by_id('id_filtering') - filtering_select = Select(filtering_input) - filtering_select.select_by_value("CUSTOM") - filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0') - filtering_inline.click() - time.sleep(0.5) - filtering_field = self.selenium.find_element_by_id('id_custom_filtering') - filtering_field.send_keys(filtering) - - name_field.submit() - self.assertNotEqual(url, self.selenium.current_url) + pass -class RESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): +class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): pass -class RESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): +class PHPFcidRESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): pass -class RESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): +class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): pass diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index 007c8cac..be4d494e 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -112,11 +112,11 @@ class Apache2Backend(ServiceController): directives += "SecRuleRemoveById %d" % rule for modsecurity in site.options.filter(name='sec_rule_off'): - directives += ( - "\n" - " SecRuleEngine Off\n" - "\n" % modsecurity.value - ) + directives += textwrap.dedent("""\ + + SecRuleEngine Off + + """ % modsecurity.value) return directives def get_protections(self, site): diff --git a/orchestra/apps/websites/models.py b/orchestra/apps/websites/models.py index 69f70292..76756c72 100644 --- a/orchestra/apps/websites/models.py +++ b/orchestra/apps/websites/models.py @@ -39,7 +39,9 @@ class Website(models.Model): @cached def get_options(self): - return { opt.name: opt.value for opt in self.options.all() } + return { + opt.name: opt.value for opt in self.options.all() + } @property def protocol(self): @@ -81,12 +83,15 @@ class Content(models.Model): class Meta: unique_together = ('website', 'path') + def __unicode__(self): + try: + return self.website.name + self.path + except Website.DoesNotExist: + return self.path + def clean(self): if not self.path.startswith('/'): self.path = '/' + self.path - - def __unicode__(self): - return self.website.name + self.path services.register(Website) diff --git a/orchestra/apps/websites/serializers.py b/orchestra/apps/websites/serializers.py index 58f8c4b6..1b0e79d3 100644 --- a/orchestra/apps/websites/serializers.py +++ b/orchestra/apps/websites/serializers.py @@ -10,6 +10,9 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Content fields = ('webapp', 'path') + + def get_identity(self, data): + return '%s-%s' % (data.get('website'), data.get('path')) class WebsiteSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): diff --git a/orchestra/apps/websites/tests/functional_tests/tests.py b/orchestra/apps/websites/tests/functional_tests/tests.py index 2095acd2..7d3796e5 100644 --- a/orchestra/apps/websites/tests/functional_tests/tests.py +++ b/orchestra/apps/websites/tests/functional_tests/tests.py @@ -38,9 +38,9 @@ class WebsiteMixin(WebAppMixin): super(WebsiteMixin, self).add_route() server = Server.objects.get() backend = backends.apache.Apache2Backend.get_name() - Route.objects.create(backend=backend, match=True, host=server) + Route.objects.get_or_create(backend=backend, match=True, host=server) backend = Bind9MasterDomainBackend.get_name() - Route.objects.create(backend=backend, match=True, host=server) + Route.objects.get_or_create(backend=backend, match=True, host=server) def validate_add_website(self, name, domain): url = 'http://%s/%s' % (domain.name, self.page[0]) @@ -54,9 +54,11 @@ class WebsiteMixin(WebAppMixin): self.save_domain(domain) webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) self.add_webapp(webapp) - self.validate_add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) website = '%s_website' % random_ascii(10) self.add_website(website, domain, webapp) + self.addCleanup(self.delete_website, website) self.validate_add_website(website, domain) @@ -65,21 +67,82 @@ class RESTWebsiteMixin(RESTWebAppMixin): def save_domain(self, domain): self.rest.domains.retrieve().get().save() - def add_website(self, name, domain, webapp): - domain = self.rest.domains.retrieve().get() - webapp = self.rest.webapps.retrieve().get() - self.rest.websites.create(name=name, domains=[domain.url], contents=[{'webapp': webapp.url}]) + @save_response_on_error + def add_website(self, name, domain, webapp, path='/'): + domain = self.rest.domains.retrieve(name=domain).get() + webapp = self.rest.webapps.retrieve(name=webapp).get() + contents = [{ + 'webapp': webapp.url, + 'path': path + }] + self.rest.websites.create(name=name, domains=[domain.url], contents=contents) + + @save_response_on_error + def delete_website(self, name): + print 'hola' + pass + self.rest.websites.retrieve(name=name).delete() +# self.rest.websites.retrieve(name=name).delete() + + @save_response_on_error + def add_content(self, website, webapp, path): + website = self.rest.websites.retrieve(name=website).get() + webapp = self.rest.webapps.retrieve(name=webapp).get() + website.contents.append({ + 'webapp': webapp.url, + 'path': path, + }) + website.save() -class RESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): +class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): + def test_mix_webapps(self): + domain_name = '%sdomain.lan' % random_ascii(10) + domain = Domain.objects.create(name=domain_name, account=self.account) + domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR) + self.save_domain(domain) + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + website = '%s_website' % random_ascii(10) + self.add_website(website, domain, webapp) + self.addCleanup(self.delete_website, website) + self.validate_add_website(website, domain) + + self.type_value = PHPFcidWebAppMixin.type_value + self.backend = PHPFcidWebAppMixin.backend + self.page = PHPFcidWebAppMixin.page + self.add_route() + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + path = '/%s' % webapp + self.add_content(website, webapp, path) + url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) + self.assertEqual(self.page[2], requests.get(url).content) + + self.type_value = PHPFPMWebAppMixin.type_value + self.backend = PHPFPMWebAppMixin.backend + self.page = PHPFPMWebAppMixin.page + self.add_route() + webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) + self.add_webapp(webapp) + self.addCleanup(self.delete_webapp, webapp) + self.upload_webapp(webapp) + path = '/%s' % webapp + + self.add_content(website, webapp, path) + url = 'http://%s%s/%s' % (domain.name, path, self.page[0]) + self.assertEqual(self.page[2], requests.get(url).content) + + +class PHPFcidRESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): pass -class RESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): - pass - - -class RESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): +class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): pass #class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase): diff --git a/orchestra/templates/admin/base_site.html b/orchestra/templates/admin/base_site.html index 6d4cd4e8..c19028d0 100644 --- a/orchestra/templates/admin/base_site.html +++ b/orchestra/templates/admin/base_site.html @@ -27,6 +27,7 @@ {% if not is_popup %} {% admin_tools_render_menu %} {% endif %} + {% endif %} {% endblock %} diff --git a/scripts/services/apache_full_stack.md b/scripts/services/apache_full_stack.md index 57cb3b0a..6a477862 100644 --- a/scripts/services/apache_full_stack.md +++ b/scripts/services/apache_full_stack.md @@ -84,6 +84,12 @@ The goal of this setup is having a high-performance state-of-the-art deployment + +# TODO pool per website or pool per user? memory consumption +events.mechanism = epoll +# TODO multiple master processes, opcache is held in master, and reload/restart affects all pools +# http://mattiasgeniar.be/2014/04/09/a-better-way-to-run-php-fpm/ + TODO CHRoot https://andrewbevitt.com/tutorials/apache-varnish-chrooted-php-fpm-wordpress-virtual-host/ @@ -92,10 +98,10 @@ TODO CHRoot [vhost] istemplate = 1 listen.mode = 0660 + pm = ondemand pm.max_children = 5 - pm.start_servers = 1 - pm.min_spare_servers = 1 - pm.max_spare_servers = 2 + pm.process_idle_timeout = 10s + pm.max_requests = 200 ' > /etc/php5/fpm/conf.d/vhost-template.conf ```