diff --git a/TODO.md b/TODO.md
index 64a38af9..ab03b9ad 100644
--- a/TODO.md
+++ b/TODO.md
@@ -138,3 +138,12 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* DN: Transaction atomicity and backend failure
* SaaS Icons
+
+* offer to create mailbox on account creation
+
+* init.d celery scripts
+ -# Required-Start: $network $local_fs $remote_fs postgresql celeryd
+ -# Required-Stop: $network $local_fs $remote_fs postgresql celeryd
+
+
+* POST only fields (account, username, name) etc
diff --git a/orchestra/apps/accounts/models.py b/orchestra/apps/accounts/models.py
index c9606b02..446135aa 100644
--- a/orchestra/apps/accounts/models.py
+++ b/orchestra/apps/accounts/models.py
@@ -72,7 +72,8 @@ class Account(auth.AbstractBaseUser):
def disable(self):
self.is_active = False
-# self.save(update_fields=['is_active'])
+ self.save(update_fields=['is_active'])
+ # Trigger save() on related objects that depend on this account
for rel in self._meta.get_all_related_objects():
if not rel.model in services:
continue
diff --git a/orchestra/apps/databases/admin.py b/orchestra/apps/databases/admin.py
index 1b11e0cf..0d1eb187 100644
--- a/orchestra/apps/databases/admin.py
+++ b/orchestra/apps/databases/admin.py
@@ -4,68 +4,24 @@ from django.contrib.auth.admin import UserAdmin
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
-from orchestra.admin import ExtendedModelAdmin
+from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
from orchestra.admin.utils import admin_link
from orchestra.apps.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin
-from .forms import (DatabaseUserChangeForm, DatabaseUserCreationForm,
- DatabaseCreationForm)
-from .models import Database, Role, DatabaseUser
-
-
-class UserInline(admin.TabularInline):
- model = Role
- verbose_name_plural = _("Users")
- readonly_fields = ('user_link',)
- extra = 0
-
- user_link = admin_link('user')
-
- def formfield_for_dbfield(self, db_field, **kwargs):
- """ Make value input widget bigger """
- if db_field.name == 'user':
- users = db_field.rel.to.objects.filter(type=self.parent_object.type)
- kwargs['queryset'] = users.filter(account=self.account)
- return super(UserInline, self).formfield_for_dbfield(db_field, **kwargs)
-
-
-class PermissionInline(AccountAdminMixin, admin.TabularInline):
- model = Role
- verbose_name_plural = _("Permissions")
- readonly_fields = ('database_link',)
- extra = 0
- filter_by_account_fields = ['database']
-
- database_link = admin_link('database', popup=True)
-
- def formfield_for_dbfield(self, db_field, **kwargs):
- """ Make value input widget bigger """
- formfield = super(PermissionInline, self).formfield_for_dbfield(db_field, **kwargs)
- if db_field.name == 'database':
- # Hack widget render in order to append ?type='db_type' to the add url
- db_type = self.parent_object.type
- old_render = formfield.widget.render
- def render(*args, **kwargs):
- output = old_render(*args, **kwargs)
- output = output.replace('/add/?', '/add/?type=%s&' % db_type)
- return mark_safe(output)
- formfield.widget.render = render
- formfield.queryset = formfield.queryset.filter(type=db_type)
- return formfield
+from .forms import DatabaseCreationForm, DatabaseUserChangeForm, DatabaseUserCreationForm
+from .models import Database, DatabaseUser
class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
list_display = ('name', 'type', 'account_link')
list_filter = ('type',)
search_fields = ['name']
- inlines = [UserInline]
- add_inlines = []
change_readonly_fields = ('name', 'type')
extra = 1
fieldsets = (
(None, {
'classes': ('extrapretty',),
- 'fields': ('account_link', 'name', 'type'),
+ 'fields': ('account_link', 'name', 'type', 'users'),
}),
)
add_fieldsets = (
@@ -92,22 +48,20 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
user = DatabaseUser(
username=form.cleaned_data['username'],
type=obj.type,
- account_id = obj.account.pk,
+ account_id=obj.account.pk,
)
user.set_password(form.cleaned_data["password1"])
user.save()
- Role.objects.create(database=obj, user=user, is_owner=True)
+ obj.users.add(user)
-class DatabaseUserAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
+class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, ExtendedModelAdmin):
list_display = ('username', 'type', 'account_link')
list_filter = ('type',)
search_fields = ['username']
form = DatabaseUserChangeForm
add_form = DatabaseUserCreationForm
change_readonly_fields = ('username', 'type')
- inlines = [PermissionInline]
- add_inlines = []
fieldsets = (
(None, {
'classes': ('extrapretty',),
diff --git a/orchestra/apps/databases/backends.py b/orchestra/apps/databases/backends.py
index ee3fe7a7..486c9887 100644
--- a/orchestra/apps/databases/backends.py
+++ b/orchestra/apps/databases/backends.py
@@ -1,3 +1,5 @@
+import textwrap
+
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
@@ -14,12 +16,12 @@ class MySQLBackend(ServiceController):
if database.type == database.MYSQL:
context = self.get_context(database)
self.append(
- "mysql -e 'CREATE DATABASE `%(database)s`;'" % context
- )
- self.append(
- "mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
- " TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context
+ "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
)
+ self.append(textwrap.dedent("""\
+ mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(owner)s"@"%(host)s" WITH GRANT OPTION;' \
+ """ % context
+ ))
def delete(self, database):
if database.type == database.MYSQL:
@@ -44,20 +46,25 @@ class MySQLUserBackend(ServiceController):
def save(self, user):
if user.type == user.MYSQL:
context = self.get_context(user)
- self.append(
- "mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context
- )
- self.append(
- "mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
- " WHERE User=\"%(username)s\";'" % context
- )
+ self.append(textwrap.dedent("""\
+ mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \
+ """ % context
+ ))
+ self.append(textwrap.dedent("""\
+ mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \
+ """ % context
+ ))
def delete(self, user):
if user.type == user.MYSQL:
context = self.get_context(database)
- self.append(
- "mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
- )
+ self.append(textwrap.dedent("""\
+ mysql -e 'DROP USER "%(username)s"@"%(host)s";' \
+ """ % context
+ ))
+
+ def commit(self):
+ self.append("mysql -e 'FLUSH PRIVILEGES;'")
def get_context(self, user):
return {
@@ -78,27 +85,28 @@ class MysqlDisk(ServiceMonitor):
def exceeded(self, db):
context = self.get_context(db)
- self.append("mysql -e '"
- "UPDATE db SET Insert_priv=\"N\", Create_priv=\"N\""
- " WHERE Db=\"%(db_name)s\";'" % context
- )
+ self.append(textwrap.dedent("""\
+ mysql -e 'UPDATE db SET Insert_priv="N", Create_priv="N" WHERE Db="%(db_name)s";' \
+ """ % context
+ ))
def recovery(self, db):
context = self.get_context(db)
- self.append("mysql -e '"
- "UPDATE db SET Insert_priv=\"Y\", Create_priv=\"Y\""
- " WHERE Db=\"%(db_name)s\";'" % context
- )
+ self.append(textwrap.dedent("""\
+ mysql -e 'UPDATE db SET Insert_priv="Y", Create_priv="Y" WHERE Db="%(db_name)s";' \
+ """ % context
+ ))
def monitor(self, db):
context = self.get_context(db)
- self.append(
- "echo %(db_id)s $(mysql -B -e '"
- " SELECT sum( data_length + index_length ) \"Size\"\n"
- " FROM information_schema.TABLES\n"
- " WHERE table_schema=\"gisp\"\n"
- " GROUP BY table_schema;' | tail -n 1)" % context
- )
+ self.append(textwrap.dedent("""\
+ echo %(db_id)s $(mysql -B -e '"
+ SELECT sum( data_length + index_length ) "Size"
+ FROM information_schema.TABLES
+ WHERE table_schema = "gisp"
+ GROUP BY table_schema;' | tail -n 1) \
+ """ % context
+ ))
def get_context(self, db):
return {
diff --git a/orchestra/apps/databases/forms.py b/orchestra/apps/databases/forms.py
index a048782a..b8717bf8 100644
--- a/orchestra/apps/databases/forms.py
+++ b/orchestra/apps/databases/forms.py
@@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.core.validators import validate_password
-from .models import DatabaseUser, Database, Role
+from .models import DatabaseUser, Database
class DatabaseUserCreationForm(forms.ModelForm):
@@ -27,13 +27,6 @@ class DatabaseUserCreationForm(forms.ModelForm):
msg = _("The two password fields didn't match.")
raise forms.ValidationError(msg)
return password2
-
- def save(self, commit=True):
- user = super(DatabaseUserCreationForm, self).save(commit=False)
-# user.set_password(self.cleaned_data["password1"])
-# if commit:
-# user.save()
- return user
class DatabaseCreationForm(DatabaseUserCreationForm):
@@ -86,20 +79,6 @@ class DatabaseCreationForm(DatabaseUserCreationForm):
elif not (cleaned_data['username'] or cleaned_data['user']):
raise forms.ValidationError(msg)
return cleaned_data
-
- def save(self, commit=True):
- db = super(DatabaseUserCreationForm, self).save(commit=False)
-# if commit:
-# user = self.cleaned_data['user']
-# if not user:
-# user = DatabaseUser(
-# username=self.cleaned_data['username'],
-# type=self.cleaned_data['type'],
-# )
-# user.set_password(self.cleaned_data["password1"])
-# user.save()
-# role, __ = Role.objects.get_or_create(database=db, user=user)
- return db
class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField):
diff --git a/orchestra/apps/databases/models.py b/orchestra/apps/databases/models.py
index f6bfda61..87795102 100644
--- a/orchestra/apps/databases/models.py
+++ b/orchestra/apps/databases/models.py
@@ -16,8 +16,8 @@ class Database(models.Model):
name = models.CharField(_("name"), max_length=64, # MySQL limit
validators=[validators.validate_name])
users = models.ManyToManyField('databases.DatabaseUser',
- verbose_name=_("users"),
- through='databases.Role', related_name='users')
+ verbose_name=_("users"),related_name='databases')
+# through='databases.Role',
type = models.CharField(_("type"), max_length=32,
choices=settings.DATABASES_TYPE_CHOICES,
default=settings.DATABASES_DEFAULT_TYPE)
@@ -32,29 +32,38 @@ class Database(models.Model):
@property
def owner(self):
- return self.roles.get(is_owner=True).user
+ """ database owner is the first user related to it """
+ # Accessing intermediary model to get which is the first user
+ users = Database.users.through.objects.filter(database_id=self.id)
+ return users.order_by('-id').first().databaseuser
-class Role(models.Model):
- database = models.ForeignKey(Database, verbose_name=_("database"),
- related_name='roles')
- user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
- related_name='roles')
- is_owner = models.BooleanField(_("owner"), default=False)
-
- class Meta:
- unique_together = ('database', 'user')
-
- def __unicode__(self):
- return "%s@%s" % (self.user, self.database)
-
- def clean(self):
- if self.user.type != self.database.type:
- msg = _("Database and user type doesn't match")
- raise validators.ValidationError(msg)
- roles = self.database.roles.values('id')
- if not roles or (len(roles) == 1 and roles[0].id == self.id):
- self.is_owner = True
+Database.users.through._meta.unique_together = (('database', 'databaseuser'),)
+
+#class Role(models.Model):
+# database = models.ForeignKey(Database, verbose_name=_("database"),
+# related_name='roles')
+# user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
+# related_name='roles')
+## is_owner = models.BooleanField(_("owner"), default=False)
+#
+# class Meta:
+# unique_together = ('database', 'user')
+#
+# def __unicode__(self):
+# return "%s@%s" % (self.user, self.database)
+#
+# @property
+# def is_owner(self):
+# return datatase.owner == self
+#
+# def clean(self):
+# if self.user.type != self.database.type:
+# msg = _("Database and user type doesn't match")
+# raise validators.ValidationError(msg)
+# roles = self.database.roles.values('id')
+# if not roles or (len(roles) == 1 and roles[0].id == self.id):
+# self.is_owner = True
class DatabaseUser(models.Model):
diff --git a/orchestra/apps/databases/serializers.py b/orchestra/apps/databases/serializers.py
index 51b51c7f..a9c70dc6 100644
--- a/orchestra/apps/databases/serializers.py
+++ b/orchestra/apps/databases/serializers.py
@@ -5,36 +5,47 @@ from rest_framework import serializers
from orchestra.apps.accounts.serializers import AccountSerializerMixin
from orchestra.core.validators import validate_password
-from .models import Database, DatabaseUser, Role
+from .models import Database, DatabaseUser
-class UserRoleSerializer(serializers.HyperlinkedModelSerializer):
+class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
- model = Role
- fields = ('user', 'is_owner',)
-
-
-class RoleSerializer(serializers.HyperlinkedModelSerializer):
- class Meta:
- model = Role
- fields = ('database', 'is_owner',)
+ model = DatabaseUser
+ fields = ('url', 'username')
+
+ def from_native(self, data, files=None):
+ return DatabaseUser.objects.get(username=data['username'])
class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
- roles = UserRoleSerializer(many=True)
+ users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True)
class Meta:
model = Database
- fields = ('url', 'name', 'type', 'roles')
+ fields = ('url', 'name', 'type', 'users')
+
+
+class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer):
+ class Meta:
+ model = Database
+ fields = ('url', 'name',)
+
+ def from_native(self, data, files=None):
+ return Database.objects.get(name=data['name'])
class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
password = serializers.CharField(max_length=128, label=_('Password'),
validators=[validate_password], write_only=True,
widget=widgets.PasswordInput)
- roles = RoleSerializer(many=True, read_only=True)
+ databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False)
class Meta:
model = DatabaseUser
- fields = ('url', 'username', 'password', 'type', 'roles')
- write_only_fields = ('username',)
+ fields = ('url', 'username', 'password', 'type', 'databases')
+
+ def save_object(self, obj, **kwargs):
+ # FIXME this method will be called when saving nested serializers :(
+ if not obj.pk:
+ obj.set_password(obj.password)
+ super(DatabaseUserSerializer, self).save_object(obj, **kwargs)
diff --git a/orchestra/apps/databases/tests/functional_tests/tests.py b/orchestra/apps/databases/tests/functional_tests/tests.py
index d202c3ba..e5b2d68d 100644
--- a/orchestra/apps/databases/tests/functional_tests/tests.py
+++ b/orchestra/apps/databases/tests/functional_tests/tests.py
@@ -1,5 +1,6 @@
import MySQLdb
import os
+import socket
import time
from functools import partial
@@ -57,6 +58,17 @@ class DatabaseTestMixin(object):
password = '@!?%spppP001' % random_ascii(5)
self.add(dbname, username, password)
self.validate_create_table(dbname, username, password)
+
+ def test_change_password(self):
+ dbname = '%s_database' % random_ascii(5)
+ username = '%s_dbuser' % random_ascii(5)
+ password = '@!?%spppP001' % random_ascii(5)
+ self.add(dbname, username, password)
+ self.validate_create_table(dbname, username, password)
+ new_password = '@!?%spppP001' % random_ascii(5)
+ self.change_password(username, new_password)
+ self.validate_login_error(dbname, username, password)
+ self.validate_create_table(dbname, username, new_password)
class MySQLBackendMixin(object):
@@ -64,7 +76,11 @@ class MySQLBackendMixin(object):
def setUp(self):
super(MySQLBackendMixin, self).setUp()
- settings.DATABASES_DEFAULT_HOST = '10.228.207.207'
+ # Get local ip address used to reach self.MASTER_SERVER
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ s.connect((self.MASTER_SERVER, 22))
+ settings.DATABASES_DEFAULT_HOST = s.getsockname()[0]
+ s.close()
def add_route(self):
server = Server.objects.create(name=self.MASTER_SERVER)
@@ -78,7 +94,11 @@ class MySQLBackendMixin(object):
def validate_create_table(self, name, username, password):
db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name)
cur = db.cursor()
- cur.execute('CREATE TABLE test ( id INT ) ;')
+ cur.execute('CREATE TABLE %s ( id INT ) ;' % random_ascii(20))
+
+ def validate_login_error(self, dbname, username, password):
+ self.assertRaises(MySQLdb.OperationalError,
+ self.validate_create_table, dbname, username, password)
def validate_delete(self, name, username, password):
self.asseRaises(MySQLdb.ConnectionError,
@@ -92,9 +112,16 @@ class RESTDatabaseMixin(DatabaseTestMixin):
@save_response_on_error
def add(self, dbname, username, password):
- user = self.rest.databaseusers.create(username=username, password=password)
- # TODO fucking nested objects
- self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type)
+ user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type)
+ users = [{
+ 'username': user.username
+ }]
+ self.rest.databases.create(name=dbname, users=users, type=self.db_type)
+
+ @save_response_on_error
+ def change_password(self, username, password):
+ user = self.rest.databaseusers.retrieve(username=username).get()
+ user.set_password(password)
class AdminDatabaseMixin(DatabaseTestMixin):
@@ -129,11 +156,16 @@ class AdminDatabaseMixin(DatabaseTestMixin):
def delete(self, dbname):
db = Database.objects.get(name=dbname)
self.admin_delete(db)
-
+
@snapshot_on_error
def delete_user(self, username):
user = DatabaseUser.objects.get(username=username)
self.admin_delete(user)
+
+ @snapshot_on_error
+ def change_password(self, username, password):
+ user = DatabaseUser.objects.get(username=username)
+ self.admin_change_password(user, password)
class RESTMysqlDatabaseTest(MySQLBackendMixin, RESTDatabaseMixin, BaseLiveServerTestCase):
diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py
index 7771a7e5..dd64aa99 100644
--- a/orchestra/apps/orchestration/methods.py
+++ b/orchestra/apps/orchestration/methods.py
@@ -37,7 +37,8 @@ def BashSSH(backend, log, server, cmds):
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
addr = server.get_address()
try:
- ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH)
+ # TODO timeout
+ ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH, timeout=10)
except socket.error:
logger.error('%s timed out on %s' % (backend, server))
log.state = BackendLog.TIMEOUT
diff --git a/orchestra/apps/webapps/backends/__init__.py b/orchestra/apps/webapps/backends/__init__.py
index e1077e22..66560d8f 100644
--- a/orchestra/apps/webapps/backends/__init__.py
+++ b/orchestra/apps/webapps/backends/__init__.py
@@ -1,6 +1,7 @@
import pkgutil
import textwrap
+
class WebAppServiceMixin(object):
model = 'webapps.WebApp'
@@ -16,6 +17,30 @@ class WebAppServiceMixin(object):
done
""" % context))
+ def get_php_init_vars(self, webapp, per_account=False):
+ """
+ process php options for inclusion on php.ini
+ per_account=True merges all (account, webapp.type) options
+ """
+ init_vars = []
+ options = webapp.options.all()
+ if per_account:
+ options = webapp.account.webapps.filter(webapp_type=webapp.type)
+ for opt in options:
+ name = opt.name.replace('PHP-', '')
+ value = "%s" % opt.value
+ init_vars.append((name, value))
+ enabled_functions = []
+ for value in options.filter(name='enabled_functions').values_list('value', flat=True):
+ enabled_functions += enabled_functions.get().value.split(',')
+ if enabled_functions:
+ disabled_functions = []
+ for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
+ if function not in enabled_functions:
+ disabled_functions.append(function)
+ init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
+ return init_vars
+
def delete_webapp_dir(self, context):
self.append("rm -fr %(app_path)s" % context)
@@ -30,5 +55,7 @@ class WebAppServiceMixin(object):
}
+
for __, module_name, __ in pkgutil.walk_packages(__path__):
+ # sorry for the exec(), but Import module function fails :(
exec('from . import %s' % module_name)
diff --git a/orchestra/apps/webapps/backends/phpfcgid.py b/orchestra/apps/webapps/backends/phpfcgid.py
index 27d45a90..ec69cee4 100644
--- a/orchestra/apps/webapps/backends/phpfcgid.py
+++ b/orchestra/apps/webapps/backends/phpfcgid.py
@@ -10,6 +10,7 @@ from .. import settings
class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
+ """ Per-webapp fcgid application """
verbose_name = _("PHP-Fcgid")
def save(self, webapp):
@@ -27,6 +28,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
def delete(self, webapp):
context = self.get_context(webapp)
+ self.append("rm '%(wrapper_path)s'" % context)
self.delete_webapp_dir(context)
def commit(self):
@@ -35,7 +37,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
def get_context(self, webapp):
context = super(PHPFcgidBackend, self).get_context(webapp)
- init_vars = webapp.get_php_init_vars()
+ init_vars = self.get_php_init_vars(webapp)
if init_vars:
init_vars = [ '%s="%s"' % (k,v) for v,k in init_vars.iteritems() ]
init_vars = ', -d '.join(init_vars)
diff --git a/orchestra/apps/webapps/backends/phpfpm.py b/orchestra/apps/webapps/backends/phpfpm.py
index ee1cebd6..eb0d5ea2 100644
--- a/orchestra/apps/webapps/backends/phpfpm.py
+++ b/orchestra/apps/webapps/backends/phpfpm.py
@@ -1,4 +1,5 @@
import os
+import textwrap
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
@@ -10,49 +11,57 @@ from .. import settings
class PHPFPMBackend(WebAppServiceMixin, ServiceController):
+ """ Per-webapp php application """
verbose_name = _("PHP-FPM")
def save(self, webapp):
context = self.get_context(webapp)
self.create_webapp_dir(context)
- self.append(
- "{ echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - ; } ||"
- " { echo -e '%(fpm_config)s' > %(fpm_path)s; UPDATEDFPM=1; }" % context
- )
+ self.append(textwrap.dedent("""\
+ {
+ echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s -
+ } || {
+ echo -e '%(fpm_config)s' > %(fpm_path)s
+ UPDATEDFPM=1
+ }""" % context))
def delete(self, webapp):
context = self.get_context(webapp)
+ self.append("rm '%(fpm_config)s'" % context)
self.delete_webapp_dir(context)
def commit(self):
super(PHPFPMBackend, self).commit()
- self.append('[[ $UPDATEDFPM == 1 ]] && service php5-fpm reload')
+ self.append(textwrap.dedent("""
+ [[ $UPDATEDFPM == 1 ]] && {
+ service php5-fpm start
+ service php5-fpm reload
+ }"""))
def get_context(self, webapp):
context = super(PHPFPMBackend, self).get_context(webapp)
context.update({
- 'init_vars': webapp.get_php_init_vars(),
+ 'init_vars': self.get_php_init_vars(webapp),
'fpm_port': webapp.get_fpm_port(),
})
context['fpm_listen'] = settings.WEBAPPS_FPM_LISTEN % context
- fpm_config = Template(
- ";; {{ banner }}\n"
- "[{{ user }}]\n"
- "user = {{ user }}\n"
- "group = {{ group }}\n\n"
- "listen = {{ fpm_listen | safe }}\n"
- "listen.owner = {{ user }}\n"
- "listen.group = {{ group }}\n"
- "pm = ondemand\n"
- "pm.max_children = 4\n"
- "{% for name,value in init_vars.iteritems %}"
- "php_admin_value[{{ name | safe }}] = {{ value | safe }}\n"
- "{% endfor %}"
- )
- fpm_file = '%(user)s.conf' % context
+ fpm_config = Template(textwrap.dedent("""\
+ ;; {{ banner }}
+ [{{ user }}]
+ user = {{ user }}
+ group = {{ group }}
+
+ listen = {{ fpm_listen | safe }}
+ listen.owner = {{ user }}
+ listen.group = {{ group }}
+ pm = ondemand
+ pm.max_children = 4
+ {% for name,value in init_vars.iteritems %}
+ php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}"""
+ ))
context.update({
'fpm_config': fpm_config.render(Context(context)),
- 'fpm_path': os.path.join(settings.WEBAPPS_PHPFPM_POOL_PATH, fpm_file),
+ 'fpm_path': settings.WEBAPPS_PHPFPM_POOL_PATH % context,
})
return context
diff --git a/orchestra/apps/webapps/models.py b/orchestra/apps/webapps/models.py
index 635e8c01..e8ca6518 100644
--- a/orchestra/apps/webapps/models.py
+++ b/orchestra/apps/webapps/models.py
@@ -39,23 +39,6 @@ class WebApp(models.Model):
def get_options(self):
return { opt.name: opt.value for opt in self.options.all() }
- def get_php_init_vars(self):
- init_vars = []
- options = WebAppOption.objects.filter(webapp__type=self.type)
- for opt in options.filter(webapp__account=self.account):
- name = opt.name.replace('PHP-', '')
- value = "%s" % opt.value
- init_vars.append((name, value))
- enabled_functions = self.options.filter(name='enabled_functions')
- if enabled_functions:
- enabled_functions = enabled_functions.get().value.split(',')
- disabled_functions = []
- for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
- if function not in enabled_functions:
- disabled_functions.append(function)
- init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
- return init_vars
-
def get_fpm_port(self):
return settings.WEBAPPS_FPM_START_PORT + self.account.pk
diff --git a/orchestra/apps/webapps/settings.py b/orchestra/apps/webapps/settings.py
index 770a1227..25d51d12 100644
--- a/orchestra/apps/webapps/settings.py
+++ b/orchestra/apps/webapps/settings.py
@@ -9,10 +9,16 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN',
# '/var/run/%(user)s-%(app_name)s.sock')
'127.0.0.1:%(fpm_port)s')
+
WEBAPPS_FPM_START_PORT = getattr(settings, 'WEBAPPS_FPM_START_PORT', 10000)
+
+WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
+ '/etc/php5/fpm/pool.d/%(app_name)s.conf')
+
+
WEBAPPS_FCGID_PATH = getattr(settings, 'WEBAPPS_FCGID_PATH',
- '/home/httpd/fcgid/%(user)s/%(type)s-wrapper')
+ '/home/httpd/fcgid/%(app_name)s-wrapper')
WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', {
@@ -166,10 +172,26 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', {
WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [
- 'exec', 'passthru', 'shell_exec', 'system', 'proc_open', 'popen', 'curl_exec',
- 'curl_multi_exec', 'show_source', 'pcntl_exec', 'proc_close',
- 'proc_get_status', 'proc_nice', 'proc_terminate', 'ini_alter', 'virtual',
- 'openlog', 'escapeshellcmd', 'escapeshellarg', 'dl'
+ 'exec',
+ 'passthru',
+ 'shell_exec',
+ 'system',
+ 'proc_open',
+ 'popen',
+ 'curl_exec',
+ 'curl_multi_exec',
+ 'show_source',
+ 'pcntl_exec',
+ 'proc_close',
+ 'proc_get_status',
+ 'proc_nice',
+ 'proc_terminate',
+ 'ini_alter',
+ 'virtual',
+ 'openlog',
+ 'escapeshellcmd',
+ 'escapeshellarg',
+ 'dl'
])
@@ -181,9 +203,6 @@ WEBAPPS_WORDPRESSMU_ADMIN_PASSWORD = getattr(settings, 'WEBAPPS_WORDPRESSMU_ADMI
'secret')
-
-
-
WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH = setattr(settings, 'WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH',
'/home/httpd/htdocs/wikifarm/template.tar.gz')
@@ -195,6 +214,3 @@ WEBAPPS_DOKUWIKIMU_FARM_PATH = getattr(settings, 'WEBAPPS_DOKUWIKIMU_FARM_PATH',
WEBAPPS_DRUPAL_SITES_PATH = getattr(settings, 'WEBAPPS_DRUPAL_SITES_PATH',
'/home/httpd/htdocs/drupal-mu/sites/%(app_name)s')
-
-WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
- '/etc/php5/fpm/pool.d')
diff --git a/orchestra/apps/webapps/tests/functional_tests/tests.py b/orchestra/apps/webapps/tests/functional_tests/tests.py
index e9d08fcd..f3dd45e6 100644
--- a/orchestra/apps/webapps/tests/functional_tests/tests.py
+++ b/orchestra/apps/webapps/tests/functional_tests/tests.py
@@ -36,39 +36,14 @@ class WebAppMixin(object):
djsettings.DEBUG = True
def add_route(self):
-# backends = [
-# # TODO MU apps on SaaS?
-# backends.awstats.AwstatsBackend,
-# backends.dokuwikimu.DokuWikiMuBackend,
-# backends.drupalmu.DrupalMuBackend,
-# backends.phpfcgid.PHPFcgidBackend,
-# backends.phpfpm.PHPFPMBackend,
-# backends.static.StaticBackend,
-# backends.wordpressmu.WordpressMuBackend,
-# ]
- server = Server.objects.create(name=self.MASTER_SERVER)
- for backend in [SystemUserBackend, self.backend]:
- backend = backend.get_name()
- Route.objects.create(backend=backend, match=True, host=server)
+ server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER)
+ backend = SystemUserBackend.get_name()
+ Route.objects.get_or_create(backend=backend, match=True, host=server)
+ backend = self.backend.get_name()
+ match = 'webapp.type == "%s"' % self.type_value
+ Route.objects.create(backend=backend, match=match, host=server)
- def test_add(self):
- name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
- self.add_webapp(name)
- self.validate_add_webapp(name)
-# self.addCleanup(self.delete, username)
-
-
-class StaticWebAppMixin(object):
- backend = backends.static.StaticBackend
- type_value = 'static'
- token = random_ascii(100)
- page = (
- 'index.html',
- 'Hello World! %s \n' % token,
- 'Hello World! %s \n' % token,
- )
-
- def validate_add_webapp(self, name):
+ def upload_webapp(self, name):
try:
ftp = ftplib.FTP(self.MASTER_SERVER)
ftp.login(user=self.account.username, passwd=self.account_password)
@@ -80,6 +55,23 @@ class StaticWebAppMixin(object):
index.close()
finally:
ftp.close()
+
+ def test_add(self):
+ name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+ self.add_webapp(name)
+ self.addCleanup(self.delete_webapp, name)
+ self.upload_webapp(name)
+
+
+class StaticWebAppMixin(object):
+ backend = backends.static.StaticBackend
+ type_value = 'static'
+ token = random_ascii(100)
+ page = (
+ 'index.html',
+ 'Hello World! %s \n' % token,
+ 'Hello World! %s \n' % token,
+ )
class PHPFcidWebAppMixin(StaticWebAppMixin):
@@ -111,12 +103,11 @@ class RESTWebAppMixin(object):
@save_response_on_error
def add_webapp(self, name, options=[]):
- self.rest.webapps.create(name=name, type=self.type_value)
+ self.rest.webapps.create(name=name, type=self.type_value, options=options)
@save_response_on_error
def delete_webapp(self, name):
- list = self.rest.lists.retrieve(name=name).get()
- list.delete()
+ self.rest.webapps.retrieve(name=name).delete()
class AdminWebAppMixin(WebAppMixin):
@@ -125,54 +116,25 @@ class AdminWebAppMixin(WebAppMixin):
self.admin_login()
# create main user
self.save_systemuser()
- # TODO save_account()
+
+ @snapshot_on_error
+ def save_systemuser(self):
+ url = ''
@snapshot_on_error
def add(self, name, password, admin_email):
- url = self.live_server_url + reverse('admin:mails_List_add')
- self.selenium.get(url)
-
- account_input = self.selenium.find_element_by_id('id_account')
- account_select = Select(account_input)
- account_select.select_by_value(str(self.account.pk))
-
- name_field = self.selenium.find_element_by_id('id_name')
- name_field.send_keys(username)
-
- password_field = self.selenium.find_element_by_id('id_password1')
- password_field.send_keys(password)
- password_field = self.selenium.find_element_by_id('id_password2')
- password_field.send_keys(password)
-
- if quota is not None:
- quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
- quota_field = self.selenium.find_element_by_id(quota_id)
- quota_field.clear()
- quota_field.send_keys(quota)
-
- if filtering is not None:
- filtering_input = self.selenium.find_element_by_id('id_filtering')
- filtering_select = Select(filtering_input)
- filtering_select.select_by_value("CUSTOM")
- filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
- filtering_inline.click()
- time.sleep(0.5)
- filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
- filtering_field.send_keys(filtering)
-
- name_field.submit()
- self.assertNotEqual(url, self.selenium.current_url)
+ pass
-class RESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
+class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass
-class RESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
+class PHPFcidRESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass
-class RESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
+class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass
diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py
index 007c8cac..be4d494e 100644
--- a/orchestra/apps/websites/backends/apache.py
+++ b/orchestra/apps/websites/backends/apache.py
@@ -112,11 +112,11 @@ class Apache2Backend(ServiceController):
directives += "SecRuleRemoveById %d" % rule
for modsecurity in site.options.filter(name='sec_rule_off'):
- directives += (
- "\n"
- " SecRuleEngine Off\n"
- "\n" % modsecurity.value
- )
+ directives += textwrap.dedent("""\
+
+ SecRuleEngine Off
+
+ """ % modsecurity.value)
return directives
def get_protections(self, site):
diff --git a/orchestra/apps/websites/models.py b/orchestra/apps/websites/models.py
index 69f70292..76756c72 100644
--- a/orchestra/apps/websites/models.py
+++ b/orchestra/apps/websites/models.py
@@ -39,7 +39,9 @@ class Website(models.Model):
@cached
def get_options(self):
- return { opt.name: opt.value for opt in self.options.all() }
+ return {
+ opt.name: opt.value for opt in self.options.all()
+ }
@property
def protocol(self):
@@ -81,12 +83,15 @@ class Content(models.Model):
class Meta:
unique_together = ('website', 'path')
+ def __unicode__(self):
+ try:
+ return self.website.name + self.path
+ except Website.DoesNotExist:
+ return self.path
+
def clean(self):
if not self.path.startswith('/'):
self.path = '/' + self.path
-
- def __unicode__(self):
- return self.website.name + self.path
services.register(Website)
diff --git a/orchestra/apps/websites/serializers.py b/orchestra/apps/websites/serializers.py
index 58f8c4b6..1b0e79d3 100644
--- a/orchestra/apps/websites/serializers.py
+++ b/orchestra/apps/websites/serializers.py
@@ -10,6 +10,9 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Content
fields = ('webapp', 'path')
+
+ def get_identity(self, data):
+ return '%s-%s' % (data.get('website'), data.get('path'))
class WebsiteSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
diff --git a/orchestra/apps/websites/tests/functional_tests/tests.py b/orchestra/apps/websites/tests/functional_tests/tests.py
index 2095acd2..7d3796e5 100644
--- a/orchestra/apps/websites/tests/functional_tests/tests.py
+++ b/orchestra/apps/websites/tests/functional_tests/tests.py
@@ -38,9 +38,9 @@ class WebsiteMixin(WebAppMixin):
super(WebsiteMixin, self).add_route()
server = Server.objects.get()
backend = backends.apache.Apache2Backend.get_name()
- Route.objects.create(backend=backend, match=True, host=server)
+ Route.objects.get_or_create(backend=backend, match=True, host=server)
backend = Bind9MasterDomainBackend.get_name()
- Route.objects.create(backend=backend, match=True, host=server)
+ Route.objects.get_or_create(backend=backend, match=True, host=server)
def validate_add_website(self, name, domain):
url = 'http://%s/%s' % (domain.name, self.page[0])
@@ -54,9 +54,11 @@ class WebsiteMixin(WebAppMixin):
self.save_domain(domain)
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
- self.validate_add_webapp(webapp)
+ self.addCleanup(self.delete_webapp, webapp)
+ self.upload_webapp(webapp)
website = '%s_website' % random_ascii(10)
self.add_website(website, domain, webapp)
+ self.addCleanup(self.delete_website, website)
self.validate_add_website(website, domain)
@@ -65,21 +67,82 @@ class RESTWebsiteMixin(RESTWebAppMixin):
def save_domain(self, domain):
self.rest.domains.retrieve().get().save()
- def add_website(self, name, domain, webapp):
- domain = self.rest.domains.retrieve().get()
- webapp = self.rest.webapps.retrieve().get()
- self.rest.websites.create(name=name, domains=[domain.url], contents=[{'webapp': webapp.url}])
+ @save_response_on_error
+ def add_website(self, name, domain, webapp, path='/'):
+ domain = self.rest.domains.retrieve(name=domain).get()
+ webapp = self.rest.webapps.retrieve(name=webapp).get()
+ contents = [{
+ 'webapp': webapp.url,
+ 'path': path
+ }]
+ self.rest.websites.create(name=name, domains=[domain.url], contents=contents)
+
+ @save_response_on_error
+ def delete_website(self, name):
+ print 'hola'
+ pass
+ self.rest.websites.retrieve(name=name).delete()
+# self.rest.websites.retrieve(name=name).delete()
+
+ @save_response_on_error
+ def add_content(self, website, webapp, path):
+ website = self.rest.websites.retrieve(name=website).get()
+ webapp = self.rest.webapps.retrieve(name=webapp).get()
+ website.contents.append({
+ 'webapp': webapp.url,
+ 'path': path,
+ })
+ website.save()
-class RESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
+class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
+ def test_mix_webapps(self):
+ domain_name = '%sdomain.lan' % random_ascii(10)
+ domain = Domain.objects.create(name=domain_name, account=self.account)
+ domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR)
+ self.save_domain(domain)
+ webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+ self.add_webapp(webapp)
+ self.addCleanup(self.delete_webapp, webapp)
+ self.upload_webapp(webapp)
+ website = '%s_website' % random_ascii(10)
+ self.add_website(website, domain, webapp)
+ self.addCleanup(self.delete_website, website)
+ self.validate_add_website(website, domain)
+
+ self.type_value = PHPFcidWebAppMixin.type_value
+ self.backend = PHPFcidWebAppMixin.backend
+ self.page = PHPFcidWebAppMixin.page
+ self.add_route()
+ webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+ self.add_webapp(webapp)
+ self.addCleanup(self.delete_webapp, webapp)
+ self.upload_webapp(webapp)
+ path = '/%s' % webapp
+ self.add_content(website, webapp, path)
+ url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
+ self.assertEqual(self.page[2], requests.get(url).content)
+
+ self.type_value = PHPFPMWebAppMixin.type_value
+ self.backend = PHPFPMWebAppMixin.backend
+ self.page = PHPFPMWebAppMixin.page
+ self.add_route()
+ webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
+ self.add_webapp(webapp)
+ self.addCleanup(self.delete_webapp, webapp)
+ self.upload_webapp(webapp)
+ path = '/%s' % webapp
+
+ self.add_content(website, webapp, path)
+ url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
+ self.assertEqual(self.page[2], requests.get(url).content)
+
+
+class PHPFcidRESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass
-class RESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
- pass
-
-
-class RESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
+class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass
#class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase):
diff --git a/orchestra/templates/admin/base_site.html b/orchestra/templates/admin/base_site.html
index 6d4cd4e8..c19028d0 100644
--- a/orchestra/templates/admin/base_site.html
+++ b/orchestra/templates/admin/base_site.html
@@ -27,6 +27,7 @@
{% if not is_popup %}
{% admin_tools_render_menu %}
{% endif %}
+
{% endif %}
{% endblock %}
diff --git a/scripts/services/apache_full_stack.md b/scripts/services/apache_full_stack.md
index 57cb3b0a..6a477862 100644
--- a/scripts/services/apache_full_stack.md
+++ b/scripts/services/apache_full_stack.md
@@ -84,6 +84,12 @@ The goal of this setup is having a high-performance state-of-the-art deployment
+
+# TODO pool per website or pool per user? memory consumption
+events.mechanism = epoll
+# TODO multiple master processes, opcache is held in master, and reload/restart affects all pools
+# http://mattiasgeniar.be/2014/04/09/a-better-way-to-run-php-fpm/
+
TODO CHRoot
https://andrewbevitt.com/tutorials/apache-varnish-chrooted-php-fpm-wordpress-virtual-host/
@@ -92,10 +98,10 @@ TODO CHRoot
[vhost]
istemplate = 1
listen.mode = 0660
+ pm = ondemand
pm.max_children = 5
- pm.start_servers = 1
- pm.min_spare_servers = 1
- pm.max_spare_servers = 2
+ pm.process_idle_timeout = 10s
+ pm.max_requests = 200
' > /etc/php5/fpm/conf.d/vhost-template.conf
```