Improvements on databases, webapps and websites

This commit is contained in:
Marc 2014-10-14 13:50:19 +00:00
parent 920f8efcd5
commit 4c7c5b5505
21 changed files with 380 additions and 299 deletions

View File

@ -138,3 +138,12 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* DN: Transaction atomicity and backend failure * DN: Transaction atomicity and backend failure
* SaaS Icons * SaaS Icons
* offer to create mailbox on account creation
* init.d celery scripts
-# Required-Start: $network $local_fs $remote_fs postgresql celeryd
-# Required-Stop: $network $local_fs $remote_fs postgresql celeryd
* POST only fields (account, username, name) etc

View File

@ -72,7 +72,8 @@ class Account(auth.AbstractBaseUser):
def disable(self): def disable(self):
self.is_active = False self.is_active = False
# self.save(update_fields=['is_active']) self.save(update_fields=['is_active'])
# Trigger save() on related objects that depend on this account
for rel in self._meta.get_all_related_objects(): for rel in self._meta.get_all_related_objects():
if not rel.model in services: if not rel.model in services:
continue continue

View File

@ -4,68 +4,24 @@ from django.contrib.auth.admin import UserAdmin
from django.utils.safestring import mark_safe from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from orchestra.admin import ExtendedModelAdmin from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
from orchestra.admin.utils import admin_link from orchestra.admin.utils import admin_link
from orchestra.apps.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin from orchestra.apps.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin
from .forms import (DatabaseUserChangeForm, DatabaseUserCreationForm, from .forms import DatabaseCreationForm, DatabaseUserChangeForm, DatabaseUserCreationForm
DatabaseCreationForm) from .models import Database, DatabaseUser
from .models import Database, Role, DatabaseUser
class UserInline(admin.TabularInline):
model = Role
verbose_name_plural = _("Users")
readonly_fields = ('user_link',)
extra = 0
user_link = admin_link('user')
def formfield_for_dbfield(self, db_field, **kwargs):
""" Make value input widget bigger """
if db_field.name == 'user':
users = db_field.rel.to.objects.filter(type=self.parent_object.type)
kwargs['queryset'] = users.filter(account=self.account)
return super(UserInline, self).formfield_for_dbfield(db_field, **kwargs)
class PermissionInline(AccountAdminMixin, admin.TabularInline):
model = Role
verbose_name_plural = _("Permissions")
readonly_fields = ('database_link',)
extra = 0
filter_by_account_fields = ['database']
database_link = admin_link('database', popup=True)
def formfield_for_dbfield(self, db_field, **kwargs):
""" Make value input widget bigger """
formfield = super(PermissionInline, self).formfield_for_dbfield(db_field, **kwargs)
if db_field.name == 'database':
# Hack widget render in order to append ?type='db_type' to the add url
db_type = self.parent_object.type
old_render = formfield.widget.render
def render(*args, **kwargs):
output = old_render(*args, **kwargs)
output = output.replace('/add/?', '/add/?type=%s&' % db_type)
return mark_safe(output)
formfield.widget.render = render
formfield.queryset = formfield.queryset.filter(type=db_type)
return formfield
class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
list_display = ('name', 'type', 'account_link') list_display = ('name', 'type', 'account_link')
list_filter = ('type',) list_filter = ('type',)
search_fields = ['name'] search_fields = ['name']
inlines = [UserInline]
add_inlines = []
change_readonly_fields = ('name', 'type') change_readonly_fields = ('name', 'type')
extra = 1 extra = 1
fieldsets = ( fieldsets = (
(None, { (None, {
'classes': ('extrapretty',), 'classes': ('extrapretty',),
'fields': ('account_link', 'name', 'type'), 'fields': ('account_link', 'name', 'type', 'users'),
}), }),
) )
add_fieldsets = ( add_fieldsets = (
@ -92,22 +48,20 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
user = DatabaseUser( user = DatabaseUser(
username=form.cleaned_data['username'], username=form.cleaned_data['username'],
type=obj.type, type=obj.type,
account_id = obj.account.pk, account_id=obj.account.pk,
) )
user.set_password(form.cleaned_data["password1"]) user.set_password(form.cleaned_data["password1"])
user.save() user.save()
Role.objects.create(database=obj, user=user, is_owner=True) obj.users.add(user)
class DatabaseUserAdmin(SelectAccountAdminMixin, ExtendedModelAdmin): class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, ExtendedModelAdmin):
list_display = ('username', 'type', 'account_link') list_display = ('username', 'type', 'account_link')
list_filter = ('type',) list_filter = ('type',)
search_fields = ['username'] search_fields = ['username']
form = DatabaseUserChangeForm form = DatabaseUserChangeForm
add_form = DatabaseUserCreationForm add_form = DatabaseUserCreationForm
change_readonly_fields = ('username', 'type') change_readonly_fields = ('username', 'type')
inlines = [PermissionInline]
add_inlines = []
fieldsets = ( fieldsets = (
(None, { (None, {
'classes': ('extrapretty',), 'classes': ('extrapretty',),

View File

@ -1,3 +1,5 @@
import textwrap
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController from orchestra.apps.orchestration import ServiceController
@ -14,12 +16,12 @@ class MySQLBackend(ServiceController):
if database.type == database.MYSQL: if database.type == database.MYSQL:
context = self.get_context(database) context = self.get_context(database)
self.append( self.append(
"mysql -e 'CREATE DATABASE `%(database)s`;'" % context "mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
)
self.append(
"mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
" TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context
) )
self.append(textwrap.dedent("""\
mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(owner)s"@"%(host)s" WITH GRANT OPTION;' \
""" % context
))
def delete(self, database): def delete(self, database):
if database.type == database.MYSQL: if database.type == database.MYSQL:
@ -44,20 +46,25 @@ class MySQLUserBackend(ServiceController):
def save(self, user): def save(self, user):
if user.type == user.MYSQL: if user.type == user.MYSQL:
context = self.get_context(user) context = self.get_context(user)
self.append( self.append(textwrap.dedent("""\
"mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \
) """ % context
self.append( ))
"mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" " self.append(textwrap.dedent("""\
" WHERE User=\"%(username)s\";'" % context mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \
) """ % context
))
def delete(self, user): def delete(self, user):
if user.type == user.MYSQL: if user.type == user.MYSQL:
context = self.get_context(database) context = self.get_context(database)
self.append( self.append(textwrap.dedent("""\
"mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context mysql -e 'DROP USER "%(username)s"@"%(host)s";' \
) """ % context
))
def commit(self):
self.append("mysql -e 'FLUSH PRIVILEGES;'")
def get_context(self, user): def get_context(self, user):
return { return {
@ -78,27 +85,28 @@ class MysqlDisk(ServiceMonitor):
def exceeded(self, db): def exceeded(self, db):
context = self.get_context(db) context = self.get_context(db)
self.append("mysql -e '" self.append(textwrap.dedent("""\
"UPDATE db SET Insert_priv=\"N\", Create_priv=\"N\"" mysql -e 'UPDATE db SET Insert_priv="N", Create_priv="N" WHERE Db="%(db_name)s";' \
" WHERE Db=\"%(db_name)s\";'" % context """ % context
) ))
def recovery(self, db): def recovery(self, db):
context = self.get_context(db) context = self.get_context(db)
self.append("mysql -e '" self.append(textwrap.dedent("""\
"UPDATE db SET Insert_priv=\"Y\", Create_priv=\"Y\"" mysql -e 'UPDATE db SET Insert_priv="Y", Create_priv="Y" WHERE Db="%(db_name)s";' \
" WHERE Db=\"%(db_name)s\";'" % context """ % context
) ))
def monitor(self, db): def monitor(self, db):
context = self.get_context(db) context = self.get_context(db)
self.append( self.append(textwrap.dedent("""\
"echo %(db_id)s $(mysql -B -e '" echo %(db_id)s $(mysql -B -e '"
" SELECT sum( data_length + index_length ) \"Size\"\n" SELECT sum( data_length + index_length ) "Size"
" FROM information_schema.TABLES\n" FROM information_schema.TABLES
" WHERE table_schema=\"gisp\"\n" WHERE table_schema = "gisp"
" GROUP BY table_schema;' | tail -n 1)" % context GROUP BY table_schema;' | tail -n 1) \
) """ % context
))
def get_context(self, db): def get_context(self, db):
return { return {

View File

@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.core.validators import validate_password from orchestra.core.validators import validate_password
from .models import DatabaseUser, Database, Role from .models import DatabaseUser, Database
class DatabaseUserCreationForm(forms.ModelForm): class DatabaseUserCreationForm(forms.ModelForm):
@ -28,13 +28,6 @@ class DatabaseUserCreationForm(forms.ModelForm):
raise forms.ValidationError(msg) raise forms.ValidationError(msg)
return password2 return password2
def save(self, commit=True):
user = super(DatabaseUserCreationForm, self).save(commit=False)
# user.set_password(self.cleaned_data["password1"])
# if commit:
# user.save()
return user
class DatabaseCreationForm(DatabaseUserCreationForm): class DatabaseCreationForm(DatabaseUserCreationForm):
username = forms.RegexField(label=_("Username"), max_length=30, username = forms.RegexField(label=_("Username"), max_length=30,
@ -87,20 +80,6 @@ class DatabaseCreationForm(DatabaseUserCreationForm):
raise forms.ValidationError(msg) raise forms.ValidationError(msg)
return cleaned_data return cleaned_data
def save(self, commit=True):
db = super(DatabaseUserCreationForm, self).save(commit=False)
# if commit:
# user = self.cleaned_data['user']
# if not user:
# user = DatabaseUser(
# username=self.cleaned_data['username'],
# type=self.cleaned_data['type'],
# )
# user.set_password(self.cleaned_data["password1"])
# user.save()
# role, __ = Role.objects.get_or_create(database=db, user=user)
return db
class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField): class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField):
class ReadOnlyPasswordHashWidget(forms.Widget): class ReadOnlyPasswordHashWidget(forms.Widget):

View File

@ -16,8 +16,8 @@ class Database(models.Model):
name = models.CharField(_("name"), max_length=64, # MySQL limit name = models.CharField(_("name"), max_length=64, # MySQL limit
validators=[validators.validate_name]) validators=[validators.validate_name])
users = models.ManyToManyField('databases.DatabaseUser', users = models.ManyToManyField('databases.DatabaseUser',
verbose_name=_("users"), verbose_name=_("users"),related_name='databases')
through='databases.Role', related_name='users') # through='databases.Role',
type = models.CharField(_("type"), max_length=32, type = models.CharField(_("type"), max_length=32,
choices=settings.DATABASES_TYPE_CHOICES, choices=settings.DATABASES_TYPE_CHOICES,
default=settings.DATABASES_DEFAULT_TYPE) default=settings.DATABASES_DEFAULT_TYPE)
@ -32,29 +32,38 @@ class Database(models.Model):
@property @property
def owner(self): def owner(self):
return self.roles.get(is_owner=True).user """ database owner is the first user related to it """
# Accessing intermediary model to get which is the first user
users = Database.users.through.objects.filter(database_id=self.id)
return users.order_by('-id').first().databaseuser
class Role(models.Model): Database.users.through._meta.unique_together = (('database', 'databaseuser'),)
database = models.ForeignKey(Database, verbose_name=_("database"),
related_name='roles')
user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
related_name='roles')
is_owner = models.BooleanField(_("owner"), default=False)
class Meta: #class Role(models.Model):
unique_together = ('database', 'user') # database = models.ForeignKey(Database, verbose_name=_("database"),
# related_name='roles')
def __unicode__(self): # user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
return "%s@%s" % (self.user, self.database) # related_name='roles')
## is_owner = models.BooleanField(_("owner"), default=False)
def clean(self): #
if self.user.type != self.database.type: # class Meta:
msg = _("Database and user type doesn't match") # unique_together = ('database', 'user')
raise validators.ValidationError(msg) #
roles = self.database.roles.values('id') # def __unicode__(self):
if not roles or (len(roles) == 1 and roles[0].id == self.id): # return "%s@%s" % (self.user, self.database)
self.is_owner = True #
# @property
# def is_owner(self):
# return datatase.owner == self
#
# def clean(self):
# if self.user.type != self.database.type:
# msg = _("Database and user type doesn't match")
# raise validators.ValidationError(msg)
# roles = self.database.roles.values('id')
# if not roles or (len(roles) == 1 and roles[0].id == self.id):
# self.is_owner = True
class DatabaseUser(models.Model): class DatabaseUser(models.Model):

View File

@ -5,36 +5,47 @@ from rest_framework import serializers
from orchestra.apps.accounts.serializers import AccountSerializerMixin from orchestra.apps.accounts.serializers import AccountSerializerMixin
from orchestra.core.validators import validate_password from orchestra.core.validators import validate_password
from .models import Database, DatabaseUser, Role from .models import Database, DatabaseUser
class UserRoleSerializer(serializers.HyperlinkedModelSerializer): class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer):
class Meta: class Meta:
model = Role model = DatabaseUser
fields = ('user', 'is_owner',) fields = ('url', 'username')
def from_native(self, data, files=None):
class RoleSerializer(serializers.HyperlinkedModelSerializer): return DatabaseUser.objects.get(username=data['username'])
class Meta:
model = Role
fields = ('database', 'is_owner',)
class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
roles = UserRoleSerializer(many=True) users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True)
class Meta: class Meta:
model = Database model = Database
fields = ('url', 'name', 'type', 'roles') fields = ('url', 'name', 'type', 'users')
class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Database
fields = ('url', 'name',)
def from_native(self, data, files=None):
return Database.objects.get(name=data['name'])
class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
password = serializers.CharField(max_length=128, label=_('Password'), password = serializers.CharField(max_length=128, label=_('Password'),
validators=[validate_password], write_only=True, validators=[validate_password], write_only=True,
widget=widgets.PasswordInput) widget=widgets.PasswordInput)
roles = RoleSerializer(many=True, read_only=True) databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False)
class Meta: class Meta:
model = DatabaseUser model = DatabaseUser
fields = ('url', 'username', 'password', 'type', 'roles') fields = ('url', 'username', 'password', 'type', 'databases')
write_only_fields = ('username',)
def save_object(self, obj, **kwargs):
# FIXME this method will be called when saving nested serializers :(
if not obj.pk:
obj.set_password(obj.password)
super(DatabaseUserSerializer, self).save_object(obj, **kwargs)

View File

@ -1,5 +1,6 @@
import MySQLdb import MySQLdb
import os import os
import socket
import time import time
from functools import partial from functools import partial
@ -58,13 +59,28 @@ class DatabaseTestMixin(object):
self.add(dbname, username, password) self.add(dbname, username, password)
self.validate_create_table(dbname, username, password) self.validate_create_table(dbname, username, password)
def test_change_password(self):
dbname = '%s_database' % random_ascii(5)
username = '%s_dbuser' % random_ascii(5)
password = '@!?%spppP001' % random_ascii(5)
self.add(dbname, username, password)
self.validate_create_table(dbname, username, password)
new_password = '@!?%spppP001' % random_ascii(5)
self.change_password(username, new_password)
self.validate_login_error(dbname, username, password)
self.validate_create_table(dbname, username, new_password)
class MySQLBackendMixin(object): class MySQLBackendMixin(object):
db_type = 'mysql' db_type = 'mysql'
def setUp(self): def setUp(self):
super(MySQLBackendMixin, self).setUp() super(MySQLBackendMixin, self).setUp()
settings.DATABASES_DEFAULT_HOST = '10.228.207.207' # Get local ip address used to reach self.MASTER_SERVER
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect((self.MASTER_SERVER, 22))
settings.DATABASES_DEFAULT_HOST = s.getsockname()[0]
s.close()
def add_route(self): def add_route(self):
server = Server.objects.create(name=self.MASTER_SERVER) server = Server.objects.create(name=self.MASTER_SERVER)
@ -78,7 +94,11 @@ class MySQLBackendMixin(object):
def validate_create_table(self, name, username, password): def validate_create_table(self, name, username, password):
db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name) db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name)
cur = db.cursor() cur = db.cursor()
cur.execute('CREATE TABLE test ( id INT ) ;') cur.execute('CREATE TABLE %s ( id INT ) ;' % random_ascii(20))
def validate_login_error(self, dbname, username, password):
self.assertRaises(MySQLdb.OperationalError,
self.validate_create_table, dbname, username, password)
def validate_delete(self, name, username, password): def validate_delete(self, name, username, password):
self.asseRaises(MySQLdb.ConnectionError, self.asseRaises(MySQLdb.ConnectionError,
@ -92,9 +112,16 @@ class RESTDatabaseMixin(DatabaseTestMixin):
@save_response_on_error @save_response_on_error
def add(self, dbname, username, password): def add(self, dbname, username, password):
user = self.rest.databaseusers.create(username=username, password=password) user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type)
# TODO fucking nested objects users = [{
self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type) 'username': user.username
}]
self.rest.databases.create(name=dbname, users=users, type=self.db_type)
@save_response_on_error
def change_password(self, username, password):
user = self.rest.databaseusers.retrieve(username=username).get()
user.set_password(password)
class AdminDatabaseMixin(DatabaseTestMixin): class AdminDatabaseMixin(DatabaseTestMixin):
@ -135,6 +162,11 @@ class AdminDatabaseMixin(DatabaseTestMixin):
user = DatabaseUser.objects.get(username=username) user = DatabaseUser.objects.get(username=username)
self.admin_delete(user) self.admin_delete(user)
@snapshot_on_error
def change_password(self, username, password):
user = DatabaseUser.objects.get(username=username)
self.admin_change_password(user, password)
class RESTMysqlDatabaseTest(MySQLBackendMixin, RESTDatabaseMixin, BaseLiveServerTestCase): class RESTMysqlDatabaseTest(MySQLBackendMixin, RESTDatabaseMixin, BaseLiveServerTestCase):
pass pass

View File

@ -37,7 +37,8 @@ def BashSSH(backend, log, server, cmds):
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
addr = server.get_address() addr = server.get_address()
try: try:
ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH) # TODO timeout
ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH, timeout=10)
except socket.error: except socket.error:
logger.error('%s timed out on %s' % (backend, server)) logger.error('%s timed out on %s' % (backend, server))
log.state = BackendLog.TIMEOUT log.state = BackendLog.TIMEOUT

View File

@ -1,6 +1,7 @@
import pkgutil import pkgutil
import textwrap import textwrap
class WebAppServiceMixin(object): class WebAppServiceMixin(object):
model = 'webapps.WebApp' model = 'webapps.WebApp'
@ -16,6 +17,30 @@ class WebAppServiceMixin(object):
done done
""" % context)) """ % context))
def get_php_init_vars(self, webapp, per_account=False):
"""
process php options for inclusion on php.ini
per_account=True merges all (account, webapp.type) options
"""
init_vars = []
options = webapp.options.all()
if per_account:
options = webapp.account.webapps.filter(webapp_type=webapp.type)
for opt in options:
name = opt.name.replace('PHP-', '')
value = "%s" % opt.value
init_vars.append((name, value))
enabled_functions = []
for value in options.filter(name='enabled_functions').values_list('value', flat=True):
enabled_functions += enabled_functions.get().value.split(',')
if enabled_functions:
disabled_functions = []
for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
if function not in enabled_functions:
disabled_functions.append(function)
init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
return init_vars
def delete_webapp_dir(self, context): def delete_webapp_dir(self, context):
self.append("rm -fr %(app_path)s" % context) self.append("rm -fr %(app_path)s" % context)
@ -30,5 +55,7 @@ class WebAppServiceMixin(object):
} }
for __, module_name, __ in pkgutil.walk_packages(__path__): for __, module_name, __ in pkgutil.walk_packages(__path__):
# sorry for the exec(), but Import module function fails :(
exec('from . import %s' % module_name) exec('from . import %s' % module_name)

View File

@ -10,6 +10,7 @@ from .. import settings
class PHPFcgidBackend(WebAppServiceMixin, ServiceController): class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
""" Per-webapp fcgid application """
verbose_name = _("PHP-Fcgid") verbose_name = _("PHP-Fcgid")
def save(self, webapp): def save(self, webapp):
@ -27,6 +28,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
def delete(self, webapp): def delete(self, webapp):
context = self.get_context(webapp) context = self.get_context(webapp)
self.append("rm '%(wrapper_path)s'" % context)
self.delete_webapp_dir(context) self.delete_webapp_dir(context)
def commit(self): def commit(self):
@ -35,7 +37,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
def get_context(self, webapp): def get_context(self, webapp):
context = super(PHPFcgidBackend, self).get_context(webapp) context = super(PHPFcgidBackend, self).get_context(webapp)
init_vars = webapp.get_php_init_vars() init_vars = self.get_php_init_vars(webapp)
if init_vars: if init_vars:
init_vars = [ '%s="%s"' % (k,v) for v,k in init_vars.iteritems() ] init_vars = [ '%s="%s"' % (k,v) for v,k in init_vars.iteritems() ]
init_vars = ', -d '.join(init_vars) init_vars = ', -d '.join(init_vars)

View File

@ -1,4 +1,5 @@
import os import os
import textwrap
from django.template import Template, Context from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
@ -10,49 +11,57 @@ from .. import settings
class PHPFPMBackend(WebAppServiceMixin, ServiceController): class PHPFPMBackend(WebAppServiceMixin, ServiceController):
""" Per-webapp php application """
verbose_name = _("PHP-FPM") verbose_name = _("PHP-FPM")
def save(self, webapp): def save(self, webapp):
context = self.get_context(webapp) context = self.get_context(webapp)
self.create_webapp_dir(context) self.create_webapp_dir(context)
self.append( self.append(textwrap.dedent("""\
"{ echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - ; } ||" {
" { echo -e '%(fpm_config)s' > %(fpm_path)s; UPDATEDFPM=1; }" % context echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s -
) } || {
echo -e '%(fpm_config)s' > %(fpm_path)s
UPDATEDFPM=1
}""" % context))
def delete(self, webapp): def delete(self, webapp):
context = self.get_context(webapp) context = self.get_context(webapp)
self.append("rm '%(fpm_config)s'" % context)
self.delete_webapp_dir(context) self.delete_webapp_dir(context)
def commit(self): def commit(self):
super(PHPFPMBackend, self).commit() super(PHPFPMBackend, self).commit()
self.append('[[ $UPDATEDFPM == 1 ]] && service php5-fpm reload') self.append(textwrap.dedent("""
[[ $UPDATEDFPM == 1 ]] && {
service php5-fpm start
service php5-fpm reload
}"""))
def get_context(self, webapp): def get_context(self, webapp):
context = super(PHPFPMBackend, self).get_context(webapp) context = super(PHPFPMBackend, self).get_context(webapp)
context.update({ context.update({
'init_vars': webapp.get_php_init_vars(), 'init_vars': self.get_php_init_vars(webapp),
'fpm_port': webapp.get_fpm_port(), 'fpm_port': webapp.get_fpm_port(),
}) })
context['fpm_listen'] = settings.WEBAPPS_FPM_LISTEN % context context['fpm_listen'] = settings.WEBAPPS_FPM_LISTEN % context
fpm_config = Template( fpm_config = Template(textwrap.dedent("""\
";; {{ banner }}\n" ;; {{ banner }}
"[{{ user }}]\n" [{{ user }}]
"user = {{ user }}\n" user = {{ user }}
"group = {{ group }}\n\n" group = {{ group }}
"listen = {{ fpm_listen | safe }}\n"
"listen.owner = {{ user }}\n" listen = {{ fpm_listen | safe }}
"listen.group = {{ group }}\n" listen.owner = {{ user }}
"pm = ondemand\n" listen.group = {{ group }}
"pm.max_children = 4\n" pm = ondemand
"{% for name,value in init_vars.iteritems %}" pm.max_children = 4
"php_admin_value[{{ name | safe }}] = {{ value | safe }}\n" {% for name,value in init_vars.iteritems %}
"{% endfor %}" php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}"""
) ))
fpm_file = '%(user)s.conf' % context
context.update({ context.update({
'fpm_config': fpm_config.render(Context(context)), 'fpm_config': fpm_config.render(Context(context)),
'fpm_path': os.path.join(settings.WEBAPPS_PHPFPM_POOL_PATH, fpm_file), 'fpm_path': settings.WEBAPPS_PHPFPM_POOL_PATH % context,
}) })
return context return context

View File

@ -39,23 +39,6 @@ class WebApp(models.Model):
def get_options(self): def get_options(self):
return { opt.name: opt.value for opt in self.options.all() } return { opt.name: opt.value for opt in self.options.all() }
def get_php_init_vars(self):
init_vars = []
options = WebAppOption.objects.filter(webapp__type=self.type)
for opt in options.filter(webapp__account=self.account):
name = opt.name.replace('PHP-', '')
value = "%s" % opt.value
init_vars.append((name, value))
enabled_functions = self.options.filter(name='enabled_functions')
if enabled_functions:
enabled_functions = enabled_functions.get().value.split(',')
disabled_functions = []
for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
if function not in enabled_functions:
disabled_functions.append(function)
init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
return init_vars
def get_fpm_port(self): def get_fpm_port(self):
return settings.WEBAPPS_FPM_START_PORT + self.account.pk return settings.WEBAPPS_FPM_START_PORT + self.account.pk

View File

@ -9,10 +9,16 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN',
# '/var/run/%(user)s-%(app_name)s.sock') # '/var/run/%(user)s-%(app_name)s.sock')
'127.0.0.1:%(fpm_port)s') '127.0.0.1:%(fpm_port)s')
WEBAPPS_FPM_START_PORT = getattr(settings, 'WEBAPPS_FPM_START_PORT', 10000) WEBAPPS_FPM_START_PORT = getattr(settings, 'WEBAPPS_FPM_START_PORT', 10000)
WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
'/etc/php5/fpm/pool.d/%(app_name)s.conf')
WEBAPPS_FCGID_PATH = getattr(settings, 'WEBAPPS_FCGID_PATH', WEBAPPS_FCGID_PATH = getattr(settings, 'WEBAPPS_FCGID_PATH',
'/home/httpd/fcgid/%(user)s/%(type)s-wrapper') '/home/httpd/fcgid/%(app_name)s-wrapper')
WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', { WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', {
@ -166,10 +172,26 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', {
WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [ WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [
'exec', 'passthru', 'shell_exec', 'system', 'proc_open', 'popen', 'curl_exec', 'exec',
'curl_multi_exec', 'show_source', 'pcntl_exec', 'proc_close', 'passthru',
'proc_get_status', 'proc_nice', 'proc_terminate', 'ini_alter', 'virtual', 'shell_exec',
'openlog', 'escapeshellcmd', 'escapeshellarg', 'dl' 'system',
'proc_open',
'popen',
'curl_exec',
'curl_multi_exec',
'show_source',
'pcntl_exec',
'proc_close',
'proc_get_status',
'proc_nice',
'proc_terminate',
'ini_alter',
'virtual',
'openlog',
'escapeshellcmd',
'escapeshellarg',
'dl'
]) ])
@ -181,9 +203,6 @@ WEBAPPS_WORDPRESSMU_ADMIN_PASSWORD = getattr(settings, 'WEBAPPS_WORDPRESSMU_ADMI
'secret') 'secret')
WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH = setattr(settings, 'WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH', WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH = setattr(settings, 'WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH',
'/home/httpd/htdocs/wikifarm/template.tar.gz') '/home/httpd/htdocs/wikifarm/template.tar.gz')
@ -195,6 +214,3 @@ WEBAPPS_DOKUWIKIMU_FARM_PATH = getattr(settings, 'WEBAPPS_DOKUWIKIMU_FARM_PATH',
WEBAPPS_DRUPAL_SITES_PATH = getattr(settings, 'WEBAPPS_DRUPAL_SITES_PATH', WEBAPPS_DRUPAL_SITES_PATH = getattr(settings, 'WEBAPPS_DRUPAL_SITES_PATH',
'/home/httpd/htdocs/drupal-mu/sites/%(app_name)s') '/home/httpd/htdocs/drupal-mu/sites/%(app_name)s')
WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
'/etc/php5/fpm/pool.d')

View File

@ -36,39 +36,14 @@ class WebAppMixin(object):
djsettings.DEBUG = True djsettings.DEBUG = True
def add_route(self): def add_route(self):
# backends = [ server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER)
# # TODO MU apps on SaaS? backend = SystemUserBackend.get_name()
# backends.awstats.AwstatsBackend, Route.objects.get_or_create(backend=backend, match=True, host=server)
# backends.dokuwikimu.DokuWikiMuBackend, backend = self.backend.get_name()
# backends.drupalmu.DrupalMuBackend, match = 'webapp.type == "%s"' % self.type_value
# backends.phpfcgid.PHPFcgidBackend, Route.objects.create(backend=backend, match=match, host=server)
# backends.phpfpm.PHPFPMBackend,
# backends.static.StaticBackend,
# backends.wordpressmu.WordpressMuBackend,
# ]
server = Server.objects.create(name=self.MASTER_SERVER)
for backend in [SystemUserBackend, self.backend]:
backend = backend.get_name()
Route.objects.create(backend=backend, match=True, host=server)
def test_add(self): def upload_webapp(self, name):
name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(name)
self.validate_add_webapp(name)
# self.addCleanup(self.delete, username)
class StaticWebAppMixin(object):
backend = backends.static.StaticBackend
type_value = 'static'
token = random_ascii(100)
page = (
'index.html',
'<html>Hello World! %s </html>\n' % token,
'<html>Hello World! %s </html>\n' % token,
)
def validate_add_webapp(self, name):
try: try:
ftp = ftplib.FTP(self.MASTER_SERVER) ftp = ftplib.FTP(self.MASTER_SERVER)
ftp.login(user=self.account.username, passwd=self.account_password) ftp.login(user=self.account.username, passwd=self.account_password)
@ -81,6 +56,23 @@ class StaticWebAppMixin(object):
finally: finally:
ftp.close() ftp.close()
def test_add(self):
name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(name)
self.addCleanup(self.delete_webapp, name)
self.upload_webapp(name)
class StaticWebAppMixin(object):
backend = backends.static.StaticBackend
type_value = 'static'
token = random_ascii(100)
page = (
'index.html',
'<html>Hello World! %s </html>\n' % token,
'<html>Hello World! %s </html>\n' % token,
)
class PHPFcidWebAppMixin(StaticWebAppMixin): class PHPFcidWebAppMixin(StaticWebAppMixin):
backend = backends.phpfcgid.PHPFcgidBackend backend = backends.phpfcgid.PHPFcgidBackend
@ -111,12 +103,11 @@ class RESTWebAppMixin(object):
@save_response_on_error @save_response_on_error
def add_webapp(self, name, options=[]): def add_webapp(self, name, options=[]):
self.rest.webapps.create(name=name, type=self.type_value) self.rest.webapps.create(name=name, type=self.type_value, options=options)
@save_response_on_error @save_response_on_error
def delete_webapp(self, name): def delete_webapp(self, name):
list = self.rest.lists.retrieve(name=name).get() self.rest.webapps.retrieve(name=name).delete()
list.delete()
class AdminWebAppMixin(WebAppMixin): class AdminWebAppMixin(WebAppMixin):
@ -125,54 +116,25 @@ class AdminWebAppMixin(WebAppMixin):
self.admin_login() self.admin_login()
# create main user # create main user
self.save_systemuser() self.save_systemuser()
# TODO save_account()
@snapshot_on_error
def save_systemuser(self):
url = ''
@snapshot_on_error @snapshot_on_error
def add(self, name, password, admin_email): def add(self, name, password, admin_email):
url = self.live_server_url + reverse('admin:mails_List_add')
self.selenium.get(url)
account_input = self.selenium.find_element_by_id('id_account')
account_select = Select(account_input)
account_select.select_by_value(str(self.account.pk))
name_field = self.selenium.find_element_by_id('id_name')
name_field.send_keys(username)
password_field = self.selenium.find_element_by_id('id_password1')
password_field.send_keys(password)
password_field = self.selenium.find_element_by_id('id_password2')
password_field.send_keys(password)
if quota is not None:
quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
quota_field = self.selenium.find_element_by_id(quota_id)
quota_field.clear()
quota_field.send_keys(quota)
if filtering is not None:
filtering_input = self.selenium.find_element_by_id('id_filtering')
filtering_select = Select(filtering_input)
filtering_select.select_by_value("CUSTOM")
filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
filtering_inline.click()
time.sleep(0.5)
filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
filtering_field.send_keys(filtering)
name_field.submit()
self.assertNotEqual(url, self.selenium.current_url)
class RESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass pass
class RESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass pass
class RESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase): class PHPFcidRESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass
class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass pass

View File

@ -112,11 +112,11 @@ class Apache2Backend(ServiceController):
directives += "SecRuleRemoveById %d" % rule directives += "SecRuleRemoveById %d" % rule
for modsecurity in site.options.filter(name='sec_rule_off'): for modsecurity in site.options.filter(name='sec_rule_off'):
directives += ( directives += textwrap.dedent("""\
"<LocationMatch %s>\n" <LocationMatch %s>
" SecRuleEngine Off\n" SecRuleEngine Off
"</LocationMatch>\n" % modsecurity.value </LocationMatch>
) """ % modsecurity.value)
return directives return directives
def get_protections(self, site): def get_protections(self, site):

View File

@ -39,7 +39,9 @@ class Website(models.Model):
@cached @cached
def get_options(self): def get_options(self):
return { opt.name: opt.value for opt in self.options.all() } return {
opt.name: opt.value for opt in self.options.all()
}
@property @property
def protocol(self): def protocol(self):
@ -81,12 +83,15 @@ class Content(models.Model):
class Meta: class Meta:
unique_together = ('website', 'path') unique_together = ('website', 'path')
def __unicode__(self):
try:
return self.website.name + self.path
except Website.DoesNotExist:
return self.path
def clean(self): def clean(self):
if not self.path.startswith('/'): if not self.path.startswith('/'):
self.path = '/' + self.path self.path = '/' + self.path
def __unicode__(self):
return self.website.name + self.path
services.register(Website) services.register(Website)

View File

@ -11,6 +11,9 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer):
model = Content model = Content
fields = ('webapp', 'path') fields = ('webapp', 'path')
def get_identity(self, data):
return '%s-%s' % (data.get('website'), data.get('path'))
class WebsiteSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer): class WebsiteSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
contents = ContentSerializer(required=False, many=True, allow_add_remove=True, contents = ContentSerializer(required=False, many=True, allow_add_remove=True,

View File

@ -38,9 +38,9 @@ class WebsiteMixin(WebAppMixin):
super(WebsiteMixin, self).add_route() super(WebsiteMixin, self).add_route()
server = Server.objects.get() server = Server.objects.get()
backend = backends.apache.Apache2Backend.get_name() backend = backends.apache.Apache2Backend.get_name()
Route.objects.create(backend=backend, match=True, host=server) Route.objects.get_or_create(backend=backend, match=True, host=server)
backend = Bind9MasterDomainBackend.get_name() backend = Bind9MasterDomainBackend.get_name()
Route.objects.create(backend=backend, match=True, host=server) Route.objects.get_or_create(backend=backend, match=True, host=server)
def validate_add_website(self, name, domain): def validate_add_website(self, name, domain):
url = 'http://%s/%s' % (domain.name, self.page[0]) url = 'http://%s/%s' % (domain.name, self.page[0])
@ -54,9 +54,11 @@ class WebsiteMixin(WebAppMixin):
self.save_domain(domain) self.save_domain(domain)
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value) webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp) self.add_webapp(webapp)
self.validate_add_webapp(webapp) self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
website = '%s_website' % random_ascii(10) website = '%s_website' % random_ascii(10)
self.add_website(website, domain, webapp) self.add_website(website, domain, webapp)
self.addCleanup(self.delete_website, website)
self.validate_add_website(website, domain) self.validate_add_website(website, domain)
@ -65,21 +67,82 @@ class RESTWebsiteMixin(RESTWebAppMixin):
def save_domain(self, domain): def save_domain(self, domain):
self.rest.domains.retrieve().get().save() self.rest.domains.retrieve().get().save()
def add_website(self, name, domain, webapp): @save_response_on_error
domain = self.rest.domains.retrieve().get() def add_website(self, name, domain, webapp, path='/'):
webapp = self.rest.webapps.retrieve().get() domain = self.rest.domains.retrieve(name=domain).get()
self.rest.websites.create(name=name, domains=[domain.url], contents=[{'webapp': webapp.url}]) webapp = self.rest.webapps.retrieve(name=webapp).get()
contents = [{
'webapp': webapp.url,
'path': path
}]
self.rest.websites.create(name=name, domains=[domain.url], contents=contents)
@save_response_on_error
def delete_website(self, name):
print 'hola'
pass
self.rest.websites.retrieve(name=name).delete()
# self.rest.websites.retrieve(name=name).delete()
@save_response_on_error
def add_content(self, website, webapp, path):
website = self.rest.websites.retrieve(name=website).get()
webapp = self.rest.webapps.retrieve(name=webapp).get()
website.contents.append({
'webapp': webapp.url,
'path': path,
})
website.save()
class RESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
def test_mix_webapps(self):
domain_name = '%sdomain.lan' % random_ascii(10)
domain = Domain.objects.create(name=domain_name, account=self.account)
domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR)
self.save_domain(domain)
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
website = '%s_website' % random_ascii(10)
self.add_website(website, domain, webapp)
self.addCleanup(self.delete_website, website)
self.validate_add_website(website, domain)
self.type_value = PHPFcidWebAppMixin.type_value
self.backend = PHPFcidWebAppMixin.backend
self.page = PHPFcidWebAppMixin.page
self.add_route()
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
path = '/%s' % webapp
self.add_content(website, webapp, path)
url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
self.assertEqual(self.page[2], requests.get(url).content)
self.type_value = PHPFPMWebAppMixin.type_value
self.backend = PHPFPMWebAppMixin.backend
self.page = PHPFPMWebAppMixin.page
self.add_route()
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
path = '/%s' % webapp
self.add_content(website, webapp, path)
url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
self.assertEqual(self.page[2], requests.get(url).content)
class PHPFcidRESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass pass
class RESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase): class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass
class RESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass pass
#class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase): #class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase):

View File

@ -27,6 +27,7 @@
{% if not is_popup %} {% if not is_popup %}
{% admin_tools_render_menu %} {% admin_tools_render_menu %}
{% endif %} {% endif %}
</div>
{% endif %} {% endif %}
{% endblock %} {% endblock %}

View File

@ -84,6 +84,12 @@ The goal of this setup is having a high-performance state-of-the-art deployment
</Directory> </Directory>
# TODO pool per website or pool per user? memory consumption
events.mechanism = epoll
# TODO multiple master processes, opcache is held in master, and reload/restart affects all pools
# http://mattiasgeniar.be/2014/04/09/a-better-way-to-run-php-fpm/
TODO CHRoot TODO CHRoot
https://andrewbevitt.com/tutorials/apache-varnish-chrooted-php-fpm-wordpress-virtual-host/ https://andrewbevitt.com/tutorials/apache-varnish-chrooted-php-fpm-wordpress-virtual-host/
@ -92,10 +98,10 @@ TODO CHRoot
[vhost] [vhost]
istemplate = 1 istemplate = 1
listen.mode = 0660 listen.mode = 0660
pm = ondemand
pm.max_children = 5 pm.max_children = 5
pm.start_servers = 1 pm.process_idle_timeout = 10s
pm.min_spare_servers = 1 pm.max_requests = 200
pm.max_spare_servers = 2
' > /etc/php5/fpm/conf.d/vhost-template.conf ' > /etc/php5/fpm/conf.d/vhost-template.conf
``` ```