pedro/django-orchestra-test
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Improvements on databases, webapps and websites

Browse source
This commit is contained in:
Marc 2014-10-14 13:50:19 +00:00
parent 920f8efcd5
commit 4c7c5b5505
21 changed files with 380 additions and 299 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
TODO.md
View file

@ -138,3 +138,12 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
* DN: Transaction atomicity and backend failure
* SaaS Icons
* offer to create mailbox on account creation
* init.d celery scripts
-# Required-Start: $network $local_fs $remote_fs postgresql celeryd
-# Required-Stop: $network $local_fs $remote_fs postgresql celeryd
* POST only fields (account, username, name) etc

3
orchestra/apps/accounts/models.py
View file

@ -72,7 +72,8 @@ class Account(auth.AbstractBaseUser):
def disable(self):
self.is_active = False
# self.save(update_fields=['is_active'])
self.save(update_fields=['is_active'])
# Trigger save() on related objects that depend on this account
for rel in self._meta.get_all_related_objects():
if not rel.model in services:
continue

60
orchestra/apps/databases/admin.py
View file

@ -4,68 +4,24 @@ from django.contrib.auth.admin import UserAdmin
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
from orchestra.admin import ExtendedModelAdmin
from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin
from orchestra.admin.utils import admin_link
from orchestra.apps.accounts.admin import AccountAdminMixin, SelectAccountAdminMixin
from .forms import (DatabaseUserChangeForm, DatabaseUserCreationForm,
DatabaseCreationForm)
from .models import Database, Role, DatabaseUser
class UserInline(admin.TabularInline):
model = Role
verbose_name_plural = _("Users")
readonly_fields = ('user_link',)
extra = 0
user_link = admin_link('user')
def formfield_for_dbfield(self, db_field, **kwargs):
""" Make value input widget bigger """
if db_field.name == 'user':
users = db_field.rel.to.objects.filter(type=self.parent_object.type)
kwargs['queryset'] = users.filter(account=self.account)
return super(UserInline, self).formfield_for_dbfield(db_field, **kwargs)
class PermissionInline(AccountAdminMixin, admin.TabularInline):
model = Role
verbose_name_plural = _("Permissions")
readonly_fields = ('database_link',)
extra = 0
filter_by_account_fields = ['database']
database_link = admin_link('database', popup=True)
def formfield_for_dbfield(self, db_field, **kwargs):
""" Make value input widget bigger """
formfield = super(PermissionInline, self).formfield_for_dbfield(db_field, **kwargs)
if db_field.name == 'database':
# Hack widget render in order to append ?type='db_type' to the add url
db_type = self.parent_object.type
old_render = formfield.widget.render
def render(*args, **kwargs):
output = old_render(*args, **kwargs)
output = output.replace('/add/?', '/add/?type=%s&' % db_type)
return mark_safe(output)
formfield.widget.render = render
formfield.queryset = formfield.queryset.filter(type=db_type)
return formfield
from .forms import DatabaseCreationForm, DatabaseUserChangeForm, DatabaseUserCreationForm
from .models import Database, DatabaseUser
class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
list_display = ('name', 'type', 'account_link')
list_filter = ('type',)
search_fields = ['name']
inlines = [UserInline]
add_inlines = []
change_readonly_fields = ('name', 'type')
extra = 1
fieldsets = (
(None, {
'classes': ('extrapretty',),
'fields': ('account_link', 'name', 'type'),
'fields': ('account_link', 'name', 'type', 'users'),
}),
)
add_fieldsets = (
@ -92,22 +48,20 @@ class DatabaseAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
user = DatabaseUser(
username=form.cleaned_data['username'],
type=obj.type,
account_id = obj.account.pk,
account_id=obj.account.pk,
)
user.set_password(form.cleaned_data["password1"])
user.save()
Role.objects.create(database=obj, user=user, is_owner=True)
obj.users.add(user)
class DatabaseUserAdmin(SelectAccountAdminMixin, ExtendedModelAdmin):
class DatabaseUserAdmin(SelectAccountAdminMixin, ChangePasswordAdminMixin, ExtendedModelAdmin):
list_display = ('username', 'type', 'account_link')
list_filter = ('type',)
search_fields = ['username']
form = DatabaseUserChangeForm
add_form = DatabaseUserCreationForm
change_readonly_fields = ('username', 'type')
inlines = [PermissionInline]
add_inlines = []
fieldsets = (
(None, {
'classes': ('extrapretty',),

68
orchestra/apps/databases/backends.py
View file

@ -1,3 +1,5 @@
import textwrap
from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
@ -14,12 +16,12 @@ class MySQLBackend(ServiceController):
if database.type == database.MYSQL:
context = self.get_context(database)
self.append(
"mysql -e 'CREATE DATABASE `%(database)s`;'" % context
)
self.append(
"mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* "
" TO \"%(owner)s\"@\"%(host)s\" WITH GRANT OPTION;'" % context
"mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
)
self.append(textwrap.dedent("""\
mysql -e 'GRANT ALL PRIVILEGES ON `%(database)s`.* TO "%(owner)s"@"%(host)s" WITH GRANT OPTION;' \
""" % context
))
def delete(self, database):
if database.type == database.MYSQL:
@ -44,20 +46,25 @@ class MySQLUserBackend(ServiceController):
def save(self, user):
if user.type == user.MYSQL:
context = self.get_context(user)
self.append(
"mysql -e 'CREATE USER \"%(username)s\"@\"%(host)s\";' || true" % context
)
self.append(
"mysql -e 'UPDATE mysql.user SET Password=\"%(password)s\" "
" WHERE User=\"%(username)s\";'" % context
)
self.append(textwrap.dedent("""\
mysql -e 'CREATE USER "%(username)s"@"%(host)s";' || true \
""" % context
))
self.append(textwrap.dedent("""\
mysql -e 'UPDATE mysql.user SET Password="%(password)s" WHERE User="%(username)s";' \
""" % context
))
def delete(self, user):
if user.type == user.MYSQL:
context = self.get_context(database)
self.append(
"mysql -e 'DROP USER \"%(username)s\"@\"%(host)s\";'" % context
)
self.append(textwrap.dedent("""\
mysql -e 'DROP USER "%(username)s"@"%(host)s";' \
""" % context
))
def commit(self):
self.append("mysql -e 'FLUSH PRIVILEGES;'")
def get_context(self, user):
return {
@ -78,27 +85,28 @@ class MysqlDisk(ServiceMonitor):
def exceeded(self, db):
context = self.get_context(db)
self.append("mysql -e '"
"UPDATE db SET Insert_priv=\"N\", Create_priv=\"N\""
" WHERE Db=\"%(db_name)s\";'" % context
)
self.append(textwrap.dedent("""\
mysql -e 'UPDATE db SET Insert_priv="N", Create_priv="N" WHERE Db="%(db_name)s";' \
""" % context
))
def recovery(self, db):
context = self.get_context(db)
self.append("mysql -e '"
"UPDATE db SET Insert_priv=\"Y\", Create_priv=\"Y\""
" WHERE Db=\"%(db_name)s\";'" % context
)
self.append(textwrap.dedent("""\
mysql -e 'UPDATE db SET Insert_priv="Y", Create_priv="Y" WHERE Db="%(db_name)s";' \
""" % context
))
def monitor(self, db):
context = self.get_context(db)
self.append(
"echo %(db_id)s $(mysql -B -e '"
" SELECT sum( data_length + index_length ) \"Size\"\n"
" FROM information_schema.TABLES\n"
" WHERE table_schema=\"gisp\"\n"
" GROUP BY table_schema;' | tail -n 1)" % context
)
self.append(textwrap.dedent("""\
echo %(db_id)s $(mysql -B -e '"
SELECT sum( data_length + index_length ) "Size"
FROM information_schema.TABLES
WHERE table_schema = "gisp"
GROUP BY table_schema;' | tail -n 1) \
""" % context
))
def get_context(self, db):
return {

23
orchestra/apps/databases/forms.py
View file

@ -6,7 +6,7 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.core.validators import validate_password
from .models import DatabaseUser, Database, Role
from .models import DatabaseUser, Database
class DatabaseUserCreationForm(forms.ModelForm):
@ -27,13 +27,6 @@ class DatabaseUserCreationForm(forms.ModelForm):
msg = _("The two password fields didn't match.")
raise forms.ValidationError(msg)
return password2
def save(self, commit=True):
user = super(DatabaseUserCreationForm, self).save(commit=False)
# user.set_password(self.cleaned_data["password1"])
# if commit:
# user.save()
return user
class DatabaseCreationForm(DatabaseUserCreationForm):
@ -86,20 +79,6 @@ class DatabaseCreationForm(DatabaseUserCreationForm):
elif not (cleaned_data['username'] or cleaned_data['user']):
raise forms.ValidationError(msg)
return cleaned_data
def save(self, commit=True):
db = super(DatabaseUserCreationForm, self).save(commit=False)
# if commit:
# user = self.cleaned_data['user']
# if not user:
# user = DatabaseUser(
# username=self.cleaned_data['username'],
# type=self.cleaned_data['type'],
# )
# user.set_password(self.cleaned_data["password1"])
# user.save()
# role, __ = Role.objects.get_or_create(database=db, user=user)
return db
class ReadOnlySQLPasswordHashField(ReadOnlyPasswordHashField):

55
orchestra/apps/databases/models.py
View file

@ -16,8 +16,8 @@ class Database(models.Model):
name = models.CharField(_("name"), max_length=64, # MySQL limit
validators=[validators.validate_name])
users = models.ManyToManyField('databases.DatabaseUser',
verbose_name=_("users"),
through='databases.Role', related_name='users')
verbose_name=_("users"),related_name='databases')
# through='databases.Role',
type = models.CharField(_("type"), max_length=32,
choices=settings.DATABASES_TYPE_CHOICES,
default=settings.DATABASES_DEFAULT_TYPE)
@ -32,29 +32,38 @@ class Database(models.Model):
@property
def owner(self):
return self.roles.get(is_owner=True).user
""" database owner is the first user related to it """
# Accessing intermediary model to get which is the first user
users = Database.users.through.objects.filter(database_id=self.id)
return users.order_by('-id').first().databaseuser
class Role(models.Model):
database = models.ForeignKey(Database, verbose_name=_("database"),
related_name='roles')
user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
related_name='roles')
is_owner = models.BooleanField(_("owner"), default=False)
class Meta:
unique_together = ('database', 'user')
def __unicode__(self):
return "%s@%s" % (self.user, self.database)
def clean(self):
if self.user.type != self.database.type:
msg = _("Database and user type doesn't match")
raise validators.ValidationError(msg)
roles = self.database.roles.values('id')
if not roles or (len(roles) == 1 and roles[0].id == self.id):
self.is_owner = True
Database.users.through._meta.unique_together = (('database', 'databaseuser'),)
#class Role(models.Model):
# database = models.ForeignKey(Database, verbose_name=_("database"),
# related_name='roles')
# user = models.ForeignKey('databases.DatabaseUser', verbose_name=_("user"),
# related_name='roles')
## is_owner = models.BooleanField(_("owner"), default=False)
#
# class Meta:
# unique_together = ('database', 'user')
#
# def __unicode__(self):
# return "%s@%s" % (self.user, self.database)
#
# @property
# def is_owner(self):
# return datatase.owner == self
#
# def clean(self):
# if self.user.type != self.database.type:
# msg = _("Database and user type doesn't match")
# raise validators.ValidationError(msg)
# roles = self.database.roles.values('id')
# if not roles or (len(roles) == 1 and roles[0].id == self.id):
# self.is_owner = True
class DatabaseUser(models.Model):

41
orchestra/apps/databases/serializers.py
View file

@ -5,36 +5,47 @@ from rest_framework import serializers
from orchestra.apps.accounts.serializers import AccountSerializerMixin
from orchestra.core.validators import validate_password
from .models import Database, DatabaseUser, Role
from .models import Database, DatabaseUser
class UserRoleSerializer(serializers.HyperlinkedModelSerializer):
class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Role
fields = ('user', 'is_owner',)
class RoleSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Role
fields = ('database', 'is_owner',)
model = DatabaseUser
fields = ('url', 'username')
def from_native(self, data, files=None):
return DatabaseUser.objects.get(username=data['username'])
class DatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
roles = UserRoleSerializer(many=True)
users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True)
class Meta:
model = Database
fields = ('url', 'name', 'type', 'roles')
fields = ('url', 'name', 'type', 'users')
class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Database
fields = ('url', 'name',)
def from_native(self, data, files=None):
return Database.objects.get(name=data['name'])
class DatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
password = serializers.CharField(max_length=128, label=_('Password'),
validators=[validate_password], write_only=True,
widget=widgets.PasswordInput)
roles = RoleSerializer(many=True, read_only=True)
databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False)
class Meta:
model = DatabaseUser
fields = ('url', 'username', 'password', 'type', 'roles')
write_only_fields = ('username',)
fields = ('url', 'username', 'password', 'type', 'databases')
def save_object(self, obj, **kwargs):
# FIXME this method will be called when saving nested serializers :(
if not obj.pk:
obj.set_password(obj.password)
super(DatabaseUserSerializer, self).save_object(obj, **kwargs)

44
orchestra/apps/databases/tests/functional_tests/tests.py
View file

@ -1,5 +1,6 @@
import MySQLdb
import os
import socket
import time
from functools import partial
@ -57,6 +58,17 @@ class DatabaseTestMixin(object):
password = '@!?%spppP001' % random_ascii(5)
self.add(dbname, username, password)
self.validate_create_table(dbname, username, password)
def test_change_password(self):
dbname = '%s_database' % random_ascii(5)
username = '%s_dbuser' % random_ascii(5)
password = '@!?%spppP001' % random_ascii(5)
self.add(dbname, username, password)
self.validate_create_table(dbname, username, password)
new_password = '@!?%spppP001' % random_ascii(5)
self.change_password(username, new_password)
self.validate_login_error(dbname, username, password)
self.validate_create_table(dbname, username, new_password)
class MySQLBackendMixin(object):
@ -64,7 +76,11 @@ class MySQLBackendMixin(object):
def setUp(self):
super(MySQLBackendMixin, self).setUp()
settings.DATABASES_DEFAULT_HOST = '10.228.207.207'
# Get local ip address used to reach self.MASTER_SERVER
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect((self.MASTER_SERVER, 22))
settings.DATABASES_DEFAULT_HOST = s.getsockname()[0]
s.close()
def add_route(self):
server = Server.objects.create(name=self.MASTER_SERVER)
@ -78,7 +94,11 @@ class MySQLBackendMixin(object):
def validate_create_table(self, name, username, password):
db = MySQLdb.connect(host=self.MASTER_SERVER, port=3306, user=username, passwd=password, db=name)
cur = db.cursor()
cur.execute('CREATE TABLE test ( id INT ) ;')
cur.execute('CREATE TABLE %s ( id INT ) ;' % random_ascii(20))
def validate_login_error(self, dbname, username, password):
self.assertRaises(MySQLdb.OperationalError,
self.validate_create_table, dbname, username, password)
def validate_delete(self, name, username, password):
self.asseRaises(MySQLdb.ConnectionError,
@ -92,9 +112,16 @@ class RESTDatabaseMixin(DatabaseTestMixin):
@save_response_on_error
def add(self, dbname, username, password):
user = self.rest.databaseusers.create(username=username, password=password)
# TODO fucking nested objects
self.rest.databases.create(name=dbname, roles=[{'user': user.url}], type=self.db_type)
user = self.rest.databaseusers.create(username=username, password=password, type=self.db_type)
users = [{
'username': user.username
}]
self.rest.databases.create(name=dbname, users=users, type=self.db_type)
@save_response_on_error
def change_password(self, username, password):
user = self.rest.databaseusers.retrieve(username=username).get()
user.set_password(password)
class AdminDatabaseMixin(DatabaseTestMixin):
@ -129,11 +156,16 @@ class AdminDatabaseMixin(DatabaseTestMixin):
def delete(self, dbname):
db = Database.objects.get(name=dbname)
self.admin_delete(db)
@snapshot_on_error
def delete_user(self, username):
user = DatabaseUser.objects.get(username=username)
self.admin_delete(user)
@snapshot_on_error
def change_password(self, username, password):
user = DatabaseUser.objects.get(username=username)
self.admin_change_password(user, password)
class RESTMysqlDatabaseTest(MySQLBackendMixin, RESTDatabaseMixin, BaseLiveServerTestCase):

3
orchestra/apps/orchestration/methods.py
View file

@ -37,7 +37,8 @@ def BashSSH(backend, log, server, cmds):
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
addr = server.get_address()
try:
ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH)
# TODO timeout
ssh.connect(addr, username='root', key_filename=settings.ORCHESTRATION_SSH_KEY_PATH, timeout=10)
except socket.error:
logger.error('%s timed out on %s' % (backend, server))
log.state = BackendLog.TIMEOUT

27
orchestra/apps/webapps/backends/__init__.py
View file

@ -1,6 +1,7 @@
import pkgutil
import textwrap
class WebAppServiceMixin(object):
model = 'webapps.WebApp'
@ -16,6 +17,30 @@ class WebAppServiceMixin(object):
done
""" % context))
def get_php_init_vars(self, webapp, per_account=False):
"""
process php options for inclusion on php.ini
per_account=True merges all (account, webapp.type) options
"""
init_vars = []
options = webapp.options.all()
if per_account:
options = webapp.account.webapps.filter(webapp_type=webapp.type)
for opt in options:
name = opt.name.replace('PHP-', '')
value = "%s" % opt.value
init_vars.append((name, value))
enabled_functions = []
for value in options.filter(name='enabled_functions').values_list('value', flat=True):
enabled_functions += enabled_functions.get().value.split(',')
if enabled_functions:
disabled_functions = []
for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
if function not in enabled_functions:
disabled_functions.append(function)
init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
return init_vars
def delete_webapp_dir(self, context):
self.append("rm -fr %(app_path)s" % context)
@ -30,5 +55,7 @@ class WebAppServiceMixin(object):
}
for __, module_name, __ in pkgutil.walk_packages(__path__):
# sorry for the exec(), but Import module function fails :(
exec('from . import %s' % module_name)

4
orchestra/apps/webapps/backends/phpfcgid.py
View file

@ -10,6 +10,7 @@ from .. import settings
class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
""" Per-webapp fcgid application """
verbose_name = _("PHP-Fcgid")
def save(self, webapp):
@ -27,6 +28,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
def delete(self, webapp):
context = self.get_context(webapp)
self.append("rm '%(wrapper_path)s'" % context)
self.delete_webapp_dir(context)
def commit(self):
@ -35,7 +37,7 @@ class PHPFcgidBackend(WebAppServiceMixin, ServiceController):
def get_context(self, webapp):
context = super(PHPFcgidBackend, self).get_context(webapp)
init_vars = webapp.get_php_init_vars()
init_vars = self.get_php_init_vars(webapp)
if init_vars:
init_vars = [ '%s="%s"' % (k,v) for v,k in init_vars.iteritems() ]
init_vars = ', -d '.join(init_vars)

53
orchestra/apps/webapps/backends/phpfpm.py
View file

@ -1,4 +1,5 @@
import os
import textwrap
from django.template import Template, Context
from django.utils.translation import ugettext_lazy as _
@ -10,49 +11,57 @@ from .. import settings
class PHPFPMBackend(WebAppServiceMixin, ServiceController):
""" Per-webapp php application """
verbose_name = _("PHP-FPM")
def save(self, webapp):
context = self.get_context(webapp)
self.create_webapp_dir(context)
self.append(
"{ echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s - ; } ||"
" { echo -e '%(fpm_config)s' > %(fpm_path)s; UPDATEDFPM=1; }" % context
)
self.append(textwrap.dedent("""\
{
echo -e '%(fpm_config)s' | diff -N -I'^\s*;;' %(fpm_path)s -
} || {
echo -e '%(fpm_config)s' > %(fpm_path)s
UPDATEDFPM=1
}""" % context))
def delete(self, webapp):
context = self.get_context(webapp)
self.append("rm '%(fpm_config)s'" % context)
self.delete_webapp_dir(context)
def commit(self):
super(PHPFPMBackend, self).commit()
self.append('[[ $UPDATEDFPM == 1 ]] && service php5-fpm reload')
self.append(textwrap.dedent("""
[[ $UPDATEDFPM == 1 ]] && {
service php5-fpm start
service php5-fpm reload
}"""))
def get_context(self, webapp):
context = super(PHPFPMBackend, self).get_context(webapp)
context.update({
'init_vars': webapp.get_php_init_vars(),
'init_vars': self.get_php_init_vars(webapp),
'fpm_port': webapp.get_fpm_port(),
})
context['fpm_listen'] = settings.WEBAPPS_FPM_LISTEN % context
fpm_config = Template(
";; {{ banner }}\n"
"[{{ user }}]\n"
"user = {{ user }}\n"
"group = {{ group }}\n\n"
"listen = {{ fpm_listen | safe }}\n"
"listen.owner = {{ user }}\n"
"listen.group = {{ group }}\n"
"pm = ondemand\n"
"pm.max_children = 4\n"
"{% for name,value in init_vars.iteritems %}"
"php_admin_value[{{ name | safe }}] = {{ value | safe }}\n"
"{% endfor %}"
)
fpm_file = '%(user)s.conf' % context
fpm_config = Template(textwrap.dedent("""\
;; {{ banner }}
[{{ user }}]
user = {{ user }}
group = {{ group }}
listen = {{ fpm_listen | safe }}
listen.owner = {{ user }}
listen.group = {{ group }}
pm = ondemand
pm.max_children = 4
{% for name,value in init_vars.iteritems %}
php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}"""
))
context.update({
'fpm_config': fpm_config.render(Context(context)),
'fpm_path': os.path.join(settings.WEBAPPS_PHPFPM_POOL_PATH, fpm_file),
'fpm_path': settings.WEBAPPS_PHPFPM_POOL_PATH % context,
})
return context

17
orchestra/apps/webapps/models.py
View file

@ -39,23 +39,6 @@ class WebApp(models.Model):
def get_options(self):
return { opt.name: opt.value for opt in self.options.all() }
def get_php_init_vars(self):
init_vars = []
options = WebAppOption.objects.filter(webapp__type=self.type)
for opt in options.filter(webapp__account=self.account):
name = opt.name.replace('PHP-', '')
value = "%s" % opt.value
init_vars.append((name, value))
enabled_functions = self.options.filter(name='enabled_functions')
if enabled_functions:
enabled_functions = enabled_functions.get().value.split(',')
disabled_functions = []
for function in settings.WEBAPPS_PHP_DISABLED_FUNCTIONS:
if function not in enabled_functions:
disabled_functions.append(function)
init_vars.append(('dissabled_functions', ','.join(disabled_functions)))
return init_vars
def get_fpm_port(self):
return settings.WEBAPPS_FPM_START_PORT + self.account.pk

38
orchestra/apps/webapps/settings.py
View file

@ -9,10 +9,16 @@ WEBAPPS_FPM_LISTEN = getattr(settings, 'WEBAPPS_FPM_LISTEN',
# '/var/run/%(user)s-%(app_name)s.sock')
'127.0.0.1:%(fpm_port)s')
WEBAPPS_FPM_START_PORT = getattr(settings, 'WEBAPPS_FPM_START_PORT', 10000)
WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
'/etc/php5/fpm/pool.d/%(app_name)s.conf')
WEBAPPS_FCGID_PATH = getattr(settings, 'WEBAPPS_FCGID_PATH',
'/home/httpd/fcgid/%(user)s/%(type)s-wrapper')
'/home/httpd/fcgid/%(app_name)s-wrapper')
WEBAPPS_TYPES = getattr(settings, 'WEBAPPS_TYPES', {
@ -166,10 +172,26 @@ WEBAPPS_OPTIONS = getattr(settings, 'WEBAPPS_OPTIONS', {
WEBAPPS_PHP_DISABLED_FUNCTIONS = getattr(settings, 'WEBAPPS_PHP_DISABLED_FUNCTION', [
'exec', 'passthru', 'shell_exec', 'system', 'proc_open', 'popen', 'curl_exec',
'curl_multi_exec', 'show_source', 'pcntl_exec', 'proc_close',
'proc_get_status', 'proc_nice', 'proc_terminate', 'ini_alter', 'virtual',
'openlog', 'escapeshellcmd', 'escapeshellarg', 'dl'
'exec',
'passthru',
'shell_exec',
'system',
'proc_open',
'popen',
'curl_exec',
'curl_multi_exec',
'show_source',
'pcntl_exec',
'proc_close',
'proc_get_status',
'proc_nice',
'proc_terminate',
'ini_alter',
'virtual',
'openlog',
'escapeshellcmd',
'escapeshellarg',
'dl'
])
@ -181,9 +203,6 @@ WEBAPPS_WORDPRESSMU_ADMIN_PASSWORD = getattr(settings, 'WEBAPPS_WORDPRESSMU_ADMI
'secret')
WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH = setattr(settings, 'WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH',
'/home/httpd/htdocs/wikifarm/template.tar.gz')
@ -195,6 +214,3 @@ WEBAPPS_DOKUWIKIMU_FARM_PATH = getattr(settings, 'WEBAPPS_DOKUWIKIMU_FARM_PATH',
WEBAPPS_DRUPAL_SITES_PATH = getattr(settings, 'WEBAPPS_DRUPAL_SITES_PATH',
'/home/httpd/htdocs/drupal-mu/sites/%(app_name)s')
WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
'/etc/php5/fpm/pool.d')

106
orchestra/apps/webapps/tests/functional_tests/tests.py
View file

@ -36,39 +36,14 @@ class WebAppMixin(object):
djsettings.DEBUG = True
def add_route(self):
# backends = [
# # TODO MU apps on SaaS?
# backends.awstats.AwstatsBackend,
# backends.dokuwikimu.DokuWikiMuBackend,
# backends.drupalmu.DrupalMuBackend,
# backends.phpfcgid.PHPFcgidBackend,
# backends.phpfpm.PHPFPMBackend,
# backends.static.StaticBackend,
# backends.wordpressmu.WordpressMuBackend,
# ]
server = Server.objects.create(name=self.MASTER_SERVER)
for backend in [SystemUserBackend, self.backend]:
backend = backend.get_name()
Route.objects.create(backend=backend, match=True, host=server)
server, __ = Server.objects.get_or_create(name=self.MASTER_SERVER)
backend = SystemUserBackend.get_name()
Route.objects.get_or_create(backend=backend, match=True, host=server)
backend = self.backend.get_name()
match = 'webapp.type == "%s"' % self.type_value
Route.objects.create(backend=backend, match=match, host=server)
def test_add(self):
name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(name)
self.validate_add_webapp(name)
# self.addCleanup(self.delete, username)
class StaticWebAppMixin(object):
backend = backends.static.StaticBackend
type_value = 'static'
token = random_ascii(100)
page = (
'index.html',
'<html>Hello World! %s </html>\n' % token,
'<html>Hello World! %s </html>\n' % token,
)
def validate_add_webapp(self, name):
def upload_webapp(self, name):
try:
ftp = ftplib.FTP(self.MASTER_SERVER)
ftp.login(user=self.account.username, passwd=self.account_password)
@ -80,6 +55,23 @@ class StaticWebAppMixin(object):
index.close()
finally:
ftp.close()
def test_add(self):
name = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(name)
self.addCleanup(self.delete_webapp, name)
self.upload_webapp(name)
class StaticWebAppMixin(object):
backend = backends.static.StaticBackend
type_value = 'static'
token = random_ascii(100)
page = (
'index.html',
'<html>Hello World! %s </html>\n' % token,
'<html>Hello World! %s </html>\n' % token,
)
class PHPFcidWebAppMixin(StaticWebAppMixin):
@ -111,12 +103,11 @@ class RESTWebAppMixin(object):
@save_response_on_error
def add_webapp(self, name, options=[]):
self.rest.webapps.create(name=name, type=self.type_value)
self.rest.webapps.create(name=name, type=self.type_value, options=options)
@save_response_on_error
def delete_webapp(self, name):
list = self.rest.lists.retrieve(name=name).get()
list.delete()
self.rest.webapps.retrieve(name=name).delete()
class AdminWebAppMixin(WebAppMixin):
@ -125,54 +116,25 @@ class AdminWebAppMixin(WebAppMixin):
self.admin_login()
# create main user
self.save_systemuser()
# TODO save_account()
@snapshot_on_error
def save_systemuser(self):
url = ''
@snapshot_on_error
def add(self, name, password, admin_email):
url = self.live_server_url + reverse('admin:mails_List_add')
self.selenium.get(url)
account_input = self.selenium.find_element_by_id('id_account')
account_select = Select(account_input)
account_select.select_by_value(str(self.account.pk))
name_field = self.selenium.find_element_by_id('id_name')
name_field.send_keys(username)
password_field = self.selenium.find_element_by_id('id_password1')
password_field.send_keys(password)
password_field = self.selenium.find_element_by_id('id_password2')
password_field.send_keys(password)
if quota is not None:
quota_id = 'id_resources-resourcedata-content_type-object_id-0-allocated'
quota_field = self.selenium.find_element_by_id(quota_id)
quota_field.clear()
quota_field.send_keys(quota)
if filtering is not None:
filtering_input = self.selenium.find_element_by_id('id_filtering')
filtering_select = Select(filtering_input)
filtering_select.select_by_value("CUSTOM")
filtering_inline = self.selenium.find_element_by_id('fieldsetcollapser0')
filtering_inline.click()
time.sleep(0.5)
filtering_field = self.selenium.find_element_by_id('id_custom_filtering')
filtering_field.send_keys(filtering)
name_field.submit()
self.assertNotEqual(url, self.selenium.current_url)
pass
class RESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
class StaticRESTWebAppTest(StaticWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass
class RESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
class PHPFcidRESTWebAppTest(PHPFcidWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass
class RESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
class PHPFPMRESTWebAppTest(PHPFPMWebAppMixin, RESTWebAppMixin, WebAppMixin, BaseLiveServerTestCase):
pass

10
orchestra/apps/websites/backends/apache.py
View file

@ -112,11 +112,11 @@ class Apache2Backend(ServiceController):
directives += "SecRuleRemoveById %d" % rule
for modsecurity in site.options.filter(name='sec_rule_off'):
directives += (
"<LocationMatch %s>\n"
" SecRuleEngine Off\n"
"</LocationMatch>\n" % modsecurity.value
)
directives += textwrap.dedent("""\
<LocationMatch %s>
SecRuleEngine Off
</LocationMatch>
""" % modsecurity.value)
return directives
def get_protections(self, site):

13
orchestra/apps/websites/models.py
View file

@ -39,7 +39,9 @@ class Website(models.Model):
@cached
def get_options(self):
return { opt.name: opt.value for opt in self.options.all() }
return {
opt.name: opt.value for opt in self.options.all()
}
@property
def protocol(self):
@ -81,12 +83,15 @@ class Content(models.Model):
class Meta:
unique_together = ('website', 'path')
def __unicode__(self):
try:
return self.website.name + self.path
except Website.DoesNotExist:
return self.path
def clean(self):
if not self.path.startswith('/'):
self.path = '/' + self.path
def __unicode__(self):
return self.website.name + self.path
services.register(Website)

3
orchestra/apps/websites/serializers.py
View file

@ -10,6 +10,9 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = Content
fields = ('webapp', 'path')
def get_identity(self, data):
return '%s-%s' % (data.get('website'), data.get('path'))
class WebsiteSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):

89
orchestra/apps/websites/tests/functional_tests/tests.py
View file

@ -38,9 +38,9 @@ class WebsiteMixin(WebAppMixin):
super(WebsiteMixin, self).add_route()
server = Server.objects.get()
backend = backends.apache.Apache2Backend.get_name()
Route.objects.create(backend=backend, match=True, host=server)
Route.objects.get_or_create(backend=backend, match=True, host=server)
backend = Bind9MasterDomainBackend.get_name()
Route.objects.create(backend=backend, match=True, host=server)
Route.objects.get_or_create(backend=backend, match=True, host=server)
def validate_add_website(self, name, domain):
url = 'http://%s/%s' % (domain.name, self.page[0])
@ -54,9 +54,11 @@ class WebsiteMixin(WebAppMixin):
self.save_domain(domain)
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.validate_add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
website = '%s_website' % random_ascii(10)
self.add_website(website, domain, webapp)
self.addCleanup(self.delete_website, website)
self.validate_add_website(website, domain)
@ -65,21 +67,82 @@ class RESTWebsiteMixin(RESTWebAppMixin):
def save_domain(self, domain):
self.rest.domains.retrieve().get().save()
def add_website(self, name, domain, webapp):
domain = self.rest.domains.retrieve().get()
webapp = self.rest.webapps.retrieve().get()
self.rest.websites.create(name=name, domains=[domain.url], contents=[{'webapp': webapp.url}])
@save_response_on_error
def add_website(self, name, domain, webapp, path='/'):
domain = self.rest.domains.retrieve(name=domain).get()
webapp = self.rest.webapps.retrieve(name=webapp).get()
contents = [{
'webapp': webapp.url,
'path': path
}]
self.rest.websites.create(name=name, domains=[domain.url], contents=contents)
@save_response_on_error
def delete_website(self, name):
print 'hola'
pass
self.rest.websites.retrieve(name=name).delete()
# self.rest.websites.retrieve(name=name).delete()
@save_response_on_error
def add_content(self, website, webapp, path):
website = self.rest.websites.retrieve(name=website).get()
webapp = self.rest.webapps.retrieve(name=webapp).get()
website.contents.append({
'webapp': webapp.url,
'path': path,
})
website.save()
class RESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
class StaticRESTWebsiteTest(RESTWebsiteMixin, StaticWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
def test_mix_webapps(self):
domain_name = '%sdomain.lan' % random_ascii(10)
domain = Domain.objects.create(name=domain_name, account=self.account)
domain.records.create(type=Record.A, value=self.MASTER_SERVER_ADDR)
self.save_domain(domain)
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
website = '%s_website' % random_ascii(10)
self.add_website(website, domain, webapp)
self.addCleanup(self.delete_website, website)
self.validate_add_website(website, domain)
self.type_value = PHPFcidWebAppMixin.type_value
self.backend = PHPFcidWebAppMixin.backend
self.page = PHPFcidWebAppMixin.page
self.add_route()
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
path = '/%s' % webapp
self.add_content(website, webapp, path)
url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
self.assertEqual(self.page[2], requests.get(url).content)
self.type_value = PHPFPMWebAppMixin.type_value
self.backend = PHPFPMWebAppMixin.backend
self.page = PHPFPMWebAppMixin.page
self.add_route()
webapp = '%s_%s_webapp' % (random_ascii(10), self.type_value)
self.add_webapp(webapp)
self.addCleanup(self.delete_webapp, webapp)
self.upload_webapp(webapp)
path = '/%s' % webapp
self.add_content(website, webapp, path)
url = 'http://%s%s/%s' % (domain.name, path, self.page[0])
self.assertEqual(self.page[2], requests.get(url).content)
class PHPFcidRESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass
class RESTWebsiteTest(RESTWebsiteMixin, PHPFcidWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass
class RESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
class PHPFPMRESTWebsiteTest(RESTWebsiteMixin, PHPFPMWebAppMixin, WebsiteMixin, BaseLiveServerTestCase):
pass
#class AdminWebsiteTest(AdminWebsiteMixin, BaseLiveServerTestCase):

1
orchestra/templates/admin/base_site.html
View file

@ -27,6 +27,7 @@
{% if not is_popup %}
{% admin_tools_render_menu %}
{% endif %}
</div>
{% endif %}
{% endblock %}

12
scripts/services/apache_full_stack.md
View file

@ -84,6 +84,12 @@ The goal of this setup is having a high-performance state-of-the-art deployment
</Directory>
# TODO pool per website or pool per user? memory consumption
events.mechanism = epoll
# TODO multiple master processes, opcache is held in master, and reload/restart affects all pools
# http://mattiasgeniar.be/2014/04/09/a-better-way-to-run-php-fpm/
TODO CHRoot
https://andrewbevitt.com/tutorials/apache-varnish-chrooted-php-fpm-wordpress-virtual-host/
@ -92,10 +98,10 @@ TODO CHRoot
[vhost]
istemplate = 1
listen.mode = 0660
pm = ondemand
pm.max_children = 5
pm.start_servers = 1
pm.min_spare_servers = 1
pm.max_spare_servers = 2
pm.process_idle_timeout = 10s
pm.max_requests = 200
' > /etc/php5/fpm/conf.d/vhost-template.conf
```