pedro
/
django-orchestra-test
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactoring rest api nested serialization

This commit is contained in:
Marc 2014-10-16 15:11:52 +00:00
parent d817fe7198
commit a7a399bcd6
18 changed files with 207 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
TODO.md
View File

@ -141,7 +141,6 @@ Remember that, as always with QuerySets, any subsequent chained methods which im
-# Required-Stop: $network $local_fs $remote_fs postgresql celeryd
* POST only fields (account, username, name) etc http://inka-labs.com/blog/2013/04/18/post-only-fields-django-rest-framework/
* for list virtual_domains cleaning up we need to know the old domain name when a list changes its address domain, but this is not possible with the current design.
* regenerate virtual_domains every time (configure a separate file for orchestra on postfix)
* update_fields=[] doesn't trigger post save!

49
orchestra/api/serializers.py
View File

@ -10,53 +10,24 @@ class SetPasswordSerializer(serializers.Serializer):
widget=widgets.PasswordInput, validators=[validate_password])
from rest_framework.serializers import (HyperlinkedModelSerializerOptions,
HyperlinkedModelSerializer)
class tHyperlinkedModelSerializerOptions(serializers.HyperlinkedModelSerializerOptions):
""" Options for PostHyperlinkedModelSerializer """
class HyperlinkedModelSerializerOptions(serializers.HyperlinkedModelSerializerOptions):
def __init__(self, meta):
super(HyperlinkedModelSerializerOptions, self).__init__(meta)
self.postonly_fields = getattr(meta, 'postonly_fields', ())
class HyperlinkedModelSerializer(HyperlinkedModelSerializer):
class HyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer):
""" support for postonly_fields, fields whose value can only be set on post """
_options_class = HyperlinkedModelSerializerOptions
def to_native(self, obj):
""" Serialize objects -> primitives. """
ret = self._dict_class()
ret.fields = {}
for field_name, field in self.fields.items():
# Ignore all postonly_fields fron serialization
if field_name in self.opts.postonly_fields:
continue
field.initialize(parent=self, field_name=field_name)
key = self.get_field_key(field_name)
value = field.field_to_native(obj, field_name)
ret[key] = value
ret.fields[key] = field
return ret
def restore_object(self, attrs, instance=None):
model_attrs, post_attrs = {}, {}
for attr, value in attrs.iteritems():
if attr in self.opts.postonly_fields:
post_attrs[attr] = value
else:
model_attrs[attr] = value
obj = super(HyperlinkedModelSerializer, self).restore_object(model_attrs, instance)
# Method to process ignored postonly_fields
self.process_postonly_fields(obj, post_attrs)
return obj
def process_postonly_fields(self, obj, post_attrs):
""" Placeholder method for processing data sent in POST. """
pass
""" removes postonly_fields from attrs when not posting """
model_attrs = dict(**attrs)
if instance is not None:
for attr, value in attrs.iteritems():
if attr in self.opts.postonly_fields:
model_attrs.pop(attr)
return super(HyperlinkedModelSerializer, self).restore_object(model_attrs, instance)
class MultiSelectField(serializers.ChoiceField):

9
orchestra/apps/accounts/serializers.py
View File

@ -12,6 +12,13 @@ class AccountSerializer(serializers.HyperlinkedModelSerializer):
class AccountSerializerMixin(object):
def __init__(self, *args, **kwargs):
super(AccountSerializerMixin, self).__init__(*args, **kwargs)
self.account = None
request = self.context.get('request')
if request:
self.account = request.user
def save_object(self, obj, **kwargs):
obj.account = self.context['request'].user
obj.account = self.account
super(AccountSerializerMixin, self).save_object(obj, **kwargs)

2
orchestra/apps/databases/backends.py
View File

@ -19,6 +19,8 @@ class MySQLBackend(ServiceController):
self.append(
"mysql -e 'CREATE DATABASE `%(database)s`;' || true" % context
)
# clean previous privileges
self.append("""mysql mysql -e 'DELETE FROM db WHERE db = "%(database)s";'""" % context)
for user in database.users.all():
context.update({
'username': user.username,

25
orchestra/apps/databases/serializers.py
View File

@ -1,5 +1,6 @@
from django.forms import widgets
from django.utils.translation import ugettext, ugettext_lazy as _
from django.shortcuts import get_object_or_404
from rest_framework import serializers
from orchestra.api.serializers import HyperlinkedModelSerializer
@ -9,32 +10,39 @@ from orchestra.core.validators import validate_password
from .models import Database, DatabaseUser
class RelatedDatabaseUserSerializer(serializers.HyperlinkedModelSerializer):
class RelatedDatabaseUserSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = DatabaseUser
fields = ('url', 'username')
def from_native(self, data, files=None):
return DatabaseUser.objects.get(username=data['username'])
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, username=data['username'])
class DatabaseSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
users = RelatedDatabaseUserSerializer(many=True, allow_add_remove=True)
# TODO clean user.type = db.type
class Meta:
model = Database
fields = ('url', 'name', 'type', 'users')
postonly_fields = ('name', 'type')
def validate(self, attrs):
for user in attrs['users']:
if user.type != attrs['type']:
raise serializers.ValidationError("User type must be" % attrs['type'])
return attrs
class RelatedDatabaseSerializer(serializers.HyperlinkedModelSerializer):
class RelatedDatabaseSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = Database
fields = ('url', 'name',)
def from_native(self, data, files=None):
return Database.objects.get(name=data['name'])
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, name=data['name'])
class DatabaseUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
@ -42,13 +50,18 @@ class DatabaseUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer)
validators=[validate_password], write_only=True,
widget=widgets.PasswordInput)
databases = RelatedDatabaseSerializer(many=True, allow_add_remove=True, required=False)
# TODO clean user.type = db.type
class Meta:
model = DatabaseUser
fields = ('url', 'username', 'password', 'type', 'databases')
postonly_fields = ('username', 'type')
def validate(self, attrs):
for database in attrs.get('databases', []):
if database.type != attrs['type']:
raise serializers.ValidationError("Database type must be" % attrs['type'])
return attrs
def save_object(self, obj, **kwargs):
# FIXME this method will be called when saving nested serializers :(
if not obj.pk:

54
orchestra/apps/databases/tests/functional_tests/tests.py
View File

@ -115,10 +115,28 @@ class DatabaseTestMixin(object):
self.add_user(username2, password2)
self.add_user_to_db(username2, dbname)
self.delete_user(username)
self.validate_delete_user(username, password)
self.validate_login_error(dbname, username, password)
self.validate_create_table(dbname, username2, password2)
self.delete_user(username2)
self.validate_login_error(dbname, username2, password2)
self.validate_delete_user(username2, password2)
def test_swap_user(self):
dbname = '%s_database' % random_ascii(5)
username = '%s_dbuser' % random_ascii(5)
password = '@!?%spppP001' % random_ascii(5)
self.add(dbname, username, password)
self.addCleanup(self.delete, dbname)
self.addCleanup(self.delete_user, username)
self.validate_create_table(dbname, username, password)
username2 = '%s_dbuser' % random_ascii(5)
password2 = '@!?%spppP001' % random_ascii(5)
self.add_user(username2, password2)
self.addCleanup(self.delete_user, username2)
self.swap_user(username, username2, dbname)
self.validate_login_error(dbname, username, password)
self.validate_create_table(dbname, username2, password2)
class MySQLBackendMixin(object):
@ -151,10 +169,10 @@ class MySQLBackendMixin(object):
self.validate_create_table, dbname, username, password
)
def validate_delete(self, name, username, password):
self.assertRaises(MySQLdb.OperationalError,
self.validate_create_table, name, username, password
)
def validate_delete(self, dbname, username, password):
self.validate_login_error(dbname, username, password)
self.assertRaises(CommandError,
sshrun, self.MASTER_SERVER, 'mysql %s' % dbname, display=False)
def validate_delete_user(self, name, username):
context = {
@ -165,8 +183,6 @@ class MySQLBackendMixin(object):
"""mysql mysql -e 'SELECT * FROM db WHERE db="%(name)s";'""" % context, display=False).stdout)
self.assertEqual('', sshrun(self.MASTER_SERVER,
"""mysql mysql -e 'SELECT * FROM user WHERE user="%(username)s";'""" % context, display=False).stdout)
# TODO remove used from database
class RESTDatabaseMixin(DatabaseTestMixin):
@ -205,6 +221,14 @@ class RESTDatabaseMixin(DatabaseTestMixin):
@save_response_on_error
def delete_user(self, username):
self.rest.databaseusers.retrieve(username=username).delete()
@save_response_on_error
def swap_user(self, username, username2, dbname):
user = self.rest.databaseusers.retrieve(username=username2).get()
db = self.rest.databases.retrieve(name=dbname).get()
db.users = db.users.exclude(username=username)
db.users.append(user)
db.save()
class AdminDatabaseMixin(DatabaseTestMixin):
@ -280,6 +304,24 @@ class AdminDatabaseMixin(DatabaseTestMixin):
save.submit()
self.assertNotEqual(url, self.selenium.current_url)
@snapshot_on_error
def swap_user(self, username, username2, dbname):
database = Database.objects.get(name=dbname, type=self.db_type)
url = self.live_server_url + change_url(database)
self.selenium.get(url)
user = DatabaseUser.objects.get(username=username, type=self.db_type)
users_input = self.selenium.find_element_by_id('id_users')
users_select = Select(users_input)
users_select.deselect_by_value(str(user.pk))
user = DatabaseUser.objects.get(username=username2, type=self.db_type)
users_select.select_by_value(str(user.pk))
save = self.selenium.find_element_by_name('_save')
save.submit()
self.assertNotEqual(url, self.selenium.current_url)
@snapshot_on_error
def delete_user(self, username):
user = DatabaseUser.objects.get(username=username)

23
orchestra/apps/lists/serializers.py
View File

@ -1,5 +1,6 @@
from django.forms import widgets
from django.utils.translation import ugettext, ugettext_lazy as _
from django.shortcuts import get_object_or_404
from rest_framework import serializers
from orchestra.api.serializers import HyperlinkedModelSerializer
@ -9,10 +10,21 @@ from orchestra.core.validators import validate_password
from .models import List
class RelatedDomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = List.address_domain.field.rel.to
fields = ('url', 'name')
def from_native(self, data, files=None):
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, name=data['name'])
class ListSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
password = serializers.CharField(max_length=128, label=_('Password'),
validators=[validate_password], write_only=True, required=False,
widget=widgets.PasswordInput)
address_domain = RelatedDomainSerializer(required=False)
class Meta:
model = List
@ -28,6 +40,17 @@ class ListSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
raise serializers.ValidationError(_("Password required"))
return attrs
def validate(self, attrs):
address_domain = attrs.get('address_domain')
address_name = attrs.get('address_name', )
if self.object:
address_domain = address_domain or self.object.address_domain
address_name = address_name or self.object.address_name
if bool(address_domain) != bool(address_name):
raise serializers.ValidationError(
_("address_name and address_domain should go in tandem"))
return attrs
def save_object(self, obj, **kwargs):
if not obj.pk:
obj.set_password(self.init_data.get('password', ''))

6
orchestra/apps/lists/tests/functional_tests/tests.py
View File

@ -142,7 +142,7 @@ class ListMixin(object):
domain_name = '%sdomain.lan' % random_ascii(10)
address_domain = Domain.objects.create(name=domain_name, account=self.account)
self.add(name, password, admin_email, address_name=address_name, address_domain=address_domain)
# self.addCleanup(self.delete, name)
self.addCleanup(self.delete, name)
# Mailman doesn't support changing the address, only the domain
address_name = '%s_name' % random_ascii(10)
self.update_address_name(name, address_name)
@ -174,7 +174,7 @@ class RESTListMixin(ListMixin):
if address_name:
extra.update({
'address_name': address_name,
'address_domain': self.rest.domains.retrieve(name=address_domain.name).get().url,
'address_domain': self.rest.domains.retrieve(name=address_domain.name).get(),
})
self.rest.lists.create(name=name, password=password, admin_email=admin_email, **extra)
@ -191,7 +191,7 @@ class RESTListMixin(ListMixin):
def update_domain(self, name, domain_name):
mail_list = self.rest.lists.retrieve(name=name).get()
domain = self.rest.domains.retrieve(name=domain_name).get()
mail_list.update(address_domain=domain.url)
mail_list.update(address_domain=domain)
@save_response_on_error
def update_address_name(self, name, address_name):

34
orchestra/apps/mails/serializers.py
View File

@ -1,4 +1,5 @@
from django.forms import widgets
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext, ugettext_lazy as _
from rest_framework import serializers
@ -37,21 +38,34 @@ class MailboxSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
super(MailboxSerializer, self).save_object(obj, **kwargs)
class RelatedMailboxSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = Mailbox
fields = ('url', 'name')
def from_native(self, data, files=None):
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, name=data['name'])
class RelatedDomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = Address.domain.field.rel.to
fields = ('url', 'name')
def from_native(self, data, files=None):
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, name=data['name'])
class AddressSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
domain = RelatedDomainSerializer()
mailboxes = RelatedMailboxSerializer(many=True, allow_add_remove=True, required=False)
class Meta:
model = Address
fields = ('url', 'name', 'domain', 'mailboxes', 'forward')
def get_fields(self, *args, **kwargs):
fields = super(AddressSerializer, self).get_fields(*args, **kwargs)
account = self.context['view'].request.user.pk
mailboxes = fields['mailboxes'].queryset
fields['mailboxes'].queryset = mailboxes.filter(account=account)
# TODO do it on permissions or in self.filter_by_account_field ?
domain = fields['domain'].queryset
fields['domain'].queryset = domain.filter(account=account)
return fields
def validate(self, attrs):
if not attrs['mailboxes'] and not attrs['forward']:
raise serializers.ValidationError("mailboxes or forward addresses should be provided")

17
orchestra/apps/saas/backends/__init__.py Normal file
View File

@ -0,0 +1,17 @@
import pkgutil
import textwrap
class SaaSServiceMixin(object):
model = 'saas.SaaS'
# TODO Match definition support on backends (mysql) and saas
def get_context(self, webapp):
# TODO
return {
}
for __, module_name, __ in pkgutil.walk_packages(__path__):
# sorry for the exec(), but Import module function fails :(
exec('from . import %s' % module_name)

4
orchestra/apps/webapps/backends/dokuwikimu.py → orchestra/apps/saas/backends/dokuwikimu.py
View File

@ -4,11 +4,11 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from . import SaaSServiceMixin
from .. import settings
class DokuWikiMuBackend(WebAppServiceMixin, ServiceController):
class DokuWikiMuBackend(SaaSServiceMixin, ServiceController):
verbose_name = _("DokuWiki multisite")
def save(self, webapp):

4
orchestra/apps/webapps/backends/drupalmu.py → orchestra/apps/saas/backends/drupalmu.py
View File

@ -4,11 +4,11 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from . import SaaSServiceMixin
from .. import settings
class DrupalMuBackend(WebAppServiceMixin, ServiceController):
class DrupalMuBackend(SaaSServiceMixin, ServiceController):
verbose_name = _("Drupal multisite")
def save(self, webapp):

4
orchestra/apps/webapps/backends/wordpressmu.py → orchestra/apps/saas/backends/wordpressmu.py
View File

@ -5,11 +5,11 @@ from django.utils.translation import ugettext_lazy as _
from orchestra.apps.orchestration import ServiceController
from . import WebAppServiceMixin
from . import SaaSServiceMixin
from .. import settings
class WordpressMuBackend(WebAppServiceMixin, ServiceController):
class WordpressMuBackend(SaaSServiceMixin, ServiceController):
verbose_name = _("Wordpress multisite")
@property

17
orchestra/apps/systemusers/serializers.py
View File

@ -1,5 +1,6 @@
from django.contrib.auth import get_user_model
from django.forms import widgets
from django.shortcuts import get_object_or_404
from django.utils.translation import ugettext, ugettext_lazy as _
from rest_framework import serializers
@ -10,13 +11,14 @@ from orchestra.core.validators import validate_password
from .models import SystemUser
class GroupSerializer(serializers.ModelSerializer):
class GroupSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = SystemUser
fields = ('username',)
fields = ('url', 'username',)
def from_native(self, data, files=None):
return SystemUser.objects.get(username=data['username'])
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, username=data['username'])
class SystemUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
@ -41,7 +43,14 @@ class SystemUserSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
raise serializers.ValidationError(_("Password required"))
return attrs
# TODO validate gruops != self
def validate_groups(self, attrs, source):
groups = attrs.get(source)
if groups:
for group in groups:
if group.username == attrs['username']:
raise serializers.ValidationError(
_("Do not make the user member of its group"))
return attrs
def save_object(self, obj, **kwargs):
# FIXME this method will be called when saving nested serializers :(

24
orchestra/apps/websites/serializers.py
View File

@ -1,3 +1,4 @@
from django.shortcuts import get_object_or_404
from rest_framework import serializers
from orchestra.api.fields import OptionField
@ -7,7 +8,29 @@ from orchestra.apps.accounts.serializers import AccountSerializerMixin
from .models import Website, Content
class RelatedDomainSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = Website.domains.field.rel.to
fields = ('url', 'name')
def from_native(self, data, files=None):
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, name=data['name'])
class RelatedWebAppSerializer(AccountSerializerMixin, serializers.HyperlinkedModelSerializer):
class Meta:
model = Content.webapp.field.rel.to
fields = ('url', 'name', 'type')
def from_native(self, data, files=None):
queryset = self.opts.model.objects.filter(account=self.account)
return get_object_or_404(queryset, name=data['name'])
class ContentSerializer(serializers.HyperlinkedModelSerializer):
webapp = RelatedWebAppSerializer()
class Meta:
model = Content
fields = ('webapp', 'path')
@ -17,6 +40,7 @@ class ContentSerializer(serializers.HyperlinkedModelSerializer):
class WebsiteSerializer(AccountSerializerMixin, HyperlinkedModelSerializer):
domains = RelatedDomainSerializer(many=True, allow_add_remove=True, required=False)
contents = ContentSerializer(required=False, many=True, allow_add_remove=True,
source='content_set')
options = OptionField(required=False)

6
orchestra/apps/websites/tests/functional_tests/tests.py
View File

@ -72,10 +72,10 @@ class RESTWebsiteMixin(RESTWebAppMixin):
domain = self.rest.domains.retrieve(name=domain).get()
webapp = self.rest.webapps.retrieve(name=webapp).get()
contents = [{
'webapp': webapp.url,
'webapp': webapp,
'path': path
}]
self.rest.websites.create(name=name, domains=[domain.url], contents=contents)
self.rest.websites.create(name=name, domains=[domain], contents=contents)
@save_response_on_error
def delete_website(self, name):
@ -86,7 +86,7 @@ class RESTWebsiteMixin(RESTWebAppMixin):
website = self.rest.websites.retrieve(name=website).get()
webapp = self.rest.webapps.retrieve(name=webapp).get()
website.contents.append({
'webapp': webapp.url,
'webapp': webapp,
'path': path,
})
website.save()

9
orchestra/utils/html.py
View File

@ -3,6 +3,9 @@ from orchestra.utils.system import run
def html_to_pdf(html):
""" converts HTL to PDF using wkhtmltopdf """
return run('xvfb-run -a -s "-screen 0 640x4800x16" '
'wkhtmltopdf --footer-center "Page [page] of [topage]" --footer-font-size 9 - -',
stdin=html.encode('utf-8'), display=False)
return run(
'PATH=$PATH:/usr/local/bin/\n'
'xvfb-run -a -s "-screen 0 640x4800x16" '
'wkhtmltopdf --footer-center "Page [page] of [topage]" --footer-font-size 9 - -',
stdin=html.encode('utf-8'), display=False
)

2
scripts/container/deploy.sh
View File

@ -101,7 +101,7 @@ cat <<- EOF | python $MANAGE shell
from orchestra.apps.accounts.models import Account
if not Account.objects.filter(username="$USER").exists():
print 'Creating orchestra superuser'
__ = Account.objects.create_superuser("$USER", "'$USER@localhost'", "$PASSWORD")
__ = Account.objects.create_superuser("$USER", "$USER@localhost", "$PASSWORD")
EOF