From d165d7f03da204b8e138f7149c228394bda9186a Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Sun, 5 Apr 2015 18:02:36 +0000 Subject: [PATCH] Secure backends from injection --- TODO.md | 91 ++++++--------- orchestra/api/options.py | 45 ++++++++ orchestra/contrib/accounts/api.py | 4 +- orchestra/contrib/bills/api.py | 4 +- orchestra/contrib/contacts/api.py | 4 +- orchestra/contrib/databases/api.py | 6 +- orchestra/contrib/databases/backends.py | 11 +- orchestra/contrib/domains/backends.py | 50 ++++----- orchestra/contrib/domains/validators.py | 2 +- orchestra/contrib/issues/api.py | 7 +- orchestra/contrib/lists/api.py | 4 +- orchestra/contrib/lists/backends.py | 25 ++--- orchestra/contrib/mailboxes/api.py | 6 +- orchestra/contrib/mailboxes/backends.py | 52 ++++----- orchestra/contrib/mailboxes/settings.py | 5 + orchestra/contrib/orchestration/__init__.py | 2 +- orchestra/contrib/orchestration/backends.py | 10 ++ .../management/commands/orchestrate.py | 23 ++-- orchestra/contrib/orchestration/manager.py | 2 + orchestra/contrib/payments/api.py | 6 +- orchestra/contrib/saas/backends/bscw.py | 5 +- orchestra/contrib/saas/backends/dokuwikimu.py | 4 +- orchestra/contrib/saas/backends/drupalmu.py | 4 +- orchestra/contrib/saas/services/options.py | 15 ++- orchestra/contrib/systemusers/api.py | 4 +- orchestra/contrib/systemusers/backends.py | 106 ++++-------------- orchestra/contrib/vps/backends.py | 4 +- orchestra/contrib/webapps/api.py | 4 +- .../contrib/webapps/backends/__init__.py | 5 +- orchestra/contrib/webapps/backends/php.py | 13 ++- .../contrib/webapps/backends/symboliclink.py | 4 +- .../contrib/webapps/backends/wordpress.py | 22 ++-- orchestra/contrib/websites/api.py | 4 +- orchestra/contrib/websites/backends/apache.py | 13 ++- .../contrib/websites/backends/webalizer.py | 4 +- orchestra/core/validators.py | 2 +- 36 files changed, 285 insertions(+), 287 deletions(-) diff --git a/TODO.md b/TODO.md index 9522428c..9ff14ead 100644 --- a/TODO.md +++ b/TODO.md @@ -1,31 +1,22 @@ ==== TODO ==== - -* scape strings before executing scripts in order to prevent exploits: django templates automatically scapes things. Most important is to ensuer that all escape ' to " -* Don't store passwords and other service parameters that can be changed by the services i.e. mailman, vps etc. Find an execution mechanism that trigger `change_password()` - -* abort transaction on orchestration when `state == TIMEOUT` ? * use format_html_join for orchestration email alerts * enforce an emergency email contact and account to contact contacts about problems when mailserver is down * add `BackendLog` retry action + * webmail identities and addresses * Permissions .filter_queryset() * env vars instead of multiple settings files: https://devcenter.heroku.com/articles/config-vars ? -* Log changes from rest api (serialized objects) - +# TODO Log changes from rest api (serialized objects) * backend logs with hal logo * LAST version of this shit http://wkhtmltopdf.org/downloads.h otml -* translations - from django.utils import translation - with translation.override('en'): - * help_text on readonly_fields specialy Bill.state. (eg. A bill is in OPEN state when bla bla ) * create log file at /var/log/orchestra.log and rotate @@ -39,53 +30,36 @@ * Maildir billing tests/ webdisk billing tests (avg metric) - * when using modeladmin to store shit like self.account, make sure to have a cleanslate in each request? no, better reuse the last one * jabber with mailbox accounts (dovecot mail notification) * rename accounts register to "account", and reated api and admin references -* prevent deletion of main user by the user itself - * AccountAdminMixin auto adds 'account__name' on searchfields -* Separate panel from server passwords? Store passwords on panel? set_password special backend operation? - * What fields we really need on contacts? name email phone and what more? * Redirect junk emails and delete every 30 days? * DOC: Complitely decouples scripts execution, billing, service definition -* delete main user -> delete account or prevent delete main user - - -* multiple domains creation; line separated domains - - * init.d celery scripts -# Required-Start: $network $local_fs $remote_fs postgresql celeryd -# Required-Stop: $network $local_fs $remote_fs postgresql celeryd - * regenerate virtual_domains every time (configure a separate file for orchestra on postfix) -* update_fields=[] doesn't trigger post save! * Backend optimization * fields = () * ignore_fields = () * based on a merge set of save(update_fields) -* parmiko write to a channel instead of transfering files? http://sysadmin.circularvale.com/programming/paramiko-channel-hangs/ - * proforma without billing contact? * print open invoices as proforma? -* env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python manage.py test orchestra.apps.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture¶ - - +* env ORCHESTRA_MASTER_SERVER='test1.orchestra.lan' ORCHESTRA_SECOND_SERVER='test2.orchestra.lan' ORCHESTRA_SLAVE_SERVER='test3.orchestra.lan' python manage.py test orchestra.apps.domains.tests.functional_tests.tests:AdminBind9BackendDomainTest --nologcapture * ForeignKey.swappable * Field.editable @@ -95,8 +69,6 @@ * caching based on "def text2int(textnum, numwords={}):" -* multiple files monitoring - * sync() ServiceController method that synchronizes orchestra and servers (delete or import) * consider removing mailbox support on forward (user@pangea.org instead) @@ -119,7 +91,6 @@ * domain validation parse named-checzone output to assign errors to fields * Directory Protection on webapp and use webapp path as base path (validate) -* User [Group] webapp/website option (validation) which overrides default mainsystemuser * validate systemuser.home on server-side @@ -137,7 +108,7 @@ * Resource graph for each related object -* SaaS model splitted into SaaSUser and SaaSSite? inherit from SaaS +* SaaS model splitted into SaaSUser and SaaSSite? inherit from SaaS, proxy model? * prevent @pangea.org email addresses on contacts, enforce at least one email without @pangea.org @@ -159,7 +130,6 @@ Php binaries should have this format: /usr/bin/php5.2-cgi * logs on panel/logs/ ? mkdir ~webapps, backend post save signal? -* transaction fault tolerant on backend.execute() * and other IfModule on backend SecRule * Orchestra global search box on the page head, based https://github.com/django/django/blob/master/django/contrib/admin/options.py#L866 and iterating over all registered services and inspectin its admin.search_fields @@ -185,7 +155,7 @@ Php binaries should have this format: /usr/bin/php5.2-cgi * tags = GenericRelation(TaggedItem, related_query_name='bookmarks') -* make home for all systemusers (/home/username) and fix monitors +# make home for all systemusers (/home/username) and fix monitors * user provided crons @@ -195,7 +165,7 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * make account available on all admin forms -* WPMU blog traffic +# WPMU blog traffic * normurlpath '' return '/' @@ -211,32 +181,30 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * Document metric interpretation help_text * document plugin serialization, data_serializer? -* bill line managemente, remove, undo (only when possible), move, copy, paste +# bill line managemente, remove, undo (only when possible), move, copy, paste * budgets: no undo feature * Autocomplete admin fields like .phplist... with js -* autoexpand mailbox.filter according to filtering options +* autoexpand mailbox.filter according to filtering options (js) * allow empty metric pack for default rates? changes on rating algo -* IMPORTANT make sure no order is created for mailboxes that include disk? or just don't produce lines with cost == 0 or quantity 0 ? maybe minimal quantity for billing? like 0.1 ? or minimal price? per line or per bill? +# IMPORTANT make sure no order is created for mailboxes that include disk? or just don't produce lines with cost == 0 or quantity 0 ? maybe minimal quantity for billing? like 0.1 ? or minimal price? per line or per bill? * Improve performance of admin change lists with debug toolbar and prefech_related -* and miscellaneous.service.name == 'domini-registre' -* DOMINI REGISTRE MIGRATION SCRIPTS +# DOMINI REGISTRE MIGRATION SCRIPTS -* lines too long on invoice, double lines or cut, and make margin wider +# lines too long on invoice, double lines or cut, and make margin wider * PHP_TIMEOUT env variable in sync with fcgid idle timeout http://foaa.de/old-blog/2010/11/php-apache-and-fastcgi-a-comprehensive-overview/trackback/index.html#pni-top0 * payment methods icons * use server.name | server.address on python backends, like gitlab instead of settings? -* saas change password feature (the only way of re.running a backend) * TODO raise404, here and everywhere -* display subline links on billlines, to show that they exists. +# display subline links on billlines, to show that they exists. * update service orders on a celery task? because it take alot -* billline quantity eval('10x100') instead of miningless description '(10*100)' +# billline quantity eval('10x100') instead of miningless description '(10*100)' # FIXME do more test, make sure billed until doesn't get uodated whhen services are billed with les metric, and don't upgrade billed_until when undoing under this circumstances * line 513: change threshold and one time service metric change should update last value if not billed, only record for recurring invoicing. postpay services should store the last metric for pricing period. @@ -249,23 +217,19 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * write down insights -* use english on services defs and so on, an translate them on render time +# use english on services defs and so on, an translate them on render time * websites directives get_location() and use it on last change view validation stage to compare with contents.location and also on the backend ? * modeladmin Default filter + search isn't working, prepend filter when searching -* IMPORTANT do all modles.py TODOs and create migrations for finished apps +# IMPORTANT do all modles.py TODOs and create migrations for finished apps -* create service templates based on urlqwargs with the most basic services. +* create service help templates based on urlqwargs with the most basic services. -* Base price: domini propi (all domains) + extra for other domains +# TDOO Base price: domini propi (all domains) + extra for other domains - -* prepend ORCHESTRA_ to orchestra/settings.py - - -* rename backends with generic names to concrete services.. eg VsFTPdTraffic, UNIXSystemUser +# TODO prepend ORCHESTRA_ to orchestra/settings.py Translation @@ -296,7 +260,7 @@ celery max-tasks-per-child * postupgradeorchestra send signals in order to hook custom stuff -* make base home for systemusers that ara homed into main account systemuser, and prevent shell users to have nested homes (if nnot implemented already) +# FIXME make base home for systemusers that ara homed into main account systemuser, and prevent shell users to have nested homes (if nnot implemented already) * autoscale celery workers http://docs.celeryproject.org/en/latest/userguide/workers.html#autoscaling @@ -312,11 +276,24 @@ https://code.djangoproject.com/ticket/24576 # FIXME what to do when deleting accounts? set fk null and fill a username charfield? issues, invoices.. we whant all this to go away? * implement delete All related services -* address name change does not remove old one :P +# FIXME address name change does not remove old one :P * read https://docs.djangoproject.com/en/dev/releases/1.8/ and fix deprecation warnings -* remove admin object links , like contents webapps +* remove admin object display_links , like contents webapps * SaaS and WebApp types and services fieldsets, and helptexts ! * replace make_option in management commands + +* welcome, pangea linke doesnt work + +# FIXME model contact info and account info (email, name, etc) correctly/unredundant/dry + + +* Use the new django.contrib.admin.RelatedOnlyFieldListFilter in ModelAdmin.list_filter to limit the list_filter choices to foreign objects which are attached to those from the ModelAdmin. ++ Query Expressions, Conditional Expressions, and Database Functions¶ +* forms: You can now pass a callable that returns an iterable of choices when instantiating a ChoiceField. + + +* migrate to DRF3.x + diff --git a/orchestra/api/options.py b/orchestra/api/options.py index 09960795..08c59c03 100644 --- a/orchestra/api/options.py +++ b/orchestra/api/options.py @@ -1,5 +1,8 @@ +from django.contrib.admin.options import get_content_type_for_model from django.conf import settings as django_settings +from django.utils.encoding import force_text from django.utils.module_loading import autodiscover_modules +from django.utils.translation import ugettext as _ from rest_framework.routers import DefaultRouter from orchestra import settings @@ -8,6 +11,48 @@ from orchestra.utils.python import import_class from .helpers import insert_links +class LogApiMixin(object): + def post(self, request, *args, **kwargs): + from django.contrib.admin.models import ADDITION + response = super(LogApiMixin, self).post(request, *args, **kwargs) + message = _('Added.') + self.log_addition(request, message, ADDITION) + return response + + def put(self, request, *args, **kwargs): + from django.contrib.admin.models import CHANGE + response = super(LogApiMixin, self).put(request, *args, **kwargs) + message = _('Changed') + self.log(request, message, CHANGE) + return response + + def patch(self, request, *args, **kwargs): + from django.contrib.admin.models import CHANGE + response = super(LogApiMixin, self).put(request, *args, **kwargs) + message = _('Changed %s') % str(response.data) + self.log(request, message, CHANGE) + return response + + def delete(self, request, *args, **kwargs): + from django.contrib.admin.models import DELETION + message = _('Deleted') + self.log(request, message, DELETION) + response = super(LogApiMixin, self).put(request, *args, **kwargs) + return response + + def log(self, request, message, action): + from django.contrib.admin.models import LogEntry + instance = self.get_object() + LogEntry.objects.log_action( + user_id=request.user.pk, + content_type_id=get_content_type_for_model(instance).pk, + object_id=instance.pk, + object_repr=force_text(instance), + action_flag=action, + change_message=message, + ) + + class LinkHeaderRouter(DefaultRouter): def get_api_root_view(self): """ returns the root view, with all the linked collections """ diff --git a/orchestra/contrib/accounts/api.py b/orchestra/contrib/accounts/api.py index b7c20e50..edd205ee 100644 --- a/orchestra/contrib/accounts/api.py +++ b/orchestra/contrib/accounts/api.py @@ -1,7 +1,7 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import viewsets, exceptions -from orchestra.api import router, SetPasswordApiMixin +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin from .models import Account from .serializers import AccountSerializer @@ -13,7 +13,7 @@ class AccountApiMixin(object): return qs.filter(account=self.request.user.pk) -class AccountViewSet(SetPasswordApiMixin, viewsets.ModelViewSet): +class AccountViewSet(LogApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): model = Account serializer_class = AccountSerializer singleton_pk = lambda _,request: request.user.pk diff --git a/orchestra/contrib/bills/api.py b/orchestra/contrib/bills/api.py index c0ae655d..d5d4bfb4 100644 --- a/orchestra/contrib/bills/api.py +++ b/orchestra/contrib/bills/api.py @@ -2,7 +2,7 @@ from django.http import HttpResponse from rest_framework import viewsets from rest_framework.decorators import detail_route -from orchestra.api import router +from orchestra.api import router, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from orchestra.utils.html import html_to_pdf @@ -11,7 +11,7 @@ from .serializers import BillSerializer -class BillViewSet(AccountApiMixin, viewsets.ModelViewSet): +class BillViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = Bill serializer_class = BillSerializer diff --git a/orchestra/contrib/contacts/api.py b/orchestra/contrib/contacts/api.py index 164ae2d4..d2575602 100644 --- a/orchestra/contrib/contacts/api.py +++ b/orchestra/contrib/contacts/api.py @@ -1,13 +1,13 @@ from rest_framework import viewsets -from orchestra.api import router +from orchestra.api import router, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from .models import Contact from .serializers import ContactSerializer -class ContactViewSet(AccountApiMixin, viewsets.ModelViewSet): +class ContactViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = Contact serializer_class = ContactSerializer diff --git a/orchestra/contrib/databases/api.py b/orchestra/contrib/databases/api.py index 645e20e1..e226ee5a 100644 --- a/orchestra/contrib/databases/api.py +++ b/orchestra/contrib/databases/api.py @@ -1,19 +1,19 @@ from rest_framework import viewsets -from orchestra.api import router, SetPasswordApiMixin +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from .models import Database, DatabaseUser from .serializers import DatabaseSerializer, DatabaseUserSerializer -class DatabaseViewSet(AccountApiMixin, viewsets.ModelViewSet): +class DatabaseViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = Database serializer_class = DatabaseSerializer filter_fields = ('name',) -class DatabaseUserViewSet(AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): +class DatabaseUserViewSet(LogApiMixin, AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): model = DatabaseUser serializer_class = DatabaseUserSerializer filter_fields = ('username',) diff --git a/orchestra/contrib/databases/backends.py b/orchestra/contrib/databases/backends.py index 4746c5e7..0cea6fc7 100644 --- a/orchestra/contrib/databases/backends.py +++ b/orchestra/contrib/databases/backends.py @@ -2,7 +2,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from orchestra.contrib.resources import ServiceMonitor from . import settings @@ -46,10 +46,11 @@ class MySQLBackend(ServiceController): super(MySQLBackend, self).commit() def get_context(self, database): - return { + context = { 'database': database.name, 'host': settings.DATABASES_DEFAULT_HOST, } + return replace(replace(context, "'", '"'), ';', '') class MySQLUserBackend(ServiceController): @@ -83,11 +84,12 @@ class MySQLUserBackend(ServiceController): self.append("mysql -e 'FLUSH PRIVILEGES;'") def get_context(self, user): - return { + context = { 'username': user.username, 'password': user.password, 'host': settings.DATABASES_DEFAULT_HOST, } + return replace(replace(context, "'", '"'), ';', '') class MysqlDisk(ServiceMonitor): @@ -135,7 +137,8 @@ class MysqlDisk(ServiceMonitor): self.append('echo %(db_id)s $(monitor "%(db_name)s")' % context) def get_context(self, db): - return { + context = { 'db_name': db.name, 'db_id': db.pk, } + return replace(replace(context, "'", '"'), ';', '') diff --git a/orchestra/contrib/domains/backends.py b/orchestra/contrib/domains/backends.py index c5570646..8d862629 100644 --- a/orchestra/contrib/domains/backends.py +++ b/orchestra/contrib/domains/backends.py @@ -3,7 +3,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from orchestra.contrib.orchestration.models import BackendOperation as Operation from . import settings @@ -28,7 +28,7 @@ class Bind9MasterDomainBackend(ServiceController): context = self.get_context(domain) domain.refresh_serial() context['zone'] = ';; %(banner)s\n' % context - context['zone'] += domain.render_zone() + context['zone'] += domain.render_zone().replace("'", '"') self.append(textwrap.dedent("""\ echo -e '%(zone)s' > %(zone_path)s.tmp diff -N -I'^\s*;;' %(zone_path)s %(zone_path)s.tmp || UPDATED=1 @@ -98,20 +98,18 @@ class Bind9MasterDomainBackend(ServiceController): 'banner': self.get_banner(), 'slaves': '; '.join(slaves) or 'none', 'also_notify': '; '.join(slaves) + ';' if slaves else '', - } - context.update({ 'conf_path': settings.DOMAINS_MASTERS_PATH, - 'conf': textwrap.dedent(""" - zone "%(name)s" { - // %(banner)s - type master; - file "%(zone_path)s"; - allow-transfer { %(slaves)s; }; - also-notify { %(also_notify)s }; - notify yes; - };""") % context - }) - return context + } + context['conf'] = textwrap.dedent(""" + zone "%(name)s" { + // %(banner)s + type master; + file "%(zone_path)s"; + allow-transfer { %(slaves)s; }; + also-notify { %(also_notify)s }; + notify yes; + };""") % context + return replace(context, "'", '"') class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): @@ -141,16 +139,14 @@ class Bind9SlaveDomainBackend(Bind9MasterDomainBackend): 'banner': self.get_banner(), 'subdomains': domain.subdomains.all(), 'masters': '; '.join(self.get_masters(domain)) or 'none', - } - context.update({ 'conf_path': settings.DOMAINS_SLAVES_PATH, - 'conf': textwrap.dedent(""" - zone "%(name)s" { - // %(banner)s - type slave; - file "%(name)s"; - masters { %(masters)s; }; - allow-notify { %(masters)s; }; - };""") % context - }) - return context + } + context['conf'] = textwrap.dedent(""" + zone "%(name)s" { + // %(banner)s + type slave; + file "%(name)s"; + masters { %(masters)s; }; + allow-notify { %(masters)s; }; + };""") % context + return replace(context, "'", '"') diff --git a/orchestra/contrib/domains/validators.py b/orchestra/contrib/domains/validators.py index 56d37770..6f3686c1 100644 --- a/orchestra/contrib/domains/validators.py +++ b/orchestra/contrib/domains/validators.py @@ -112,7 +112,7 @@ def validate_zone(zone): checkzone = settings.DOMAINS_CHECKZONE_BIN_PATH try: with open(zone_path, 'wb') as f: - f.write(zone_path) + f.write(zone.encode('ascii')) # Don't use /dev/stdin becuase the 'argument list is too long' error check = run(' '.join([checkzone, zone_name, zone_path]), error_codes=[0,1], display=False) finally: diff --git a/orchestra/contrib/issues/api.py b/orchestra/contrib/issues/api.py index 11d58a27..ec89d48d 100644 --- a/orchestra/contrib/issues/api.py +++ b/orchestra/contrib/issues/api.py @@ -2,14 +2,14 @@ from rest_framework import viewsets, mixins from rest_framework.decorators import action from rest_framework.response import Response -from orchestra.api import router +from orchestra.api import router, LogApiMixin from .models import Ticket, Queue from .serializers import TicketSerializer, QueueSerializer -class TicketViewSet(viewsets.ModelViewSet): +class TicketViewSet(LogApiMixin, viewsets.ModelViewSet): model = Ticket serializer_class = TicketSerializer @@ -32,7 +32,8 @@ class TicketViewSet(viewsets.ModelViewSet): return qs.filter(creator=self.request.user) -class QueueViewSet(mixins.ListModelMixin, +class QueueViewSet(LogApiMixin, + mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet): model = Queue diff --git a/orchestra/contrib/lists/api.py b/orchestra/contrib/lists/api.py index 9430e4a2..7205cc66 100644 --- a/orchestra/contrib/lists/api.py +++ b/orchestra/contrib/lists/api.py @@ -1,13 +1,13 @@ from rest_framework import viewsets -from orchestra.api import router, SetPasswordApiMixin +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from .models import List from .serializers import ListSerializer -class ListViewSet(AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): +class ListViewSet(LogApiMixin, AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): model = List serializer_class = ListSerializer filter_fields = ('name',) diff --git a/orchestra/contrib/lists/backends.py b/orchestra/contrib/lists/backends.py index 1216c2c5..033b0682 100644 --- a/orchestra/contrib/lists/backends.py +++ b/orchestra/contrib/lists/backends.py @@ -2,7 +2,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from orchestra.contrib.resources import ServiceMonitor from . import settings @@ -26,14 +26,10 @@ class MailmanBackend(ServiceController): ] def include_virtual_alias_domain(self, context): - # TODO for list virtual_domains cleaning up we need to know the old domain name when a list changes its address - # domain, but this is not possible with the current design. - # sync the whole file everytime? - # TODO same for mailbox virtual domains if context['address_domain']: self.append(textwrap.dedent(""" - [[ $(grep "^\s*%(address_domain)s\s*$" %(virtual_alias_domains)s) ]] || { - echo "%(address_domain)s" >> %(virtual_alias_domains)s + [[ $(grep '^\s*%(address_domain)s\s*$' %(virtual_alias_domains)s) ]] || { + echo '%(address_domain)s' >> %(virtual_alias_domains)s UPDATED_VIRTUAL_ALIAS_DOMAINS=1 }""") % context ) @@ -41,7 +37,7 @@ class MailmanBackend(ServiceController): def exclude_virtual_alias_domain(self, context): address_domain = context['address_domain'] if not List.objects.filter(address_domain=address_domain).exists(): - self.append('sed -i "/^%(address_domain)s\s*$/d" %(virtual_alias_domains)s' % context) + self.append("sed -i '/^%(address_domain)s\s*$/d' %(virtual_alias_domains)s" % context) def get_virtual_aliases(self, context): aliases = ['# %(banner)s' % context] @@ -54,7 +50,7 @@ class MailmanBackend(ServiceController): context = self.get_context(mail_list) # Create list self.append(textwrap.dedent("""\ - [[ ! -e %(mailman_root)s/lists/%(name)s ]] && { + [[ ! -e '%(mailman_root)s/lists/%(name)s' ]] && { newlist --quiet --emailhost='%(domain)s' '%(name)s' '%(admin)s' '%(password)s' }""") % context) # Custom domain @@ -150,7 +146,7 @@ class MailmanBackend(ServiceController): 'admin': mail_list.admin_email, 'mailman_root': settings.LISTS_MAILMAN_ROOT_PATH, }) - return context + return replace(context, "'", '"') class MailmanTrafficBash(ServiceMonitor): @@ -213,11 +209,12 @@ class MailmanTrafficBash(ServiceMonitor): ) def get_context(self, mail_list): - return { + context = { 'list_name': mail_list.name, 'object_id': mail_list.pk, 'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), } + return replace(context, "'", '"') class MailmanTraffic(ServiceMonitor): @@ -312,11 +309,12 @@ class MailmanTraffic(ServiceMonitor): self.append('monitor(lists, end_date, months, postlogs)') def get_context(self, mail_list): - return { + context = { 'list_name': mail_list.name, 'object_id': mail_list.pk, 'last_date': self.get_last_date(mail_list.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), } + return replace(context, "'", '"') class MailmanSubscribers(ServiceMonitor): @@ -328,7 +326,8 @@ class MailmanSubscribers(ServiceMonitor): self.append('echo %(object_id)i $(list_members %(list_name)s | wc -l)' % context) def get_context(self, mail_list): - return { + context = { 'list_name': mail_list.name, 'object_id': mail_list.pk, } + return replace(context, "'", '"') diff --git a/orchestra/contrib/mailboxes/api.py b/orchestra/contrib/mailboxes/api.py index 40d3e85b..f9731727 100644 --- a/orchestra/contrib/mailboxes/api.py +++ b/orchestra/contrib/mailboxes/api.py @@ -1,19 +1,19 @@ from rest_framework import viewsets -from orchestra.api import router, SetPasswordApiMixin +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from .models import Address, Mailbox from .serializers import AddressSerializer, MailboxSerializer -class AddressViewSet(AccountApiMixin, viewsets.ModelViewSet): +class AddressViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = Address serializer_class = AddressSerializer -class MailboxViewSet(SetPasswordApiMixin, AccountApiMixin, viewsets.ModelViewSet): +class MailboxViewSet(LogApiMixin, SetPasswordApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = Mailbox serializer_class = MailboxSerializer diff --git a/orchestra/contrib/mailboxes/backends.py b/orchestra/contrib/mailboxes/backends.py index 760ed76c..7e933658 100644 --- a/orchestra/contrib/mailboxes/backends.py +++ b/orchestra/contrib/mailboxes/backends.py @@ -4,7 +4,7 @@ import textwrap from django.core.exceptions import ObjectDoesNotExist from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from orchestra.contrib.resources import ServiceMonitor #from orchestra.utils.humanize import unit_to_bytes @@ -19,8 +19,8 @@ from .models import Address logger = logging.getLogger(__name__) -class MailSystemUserBackend(ServiceController): - verbose_name = _("Mail system users") +class UNIXUserMaildirBackend(ServiceController): + verbose_name = _("UNIX maildir user") model = 'mailboxes.Mailbox' def save(self, mailbox): @@ -70,11 +70,11 @@ class MailSystemUserBackend(ServiceController): 'home': mailbox.get_home(), 'initial_shell': '/dev/null', } - return context + return replace(context, "'", '"') -class PasswdVirtualUserBackend(ServiceController): - verbose_name = _("Mail virtual user (passwd-file)") +class DovecotPostfixPasswdVirtualUserBackend(ServiceController): + verbose_name = _("Dovecot-Postfix virtualuser") model = 'mailboxes.Mailbox' # TODO related_models = ('resources__content_type') ?? needed for updating disk usage from resource.data @@ -166,7 +166,7 @@ class PasswdVirtualUserBackend(ServiceController): } context['extra_fields'] = self.get_extra_fields(mailbox, context) context['passwd'] = '{user}:{password}:{uid}:{gid}::{home}::{extra_fields}'.format(**context) - return context + return replace(context, "'", '"') class PostfixAddressBackend(ServiceController): @@ -178,15 +178,15 @@ class PostfixAddressBackend(ServiceController): def include_virtual_alias_domain(self, context): self.append(textwrap.dedent(""" - [[ $(grep "^\s*%(domain)s\s*$" %(virtual_alias_domains)s) ]] || { - echo "%(domain)s" >> %(virtual_alias_domains)s + [[ $(grep '^\s*%(domain)s\s*$' %(virtual_alias_domains)s) ]] || { + echo '%(domain)s' >> %(virtual_alias_domains)s UPDATED_VIRTUAL_ALIAS_DOMAINS=1 }""") % context) def exclude_virtual_alias_domain(self, context): domain = context['domain'] if not Address.objects.filter(domain=domain).exists(): - self.append('sed -i "/^%(domain)s\s*/d" %(virtual_alias_domains)s' % context) + self.append("sed -i '/^%(domain)s\s*/d' %(virtual_alias_domains)s" % context) def update_virtual_alias_maps(self, address, context): # Virtual mailbox stuff @@ -201,8 +201,8 @@ class PostfixAddressBackend(ServiceController): if destination: context['destination'] = destination self.append(textwrap.dedent(""" - LINE="%(email)s\t%(destination)s" - if [[ ! $(grep "^%(email)s\s" %(virtual_alias_maps)s) ]]; then + LINE='%(email)s\t%(destination)s' + if [[ ! $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then echo "${LINE}" >> %(virtual_alias_maps)s UPDATED_VIRTUAL_ALIAS_MAPS=1 else @@ -213,13 +213,13 @@ class PostfixAddressBackend(ServiceController): fi""") % context) else: logger.warning("Address %i is empty" % address.pk) - self.append('sed -i "/^%(email)s\s/d" %(virtual_alias_maps)s' % context) + self.append("sed -i '/^%(email)s\s/d' %(virtual_alias_maps)s" % context) self.append('UPDATED_VIRTUAL_ALIAS_MAPS=1') def exclude_virtual_alias_maps(self, context): self.append(textwrap.dedent(""" - if [[ $(grep "^%(email)s\s" %(virtual_alias_maps)s) ]]; then - sed -i "/^%(email)s\s.*$/d" %(virtual_alias_maps)s + if [[ $(grep '^%(email)s\s' %(virtual_alias_maps)s) ]]; then + sed -i '/^%(email)s\s.*$/d' %(virtual_alias_maps)s UPDATED_VIRTUAL_ALIAS_MAPS=1 fi""") % context) @@ -255,15 +255,15 @@ class PostfixAddressBackend(ServiceController): 'email': address.email, 'mailbox_domain': settings.MAILBOXES_VIRTUAL_MAILBOX_DEFAULT_DOMAIN, }) - return context + return replace(context, "'", '"') class AutoresponseBackend(ServiceController): verbose_name = _("Mail autoresponse") - model = 'mail.Autoresponse' + model = 'mailboxes.Autoresponse' -class MaildirDisk(ServiceMonitor): +class DovecotMaildirDisk(ServiceMonitor): """ Maildir disk usage based on Dovecot maildirsize file @@ -271,10 +271,10 @@ class MaildirDisk(ServiceMonitor): """ model = 'mailboxes.Mailbox' resource = ServiceMonitor.DISK - verbose_name = _("Maildir disk usage") + verbose_name = _("Dovecot Maildir size") def prepare(self): - super(MaildirDisk, self).prepare() + super(DovecotMaildirDisk, self).prepare() current_date = self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z") self.append(textwrap.dedent("""\ function monitor () { @@ -291,21 +291,21 @@ class MaildirDisk(ServiceMonitor): 'object_id': mailbox.pk } context['maildir_path'] = settings.MAILBOXES_MAILDIRSIZE_PATH % context - return context + return replace(context, "'", '"') -class PostfixTraffic(ServiceMonitor): +class PostfixMailscannerTraffic(ServiceMonitor): """ A high-performance log parser Reads the mail.log file only once, for all users """ model = 'mailboxes.Mailbox' resource = ServiceMonitor.TRAFFIC - verbose_name = _("Postfix traffic usage") + verbose_name = _("Postfix-Mailscanner traffic") script_executable = '/usr/bin/python' def prepare(self): - mail_log = '/var/log/mail.log' + mail_log = settings.MAILBOXES_MAIL_LOG_PATH context = { 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), 'mail_logs': str((mail_log, mail_log+'.1')), @@ -444,12 +444,12 @@ class PostfixTraffic(ServiceMonitor): self.append("prepare(%(object_id)s, '%(mailbox)s', '%(last_date)s')" % context) def get_context(self, mailbox): - return { -# 'mainlog': settings.LISTS_MAILMAN_POST_LOG_PATH, + context = { 'mailbox': mailbox.name, 'object_id': mailbox.pk, 'last_date': self.get_last_date(mailbox.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), } + return replace(context, "'", '"') diff --git a/orchestra/contrib/mailboxes/settings.py b/orchestra/contrib/mailboxes/settings.py index e5b3db55..2717a1a3 100644 --- a/orchestra/contrib/mailboxes/settings.py +++ b/orchestra/contrib/mailboxes/settings.py @@ -89,3 +89,8 @@ MAILBOXES_MAILDIRSIZE_PATH = getattr(settings, 'MAILBOXES_MAILDIRSIZE_PATH', MAILBOXES_LOCAL_ADDRESS_DOMAIN = getattr(settings, 'MAILBOXES_LOCAL_ADDRESS_DOMAIN', BASE_DOMAIN ) + + +MAILBOXES_MAIL_LOG_PATH = getattr(settings, 'MAILBOXES_MAIL_LOG_PATH', + '/var/log/mail.log' +) diff --git a/orchestra/contrib/orchestration/__init__.py b/orchestra/contrib/orchestration/__init__.py index 6c10a602..2ea5b1e3 100644 --- a/orchestra/contrib/orchestration/__init__.py +++ b/orchestra/contrib/orchestration/__init__.py @@ -1 +1 @@ -from .backends import ServiceBackend, ServiceController +from .backends import ServiceBackend, ServiceController, replace diff --git a/orchestra/contrib/orchestration/backends.py b/orchestra/contrib/orchestration/backends.py index ff63ed6e..8aba5959 100644 --- a/orchestra/contrib/orchestration/backends.py +++ b/orchestra/contrib/orchestration/backends.py @@ -1,3 +1,4 @@ +import re from functools import partial from django.apps import apps @@ -9,6 +10,15 @@ from orchestra import plugins from . import methods +def replace(context, pattern, repl): + if isinstance(context, str): + return context.replace(patter, repl) + for key, value in context.items(): + if isinstance(value, str): + context[key] = value.replace(pattern, repl) + return context + + class ServiceMount(plugins.PluginMount): def __init__(cls, name, bases, attrs): # Make sure backends specify a model attribute diff --git a/orchestra/contrib/orchestration/management/commands/orchestrate.py b/orchestra/contrib/orchestration/management/commands/orchestrate.py index a3f0c229..24b6893e 100644 --- a/orchestra/contrib/orchestration/management/commands/orchestrate.py +++ b/orchestra/contrib/orchestration/management/commands/orchestrate.py @@ -8,17 +8,23 @@ from orchestra.contrib.orchestration import manager class Command(BaseCommand): help = 'Runs orchestration backends.' - option_list = BaseCommand.option_list - args = "[app_label] [filter]" + + def add_arguments(self, parser): + parser.add_argument('model', nargs='+', + help='App label of an application to synchronize the + parser.add_argument('query', nargs='?', + help='Query arguments for filter().') + parser.add_argument('--noinput', action='store_false', dest='interactive', default=True, + help='Tells Django to NOT prompt the user for input of any kind.') + parser.add_argument('--action', action='store', dest='database', + default='save', help='Executes action. Defaults to "save".') def handle(self, *args, **options): - model_label = args[0] - model = get_model(*model_label.split('.')) - # TODO options - action = options.get('action', 'save') - interactive = options.get('interactive', True) + model = get_model(*options['model'].split('.')) + action = options.get('action') + interactive = options.get('interactive') kwargs = {} - for comp in args[1:]: + for comp in options.get('query', []): comps = iter(comp.split('=')) for arg in comps: kwargs[arg] = next(comps).strip().rstrip(',') @@ -51,4 +57,3 @@ class Command(BaseCommand): return break # manager.execute(scripts, block=block) - diff --git a/orchestra/contrib/orchestration/manager.py b/orchestra/contrib/orchestration/manager.py index 47b06bc6..08b602e6 100644 --- a/orchestra/contrib/orchestration/manager.py +++ b/orchestra/contrib/orchestration/manager.py @@ -173,6 +173,8 @@ def collect(instance, action, **kwargs): else: update_fields = kwargs.get('update_fields', None) if update_fields is not None: + # TODO remove this, django does not execute post_save if update_fields=[]... + # Maybe open a ticket at Djangoproject ? # "update_fileds=[]" is a convention for explicitly executing backend # i.e. account.disable() if update_fields != []: diff --git a/orchestra/contrib/payments/api.py b/orchestra/contrib/payments/api.py index a6976796..03effe6a 100644 --- a/orchestra/contrib/payments/api.py +++ b/orchestra/contrib/payments/api.py @@ -1,18 +1,18 @@ from rest_framework import viewsets -from orchestra.api import router +from orchestra.api import router, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from .models import PaymentSource, Transaction from .serializers import PaymentSourceSerializer, TransactionSerializer -class PaymentSourceViewSet(AccountApiMixin, viewsets.ModelViewSet): +class PaymentSourceViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = PaymentSource serializer_class = PaymentSourceSerializer -class TransactionViewSet(viewsets.ModelViewSet): +class TransactionViewSet(LogApiMixin, viewsets.ModelViewSet): model = Transaction serializer_class = TransactionSerializer diff --git a/orchestra/contrib/saas/backends/bscw.py b/orchestra/contrib/saas/backends/bscw.py index 62fcc6d5..b45d0a99 100644 --- a/orchestra/contrib/saas/backends/bscw.py +++ b/orchestra/contrib/saas/backends/bscw.py @@ -2,7 +2,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from .. import settings @@ -46,9 +46,10 @@ class BSCWBackend(ServiceController): self.append("%(bsadmin)s rmuser -n %(username)s" % context) def get_context(self, saas): - return { + context = { 'bsadmin': settings.SAAS_BSCW_BSADMIN_PATH, 'email': saas.data.get('email'), 'username': saas.name, 'password': getattr(saas, 'password', None), } + return replace(context, "'", '"') diff --git a/orchestra/contrib/saas/backends/dokuwikimu.py b/orchestra/contrib/saas/backends/dokuwikimu.py index 9d64d2e5..ce19043b 100644 --- a/orchestra/contrib/saas/backends/dokuwikimu.py +++ b/orchestra/contrib/saas/backends/dokuwikimu.py @@ -2,7 +2,7 @@ import os from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from .. import settings @@ -28,4 +28,4 @@ class DokuWikiMuBackend(ServiceController): 'template': settings.WEBAPPS_DOKUWIKIMU_TEMPLATE_PATH, 'app_path': os.path.join(settings.WEBAPPS_DOKUWIKIMU_FARM_PATH, webapp.name) }) - return context + return replace(context, "'", '"') diff --git a/orchestra/contrib/saas/backends/drupalmu.py b/orchestra/contrib/saas/backends/drupalmu.py index 26da93c2..5cd8f16b 100644 --- a/orchestra/contrib/saas/backends/drupalmu.py +++ b/orchestra/contrib/saas/backends/drupalmu.py @@ -3,7 +3,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from .. import settings @@ -34,4 +34,4 @@ class DrupalMuBackend(ServiceController): context = super(DrupalMuBackend, self).get_context(webapp) context['drupal_path'] = settings.WEBAPPS_DRUPAL_SITES_PATH % context context['drupal_settings'] = os.path.join(context['drupal_path'], 'settings.php') - return context + return replace(context, "'", '"') diff --git a/orchestra/contrib/saas/services/options.py b/orchestra/contrib/saas/services/options.py index 1861f5ae..c1d956b5 100644 --- a/orchestra/contrib/saas/services/options.py +++ b/orchestra/contrib/saas/services/options.py @@ -1,5 +1,6 @@ from django import forms from django.core.exceptions import ValidationError +from django.core.validators import RegexValidator from django.utils.translation import ugettext_lazy as _ from orchestra import plugins @@ -16,10 +17,16 @@ from .. import settings class SoftwareServiceForm(PluginDataForm): site_url = forms.CharField(label=_("Site URL"), widget=widgets.ShowTextWidget, required=False) password = forms.CharField(label=_("Password"), required=False, - widget=widgets.ReadOnlyWidget('Unknown password'), - help_text=_("Passwords are not stored, so there is no way to see this " - "service's password, but you can change the password using " - "this form.")) + widget=widgets.ReadOnlyWidget('Unknown password'), + validators=[ + RegexValidator(r'^[^"\'\\]+$', + _('Enter a valid password. ' + 'This value may contain any ascii character except for ' + ' \'/"/\\/ characters.'), 'invalid'), + ], + help_text=_("Passwords are not stored, so there is no way to see this " + "service's password, but you can change the password using " + "this form.")) password1 = forms.CharField(label=_("Password"), validators=[validators.validate_password], widget=forms.PasswordInput) password2 = forms.CharField(label=_("Password confirmation"), diff --git a/orchestra/contrib/systemusers/api.py b/orchestra/contrib/systemusers/api.py index 2775edbb..f7ddae18 100644 --- a/orchestra/contrib/systemusers/api.py +++ b/orchestra/contrib/systemusers/api.py @@ -1,14 +1,14 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework import viewsets, exceptions -from orchestra.api import router, SetPasswordApiMixin +from orchestra.api import router, SetPasswordApiMixin, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from .models import SystemUser from .serializers import SystemUserSerializer -class SystemUserViewSet(AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): +class SystemUserViewSet(LogApiMixin, AccountApiMixin, SetPasswordApiMixin, viewsets.ModelViewSet): model = SystemUser serializer_class = SystemUserSerializer filter_fields = ('username',) diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index 96c1ff6f..98f3bcbe 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -3,14 +3,14 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from orchestra.contrib.resources import ServiceMonitor from . import settings -class SystemUserBackend(ServiceController): - verbose_name = _("System user") +class UNIXUserBackend(ServiceController): + verbose_name = _("UNIX user") model = 'systemusers.SystemUser' actions = ('save', 'delete', 'grant_permission') @@ -73,16 +73,16 @@ class SystemUserBackend(ServiceController): 'mainuser': user.username if user.is_main else user.account.username, 'home': user.get_home() } - return context + return replace(context, "'", '"') -class SystemUserDisk(ServiceMonitor): +class UNIXUserDisk(ServiceMonitor): model = 'systemusers.SystemUser' resource = ServiceMonitor.DISK - verbose_name = _('Systemuser disk') + verbose_name = _('UNIX user disk') def prepare(self): - super(SystemUserDisk, self).prepare() + super(UNIXUserDisk, self).prepare() self.append(textwrap.dedent("""\ function monitor () { { du -bs "$1" || echo 0; } | awk {'print $1'} @@ -98,85 +98,21 @@ class SystemUserDisk(ServiceMonitor): self.append("echo %(object_id)s 0" % context) def get_context(self, user): - return { + context = { 'object_id': user.pk, 'home': user.home, } - - -class FTPTrafficBash(ServiceMonitor): - model = 'systemusers.SystemUser' - resource = ServiceMonitor.TRAFFIC - verbose_name = _('Systemuser FTP traffic (Bash)') - - def prepare(self): - super(FTPTrafficBash, self).prepare() - context = { - 'log_file': '%s{,.1}' % settings.SYSTEMUSERS_FTP_LOG_PATH, - 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), - } - self.append(textwrap.dedent("""\ - function monitor () { - OBJECT_ID=$1 - INI_DATE=$(date "+%%Y%%m%%d%%H%%M%%S" -d "$2") - END_DATE=$(date '+%%Y%%m%%d%%H%%M%%S' -d '%(current_date)s') - USERNAME="$3" - LOG_FILE=%(log_file)s - { - grep " bytes, " ${LOG_FILE} \\ - | grep " \\[${USERNAME}\\] " \\ - | awk -v ini="${INI_DATE}" -v end="${END_DATE}" ' - BEGIN { - sum = 0 - months["Jan"] = "01" - months["Feb"] = "02" - months["Mar"] = "03" - months["Apr"] = "04" - months["May"] = "05" - months["Jun"] = "06" - months["Jul"] = "07" - months["Aug"] = "08" - months["Sep"] = "09" - months["Oct"] = "10" - months["Nov"] = "11" - months["Dec"] = "12" - } { - # Fri Jul 1 13:23:17 2014 - split($4, time, ":") - day = sprintf("%%02d", $3) - # line_date = year month day hour minute second - line_date = $5 months[$2] day time[1] time[2] time[3] - if ( line_date > ini && line_date < end) { - sum += $(NF-2) - } - } END { - print sum - }' || [[ $? == 1 ]] && true - } | xargs echo ${OBJECT_ID} - }""") % context) - - def monitor(self, user): - context = self.get_context(user) - self.append( - 'monitor {object_id} "{last_date}" "{username}"'.format(**context) - ) - - def get_context(self, user): - return { - 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), - 'object_id': user.pk, - 'username': user.username, - } + return replace(context, "'", '"') class Exim4Traffic(ServiceMonitor): model = 'systemusers.SystemUser' resource = ServiceMonitor.TRAFFIC - verbose_name = _("Exim4 traffic usage") + verbose_name = _("Exim4 traffic") script_executable = '/usr/bin/python' def prepare(self): - mainlog = '/var/log/exim4/mainlog' + mainlog = settings.LISTS_MAILMAN_POST_LOG_PATH context = { 'current_date': self.current_date.strftime("%Y-%m-%d %H:%M:%S %Z"), 'mainlogs': str((mainlog, mainlog+'.1')), @@ -240,19 +176,18 @@ class Exim4Traffic(ServiceMonitor): self.append("prepare(%(object_id)s, '%(username)s', '%(last_date)s')" % context) def get_context(self, user): - return { -# 'mainlog': settings.LISTS_MAILMAN_POST_LOG_PATH, + context = { 'username': user.username, 'object_id': user.pk, 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), } + return replace(context, "'", '"') - -class FTPTraffic(ServiceMonitor): +class VsFTPdTraffic(ServiceMonitor): model = 'systemusers.SystemUser' resource = ServiceMonitor.TRAFFIC - verbose_name = _('Systemuser FTP traffic') + verbose_name = _('VsFTPd traffic') script_executable = '/usr/bin/python' def prepare(self): @@ -266,13 +201,13 @@ class FTPTraffic(ServiceMonitor): import sys from datetime import datetime from dateutil import tz - + def to_local_timezone(date, tzlocal=tz.tzlocal()): date = datetime.strptime(date, '%Y-%m-%d %H:%M:%S %Z') date = date.replace(tzinfo=tz.tzutc()) date = date.astimezone(tzlocal) return date - + vsftplogs = {vsftplogs} # Use local timezone end_date = to_local_timezone('{current_date}') @@ -292,13 +227,13 @@ class FTPTraffic(ServiceMonitor): 'Nov': '11', 'Dec': '12', }} - + def prepare(object_id, username, ini_date): global users ini_date = to_local_timezone(ini_date) ini_date = int(ini_date.strftime('%Y%m%d%H%M%S')) users[username] = [ini_date, object_id, 0] - + def monitor(users, end_date, months, vsftplogs): user_regex = re.compile(r'\] \[([^ ]+)\] (OK|FAIL) ') bytes_regex = re.compile(r', ([0-9]+) bytes, ') @@ -335,9 +270,10 @@ class FTPTraffic(ServiceMonitor): self.append('monitor(users, end_date, months, vsftplogs)') def get_context(self, user): - return { + context = { 'last_date': self.get_last_date(user.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': user.pk, 'username': user.username, } + return replace(context, "'", '"') diff --git a/orchestra/contrib/vps/backends.py b/orchestra/contrib/vps/backends.py index edbd303a..7a83bd55 100644 --- a/orchestra/contrib/vps/backends.py +++ b/orchestra/contrib/vps/backends.py @@ -1,3 +1,4 @@ +from orchestra.contrib.orchestration import replace from orchestra.contrib.resources import ServiceMonitor @@ -27,6 +28,7 @@ class OpenVZTraffic(ServiceMonitor): " | awk '{print $1+$9}'") def get_context(self, container): - return { + context = { 'hostname': container.hostname, } + return replace(context, "'", '"') diff --git a/orchestra/contrib/webapps/api.py b/orchestra/contrib/webapps/api.py index 86231b1e..b9d31381 100644 --- a/orchestra/contrib/webapps/api.py +++ b/orchestra/contrib/webapps/api.py @@ -1,6 +1,6 @@ from rest_framework import viewsets -from orchestra.api import router +from orchestra.api import router, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from . import settings @@ -8,7 +8,7 @@ from .models import WebApp from .serializers import WebAppSerializer -class WebAppViewSet(AccountApiMixin, viewsets.ModelViewSet): +class WebAppViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = WebApp serializer_class = WebAppSerializer filter_fields = ('name',) diff --git a/orchestra/contrib/webapps/backends/__init__.py b/orchestra/contrib/webapps/backends/__init__.py index 8a7e3c48..593f4cad 100644 --- a/orchestra/contrib/webapps/backends/__init__.py +++ b/orchestra/contrib/webapps/backends/__init__.py @@ -1,6 +1,8 @@ import pkgutil import textwrap +from orchestra.contrib.orchestration.backends import replace + from .. import settings @@ -30,7 +32,7 @@ class WebAppServiceMixin(object): self.append("rm -fr %(app_path)s" % context) def get_context(self, webapp): - return { + context = { 'user': webapp.get_username(), 'group': webapp.get_groupname(), 'app_name': webapp.name, @@ -40,6 +42,7 @@ class WebAppServiceMixin(object): 'under_construction_path': settings.settings.WEBAPPS_UNDER_CONSTRUCTION_PATH, 'is_mounted': webapp.content_set.exists(), } + replace(context, "'", '"') for __, module_name, __ in pkgutil.walk_packages(__path__): diff --git a/orchestra/contrib/webapps/backends/php.py b/orchestra/contrib/webapps/backends/php.py index 42b9a2a5..3d43575e 100644 --- a/orchestra/contrib/webapps/backends/php.py +++ b/orchestra/contrib/webapps/backends/php.py @@ -4,7 +4,7 @@ import textwrap from django.template import Template, Context from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from . import WebAppServiceMixin from .. import settings @@ -132,7 +132,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): # Format PHP init vars init_vars = opt.get_php_init_vars(merge=self.MERGE) if init_vars: - init_vars = [ '-d %s="%s"' % (k,v) for k,v in init_vars.items() ] + init_vars = [ "-d %s='%s'" % (k, v.replace("'", '"')) for k,v in init_vars.items() ] init_vars = ', '.join(init_vars) context.update({ 'php_binary': os.path.normpath(settings.WEBAPPS_PHP_CGI_BINARY_PATH % context), @@ -156,7 +156,9 @@ class PHPBackend(WebAppServiceMixin, ServiceController): cmd_options = [] for directive, value in maps.items(): if value: - cmd_options.append("%s %s" % (directive, value)) + cmd_options.append( + "%s %s" % (directive, value.replace("'", '"')) + ) if cmd_options: head = ( '# %(banner)s\n' @@ -172,6 +174,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController): 'wrapper_path': wrapper_path, 'wrapper_dir': os.path.dirname(wrapper_path), }) + replace(context, "'", '"') context.update({ 'cmd_options': self.get_fcgid_cmd_options(webapp, context), 'cmd_options_path': settings.WEBAPPS_FCGID_CMD_OPTIONS_PATH % context, @@ -191,6 +194,8 @@ class PHPBackend(WebAppServiceMixin, ServiceController): 'php_version_number': webapp.type_instance.get_php_version_number(), 'max_requests': settings.WEBAPPS_PHP_MAX_REQUESTS, }) - self.update_fcgid_context(webapp, context) self.update_fpm_context(webapp, context) + # Fcgid context do contain special charactes + replace(context, "'", '"') + self.update_fcgid_context(webapp, context) return context diff --git a/orchestra/contrib/webapps/backends/symboliclink.py b/orchestra/contrib/webapps/backends/symboliclink.py index b725ceef..6ac994f7 100644 --- a/orchestra/contrib/webapps/backends/symboliclink.py +++ b/orchestra/contrib/webapps/backends/symboliclink.py @@ -1,6 +1,6 @@ from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from . import WebAppServiceMixin @@ -24,4 +24,4 @@ class SymbolicLinkBackend(WebAppServiceMixin, ServiceController): context.update({ 'link_path': webapp.data['path'], }) - return context + return replace(context, "'", '"') diff --git a/orchestra/contrib/webapps/backends/wordpress.py b/orchestra/contrib/webapps/backends/wordpress.py index bad1948b..6347db18 100644 --- a/orchestra/contrib/webapps/backends/wordpress.py +++ b/orchestra/contrib/webapps/backends/wordpress.py @@ -2,7 +2,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from .. import settings @@ -49,10 +49,10 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): } array_pop($secret_keys); - $config_file = str_replace('database_name_here', '%(db_name)s', $config_file); - $config_file = str_replace('username_here', '%(db_user)s', $config_file); - $config_file = str_replace('password_here', '%(password)s', $config_file); - $config_file = str_replace('localhost', '%(db_host)s', $config_file); + $config_file = str_replace('database_name_here', "%(db_name)s", $config_file); + $config_file = str_replace('username_here', "%(db_user)s", $config_file); + $config_file = str_replace('password_here', "%(password)s", $config_file); + $config_file = str_replace('localhost', "%(db_host)s", $config_file); $config_file = str_replace("'AUTH_KEY', 'put your unique phrase here'", "'AUTH_KEY', '{$secret_keys[0]}'", $config_file); $config_file = str_replace("'SECURE_AUTH_KEY', 'put your unique phrase here'", "'SECURE_AUTH_KEY', '{$secret_keys[1]}'", $config_file); $config_file = str_replace("'LOGGED_IN_KEY', 'put your unique phrase here'", "'LOGGED_IN_KEY', '{$secret_keys[2]}'", $config_file); @@ -73,10 +73,10 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): define('WP_CONTENT_DIR', 'wp-content/'); define('WP_LANG_DIR', WP_CONTENT_DIR . '/languages' ); define('WP_USE_THEMES', true); - define('DB_NAME', '%(db_name)s'); - define('DB_USER', '%(db_user)s'); - define('DB_PASSWORD', '%(password)s'); - define('DB_HOST', '%(db_host)s'); + define('DB_NAME', "%(db_name)s"); + define('DB_USER', "%(db_user)s"); + define('DB_PASSWORD', "%(password)s"); + define('DB_HOST', "%(db_host)s"); $_GET['step'] = 2; $_POST['weblog_title'] = "%(title)s"; @@ -114,7 +114,7 @@ class WordPressBackend(WebAppServiceMixin, ServiceController): 'db_user': webapp.data['db_user'], 'password': webapp.data['password'], 'db_host': settings.WEBAPPS_DEFAULT_MYSQL_DATABASE_HOST, - 'title': "%s blog's" % webapp.account.get_full_name(), 'email': webapp.account.email, + 'title': "%s blog's" % webapp.account.get_full_name(), }) - return context + return replace(context, '"', "'") diff --git a/orchestra/contrib/websites/api.py b/orchestra/contrib/websites/api.py index 044836ae..8cf8e130 100644 --- a/orchestra/contrib/websites/api.py +++ b/orchestra/contrib/websites/api.py @@ -1,6 +1,6 @@ from rest_framework import viewsets -from orchestra.api import router +from orchestra.api import router, LogApiMixin from orchestra.contrib.accounts.api import AccountApiMixin from . import settings @@ -8,7 +8,7 @@ from .models import Website from .serializers import WebsiteSerializer -class WebsiteViewSet(AccountApiMixin, viewsets.ModelViewSet): +class WebsiteViewSet(LogApiMixin, AccountApiMixin, viewsets.ModelViewSet): model = Website serializer_class = WebsiteSerializer filter_fields = ('name',) diff --git a/orchestra/contrib/websites/backends/apache.py b/orchestra/contrib/websites/backends/apache.py index 4186967e..390bf730 100644 --- a/orchestra/contrib/websites/backends/apache.py +++ b/orchestra/contrib/websites/backends/apache.py @@ -5,7 +5,7 @@ import textwrap from django.template import Template, Context from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from orchestra.contrib.resources import ServiceMonitor from .. import settings @@ -82,7 +82,7 @@ class Apache2Backend(ServiceController): apache_conf += self.render_virtual_host(site, context, ssl=True) if site.protocol == site.HTTPS_ONLY: apache_conf += self.render_redirect_https(context) - context['apache_conf'] = apache_conf + context['apache_conf'] = apache_conf.replace("'", '"') self.append(textwrap.dedent("""\ apache_conf='%(apache_conf)s' { @@ -172,7 +172,7 @@ class Apache2Backend(ServiceController): ca = [settings.WEBSITES_DEFAULT_SSL_CA] if not (cert and key): return [] - config = 'SSLEngine on\n' + config = "SSLEngine on\n" config += "SSLCertificateFile %s\n" % cert[0] config += "SSLCertificateKeyFile %s\n" % key[0] if ca: @@ -297,7 +297,7 @@ class Apache2Backend(ServiceController): 'error_log': site.get_www_error_log_path(), 'banner': self.get_banner(), } - return context + return replace(context, "'", '"') def get_content_context(self, content): context = self.get_context(content.website) @@ -307,7 +307,7 @@ class Apache2Backend(ServiceController): 'app_name': content.webapp.name, 'app_path': content.webapp.get_path(), }) - return context + return replace(context, "'", '"') class Apache2Traffic(ServiceMonitor): @@ -368,8 +368,9 @@ class Apache2Traffic(ServiceMonitor): self.append('monitor {object_id} "{last_date}" {log_file}'.format(**context)) def get_context(self, site): - return { + context = { 'log_file': '%s{,.1}' % site.get_www_access_log_path(), 'last_date': self.get_last_date(site.pk).strftime("%Y-%m-%d %H:%M:%S %Z"), 'object_id': site.pk, } + return replace(context, "'", '"') diff --git a/orchestra/contrib/websites/backends/webalizer.py b/orchestra/contrib/websites/backends/webalizer.py index bfb196a5..e775efe4 100644 --- a/orchestra/contrib/websites/backends/webalizer.py +++ b/orchestra/contrib/websites/backends/webalizer.py @@ -3,7 +3,7 @@ import textwrap from django.utils.translation import ugettext_lazy as _ -from orchestra.contrib.orchestration import ServiceController +from orchestra.contrib.orchestration import ServiceController, replace from .. import settings @@ -91,4 +91,4 @@ class WebalizerBackend(ServiceController): SearchEngine alltheweb.com query= DumpSites yes""") % context - return context + return replace(context, "'", '"') diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index 2b473d90..5a90564e 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -61,7 +61,7 @@ def validate_name(value): def validate_ascii(value): try: - value.decode('ascii') + value.encode('ascii') except UnicodeDecodeError: raise ValidationError('This is not an ASCII string.')