From 7133bd31eabc38bdb7a116446f5351a9d1c3822b Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Sat, 4 Apr 2015 17:44:07 +0000 Subject: [PATCH] flake8 --- INSTALL.md | 24 ++-- TODO.md | 29 ++-- orchestra/admin/actions.py | 2 +- orchestra/admin/forms.py | 6 +- orchestra/admin/menu.py | 3 - orchestra/admin/options.py | 11 +- orchestra/api/fields.py | 2 +- orchestra/api/helpers.py | 6 - orchestra/api/options.py | 4 +- orchestra/api/serializers.py | 2 +- orchestra/apps/accounts/actions.py | 131 ++++++++++++++++-- orchestra/apps/accounts/admin.py | 4 +- orchestra/apps/accounts/forms.py | 3 +- orchestra/apps/accounts/models.py | 1 - orchestra/apps/bills/actions.py | 1 + orchestra/apps/domains/serializers.py | 1 + orchestra/apps/orchestration/admin.py | 1 + orchestra/apps/orders/models.py | 7 +- orchestra/apps/payments/models.py | 4 +- orchestra/apps/resources/methods.py | 2 +- orchestra/apps/saas/admin.py | 3 +- orchestra/apps/saas/backends/__init__.py | 1 - orchestra/apps/saas/backends/phplist.py | 4 - orchestra/apps/saas/services/bscw.py | 1 - orchestra/apps/saas/services/phplist.py | 3 +- orchestra/apps/services/admin.py | 1 - orchestra/apps/services/handlers.py | 2 + orchestra/apps/services/models.py | 4 - orchestra/apps/services/settings.py | 2 - .../tests/functional_tests/__init__.py | 9 -- orchestra/apps/systemusers/backends.py | 4 + orchestra/apps/webapps/types/__init__.py | 3 +- orchestra/apps/webapps/types/php.py | 1 + orchestra/apps/websites/backends/apache.py | 2 +- orchestra/apps/websites/forms.py | 1 + orchestra/apps/websites/serializers.py | 1 - orchestra/bin/orchestra-admin | 1 + orchestra/conf/base_settings.py | 4 +- orchestra/core/__init__.py | 2 +- orchestra/core/middlewares.py | 47 ------- orchestra/core/validators.py | 3 +- orchestra/forms/options.py | 2 +- orchestra/management/commands/staticcheck.py | 100 +------------ orchestra/models/fields.py | 2 +- orchestra/plugins/admin.py | 1 - orchestra/utils/system.py | 6 +- orchestra/utils/tests.py | 1 - scripts/container/create.sh | 3 +- scripts/container/deploy.sh | 24 ++-- 49 files changed, 220 insertions(+), 262 deletions(-) delete mode 100644 orchestra/core/middlewares.py diff --git a/INSTALL.md b/INSTALL.md index e21290a8..10ee038d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -5,7 +5,7 @@ Django-orchestra ships with a set of management commands for automating some of These commands are meant to be run within a **clean** Debian-like distribution, you should be specially careful while following this guide on a customized system. -Django-orchestra can be installed on any Linux system, however it is **strongly recommended** to chose the reference platform for your deployment (Debian 7.0 wheezy and Python 2.7). +Django-orchestra can be installed on any Linux system, however it is **strongly recommended** to chose the reference platform for your deployment (Debian 8.0 jessie and Python 3.4). 1. Create a system user for running Orchestra @@ -18,7 +18,7 @@ Django-orchestra can be installed on any Linux system, however it is **strongly 2. Install django-orchestra's source code ```bash - sudo apt-get install python-pip + sudo apt-get install python3-pip sudo pip install django-orchestra==dev ``` @@ -38,21 +38,21 @@ Django-orchestra can be installed on any Linux system, however it is **strongly 5. Create and configure a Postgres database ```bash - sudo python manage.py setuppostgres --db_password - python manage.py syncdb - python manage.py migrate + sudo python3 manage.py setuppostgres --db_password + python3 manage.py syncdb + python3 manage.py migrate ``` 7. Configure celeryd ```bash - sudo python manage.py setupcelery --username orchestra + sudo python3 manage.py setupcelery --username orchestra ``` 8. Configure the web server: ```bash - python manage.py collectstatic --noinput - sudo apt-get install nginx-full uwsgi uwsgi-plugin-python - sudo python manage.py setupnginx + python3 manage.py collectstatic --noinput + sudo apt-get install nginx-full uwsgi uwsgi-plugin-python3 + sudo python3 manage.py setupnginx ``` 9. Start all services: @@ -65,17 +65,17 @@ Upgrade ======= To upgrade your Orchestra installation to the last release you can use `upgradeorchestra` management command. Before rolling the upgrade it is strongly recommended to check the [release notes](http://django-orchestra.readthedocs.org/en/latest/). ```bash -sudo python manage.py upgradeorchestra +sudo python3 manage.py upgradeorchestra ``` Current in *development* version (master branch) can be installed by ```bash -sudo python manage.py upgradeorchestra dev +sudo python3 manage.py upgradeorchestra dev ``` Additionally the following command can be used in order to determine the currently installed version: ```bash -python manage.py orchestraversion +python3 manage.py orchestraversion ``` diff --git a/TODO.md b/TODO.md index 48edb8ae..0042680b 100644 --- a/TODO.md +++ b/TODO.md @@ -286,7 +286,7 @@ ugettext("Description") * saas validate_creation generic approach, for all backends. standard output -* html code x: × +* html code x: × for bill line verbose quantity * periodic task to cleanup backendlogs, monitor data and metricstorage @@ -299,29 +299,36 @@ celery max-tasks-per-child * postupgradeorchestra send signals in order to hook custom stuff -* make base home for systemusers that ara homed into main account systemuser +* make base home for systemusers that ara homed into main account systemuser, and prevent shell users to have nested homes (if nnot implemented already) -* user force_text instead of unicode for _() - * autoscale celery workers http://docs.celeryproject.org/en/latest/userguide/workers.html#autoscaling - -* Delete transaction middleware - - * webapp has_website list filter glic3rinu's django-fluent-dashboard * gevent is not ported to python3 :'( * uwsgi python3 -https://github.com/django-nose/django-nose/archive/master.zip -django_debug_toolbar-1.3.0-py2.py3-none-any.whl -# FIXME account deletion generates a integrity error +# FIXME account deletion generates an integrity error +https://code.djangoproject.com/ticket/24576 # FIXME what to do when deleting accounts? set fk null and fill a username charfield? issues, invoices.. we whant all this to go away? * implement delete All related services * address name change does not remove old one :P + +* read https://docs.djangoproject.com/en/dev/releases/1.8/ and fix deprecation warnings +* remove admin object links , like contents webapps + +* SaaS and WebApp fieldsets, and helptexts ! + +* remove all six stuff "from django.utils.six.moves import input" +* replace make_option in management commands + + +* rename apps to contrib find . -type f -name "*py"|xargs grep apps | grep -v 'orchestra\.apps\.' + +* replace staticcheck by run(flake8 {orchestra,project} | grep -v "W293\|E501") +* rename utils.system to utils.sys diff --git a/orchestra/admin/actions.py b/orchestra/admin/actions.py index 1f5a32d1..19b756d8 100644 --- a/orchestra/admin/actions.py +++ b/orchestra/admin/actions.py @@ -1,4 +1,4 @@ -from django.contrib import admin, messages +from django.contrib import admin from django.core.mail import send_mass_mail from django.shortcuts import render from django.utils.translation import ungettext, ugettext_lazy as _ diff --git a/orchestra/admin/forms.py b/orchestra/admin/forms.py index 18ca411c..548bb688 100644 --- a/orchestra/admin/forms.py +++ b/orchestra/admin/forms.py @@ -139,13 +139,13 @@ class AdminPasswordChangeForm(forms.Form): class SendEmailForm(forms.Form): email_from = forms.EmailField(label=_("From"), - widget=forms.TextInput(attrs={'size':'118'})) + widget=forms.TextInput(attrs={'size': '118'})) to = forms.CharField(label="To", required=False, widget=ShowTextWidget()) extra_to = forms.CharField(label="To (extra)", required=False, - widget=forms.TextInput(attrs={'size':'118'})) + widget=forms.TextInput(attrs={'size': '118'})) subject = forms.CharField(label=_("Subject"), - widget=forms.TextInput(attrs={'size':'118'})) + widget=forms.TextInput(attrs={'size': '118'})) message = forms.CharField(label=_("Message"), widget=forms.Textarea(attrs={'cols': 118, 'rows': 15})) diff --git a/orchestra/admin/menu.py b/orchestra/admin/menu.py index bf268378..b653307f 100644 --- a/orchestra/admin/menu.py +++ b/orchestra/admin/menu.py @@ -1,11 +1,8 @@ from admin_tools.menu import items, Menu from django.core.urlresolvers import reverse -from django.utils.encoding import force_text from django.utils.text import capfirst from django.utils.translation import ugettext_lazy as _ -from django.utils.safestring import mark_safe -from orchestra import get_version, settings from orchestra.core import services, accounts from orchestra.utils.apps import isinstalled diff --git a/orchestra/admin/options.py b/orchestra/admin/options.py index cd936cf9..8b66551c 100644 --- a/orchestra/admin/options.py +++ b/orchestra/admin/options.py @@ -5,19 +5,18 @@ from django.contrib.admin.options import IS_POPUP_VAR from django.contrib.admin.utils import unquote from django.contrib.auth import update_session_auth_hash from django.core.exceptions import PermissionDenied -from django.http import HttpResponseRedirect, Http404 +from django.http import HttpResponseRedirect from django.forms.models import BaseInlineFormSet -from django.shortcuts import render, redirect, get_object_or_404 +from django.shortcuts import get_object_or_404 from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.utils.html import escape -from django.utils.text import camel_case_to_spaces from django.utils.translation import ugettext_lazy as _ from django.views.decorators.debug import sensitive_post_parameters from .forms import AdminPasswordChangeForm #from django.contrib.auth.forms import AdminPasswordChangeForm -from .utils import set_url_query, action_to_view, wrap_admin_view +from .utils import set_url_query, action_to_view sensitive_post_parameters_m = method_decorator(sensitive_post_parameters()) @@ -35,8 +34,8 @@ class ChangeListDefaultFilter(object): """ Default filter as 'my_nodes=True' """ defaults = [] for key, value in self.default_changelist_filters: - set_url_query(request, key, value) - defaults.append(key) + set_url_query(request, key, value) + defaults.append(key) # hack response cl context in order to hook default filter awaearness # into search_form.html template response = super(ChangeListDefaultFilter, self).changelist_view(request, extra_context) diff --git a/orchestra/api/fields.py b/orchestra/api/fields.py index 6183898b..18ff0236 100644 --- a/orchestra/api/fields.py +++ b/orchestra/api/fields.py @@ -27,7 +27,7 @@ class OptionField(serializers.WritableField): raise exceptions.ParseError("Malformed property: %s" % str(value)) if not related_manager: # POST (new parent object) - return [ model(name=n, value=v) for n,v in value.items() ] + return [model(name=n, value=v) for n,v in value.items()] # PUT to_save = [] for (name, value) in value.items(): diff --git a/orchestra/api/helpers.py b/orchestra/api/helpers.py index 80b7c6a9..6f6ccc9e 100644 --- a/orchestra/api/helpers.py +++ b/orchestra/api/helpers.py @@ -3,12 +3,6 @@ from rest_framework.reverse import reverse from rest_framework.routers import replace_methodname -def replace_collectionmethodname(format_string, methodname): - ret = replace_methodname(format_string, methodname) - ret = ret.replace('{collectionmethodname}', methodname) - return ret - - def link_wrap(view, view_names): def wrapper(self, request, view=view, *args, **kwargs): """ wrapper function that inserts HTTP links on view """ diff --git a/orchestra/api/options.py b/orchestra/api/options.py index 64fb911e..5df8187b 100644 --- a/orchestra/api/options.py +++ b/orchestra/api/options.py @@ -1,12 +1,12 @@ from django.conf import settings as django_settings from django.core.exceptions import ImproperlyConfigured from django.utils.module_loading import autodiscover_modules -from rest_framework.routers import DefaultRouter, Route, flatten, replace_methodname +from rest_framework.routers import DefaultRouter, Route, replace_methodname from orchestra import settings from orchestra.utils.python import import_class -from .helpers import insert_links, replace_collectionmethodname +from .helpers import insert_links class LinkHeaderRouter(DefaultRouter): diff --git a/orchestra/api/serializers.py b/orchestra/api/serializers.py index 21f17727..9096383b 100644 --- a/orchestra/api/serializers.py +++ b/orchestra/api/serializers.py @@ -1,5 +1,5 @@ from django.forms import widgets -from django.utils.translation import ugettext, ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from ..core.validators import validate_password diff --git a/orchestra/apps/accounts/actions.py b/orchestra/apps/accounts/actions.py index 31994b02..a507c043 100644 --- a/orchestra/apps/accounts/actions.py +++ b/orchestra/apps/accounts/actions.py @@ -1,8 +1,15 @@ from django.contrib import messages -from django.core.urlresolvers import reverse -from django.db import transaction +from django.contrib.admin import helpers +from django.contrib.admin.utils import NestedObjects, quote, model_ngettext +from django.contrib.auth import get_permission_codename +from django.core.urlresolvers import reverse, NoReverseMatch +from django.db import router from django.shortcuts import redirect, render +from django.template.response import TemplateResponse from django.utils import timezone +from django.utils.encoding import force_text +from django.utils.html import format_html +from django.utils.text import capfirst from django.utils.translation import ungettext, ugettext_lazy as _ from orchestra.admin.decorators import action_with_confirmation @@ -11,7 +18,6 @@ from orchestra.core import services from . import settings -@transaction.atomic @action_with_confirmation() def disable(modeladmin, request, queryset): num = 0 @@ -43,12 +49,13 @@ def service_report(modeladmin, request, queryset): # TODO resources accounts = [] fields = [] + registered_services = services.get() # First we get related manager names to fire a prefetch related - for name, field in queryset.model._meta._name_map.items(): - model = field[0].model - if model in services.get() and model != queryset.model: + for name, field in queryset.model._meta.fields_map.items(): + model = field.related_model + if model in registered_services and model != queryset.model: fields.append((model, name)) - sorted(fields, key=lambda i: i[0]._meta.verbose_name_plural.lower()) + sorted(fields, key=lambda f: f[0]._meta.verbose_name_plural.lower()) fields = [field for model, field in fields] for account in queryset.prefetch_related(*fields): @@ -66,4 +73,112 @@ def service_report(modeladmin, request, queryset): def delete_related_services(modeladmin, request, queryset): - pass + opts = modeladmin.model._meta + app_label = opts.app_label + + using = router.db_for_write(modeladmin.model) + collector = NestedObjects(using=using) + collector.collect(queryset) + registered_services = services.get() + related_services = [] + to_delete = [] + + user = request.user + admin_site = modeladmin.admin_site + + def format(obj): + has_admin = obj.__class__ in admin_site._registry + opts = obj._meta + no_edit_link = '%s: %s' % (capfirst(opts.verbose_name), force_text(obj)) + + if has_admin: + try: + admin_url = reverse('admin:%s_%s_change' % (opts.app_label, opts.model_name), + None, (quote(obj._get_pk_val()),) + ) + except NoReverseMatch: + # Change url doesn't exist -- don't display link to edit + return no_edit_link + + p = '%s.%s' % (opts.app_label, get_permission_codename('delete', opts)) + if not user.has_perm(p): + perms_needed.add(opts.verbose_name) + # Display a link to the admin page. + return format_html('{}: {}', capfirst(opts.verbose_name), admin_url, obj) + else: + # Don't display link to edit, because it either has no + # admin or is edited inline. + return no_edit_link + + def format_nested(objs, result): + if isinstance(objs, list): + current = [] + for obj in objs: + format_nested(obj, current) + result.append(current) + else: + result.append(format(objs)) + + for account in collector.nested(): + if isinstance(account, list): + current = [] + is_service = False + for service in account: + if type(service) in registered_services: + if service == main_systemuser: + continue + current.append(format(service)) + to_delete.append(service) + is_service = True + elif is_service and isinstance(service, list): + nested = [] + format_nested(service, nested) + current.append(nested) + is_service = False + else: + is_service = False + related_services.append(current) + elif isinstance(account, modeladmin.model): + # Prevent the deletion of the main system user, which will delete the account + main_systemuser = account.main_systemuser + related_services.append(format(account)) + + # The user has already confirmed the deletion. + # Do the deletion and return a None to display the change list view again. + if request.POST.get('post'): + n = queryset.count() + if n: + for obj in to_delete: + obj_display = force_text(obj) + modeladmin.log_deletion(request, obj, obj_display) + # TODO This probably will fail in certain conditions, just capture exception + obj.delete() + modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % { + "count": n, "items": model_ngettext(modeladmin.opts, n) + }, messages.SUCCESS) + # Return None to display the change list page again. + return None + + if len(queryset) == 1: + objects_name = force_text(opts.verbose_name) + else: + objects_name = force_text(opts.verbose_name_plural) + + context = dict( + modeladmin.admin_site.each_context(request), + title=_("Are you sure?"), + objects_name=objects_name, + deletable_objects=[related_services], + model_count=dict(collector.model_count).items(), + queryset=queryset, + opts=opts, + action_checkbox_name=helpers.ACTION_CHECKBOX_NAME, + ) + request.current_app = modeladmin.admin_site.name + # Display the confirmation page + return TemplateResponse(request, modeladmin.delete_selected_confirmation_template or [ + "admin/%s/%s/delete_selected_confirmation.html" % (app_label, opts.model_name), + "admin/%s/delete_selected_confirmation.html" % app_label, + "admin/delete_selected_confirmation.html" + ], context) +delete_related_services.short_description = _("Delete related services") diff --git a/orchestra/apps/accounts/admin.py b/orchestra/apps/accounts/admin.py index ce5d13a8..da5b87bd 100644 --- a/orchestra/apps/accounts/admin.py +++ b/orchestra/apps/accounts/admin.py @@ -19,7 +19,7 @@ from orchestra.core import services, accounts from orchestra.forms import UserChangeForm from . import settings -from .actions import disable, list_contacts, service_report +from .actions import disable, list_contacts, service_report, delete_related_services from .filters import HasMainUserListFilter from .forms import AccountCreationForm from .models import Account @@ -62,7 +62,7 @@ class AccountAdmin(ChangePasswordAdminMixin, auth.UserAdmin, ExtendedModelAdmin) filter_horizontal = () change_readonly_fields = ('username', 'main_systemuser_link') change_form_template = 'admin/accounts/account/change_form.html' - actions = [disable, list_contacts, service_report, SendEmail()] + actions = [disable, list_contacts, service_report, SendEmail(), delete_related_services] change_view_actions = [disable, service_report] list_select_related = ('billcontact',) ordering = () diff --git a/orchestra/apps/accounts/forms.py b/orchestra/apps/accounts/forms.py index faeb842f..104af327 100644 --- a/orchestra/apps/accounts/forms.py +++ b/orchestra/apps/accounts/forms.py @@ -1,6 +1,7 @@ from collections import OrderedDict from django import forms +from django.core.exceptions import ValidationError from django.db.models.loading import get_model from django.utils.translation import ugettext_lazy as _ @@ -47,7 +48,7 @@ def create_account_creation_form(): if model.objects.filter(**kwargs).exists(): verbose_name = model._meta.verbose_name field_name = 'create_%s' % model._meta.model_name - errors[field] = ValidationError( + errors[field_name] = ValidationError( _("A %(type)s with this name already exists."), params={'type': verbose_name}) if errors: diff --git a/orchestra/apps/accounts/models.py b/orchestra/apps/accounts/models.py index e68aaa2e..40b8fc94 100644 --- a/orchestra/apps/accounts/models.py +++ b/orchestra/apps/accounts/models.py @@ -1,5 +1,4 @@ from django.contrib.auth import models as auth -from django.conf import settings as djsettings from django.core import validators from django.db import models from django.db.models.loading import get_model diff --git a/orchestra/apps/bills/actions.py b/orchestra/apps/bills/actions.py index ffd3c083..6edbfa81 100644 --- a/orchestra/apps/bills/actions.py +++ b/orchestra/apps/bills/actions.py @@ -3,6 +3,7 @@ from io import StringIO from django.contrib import messages from django.contrib.admin import helpers +from django.core.exceptions import ValidationError from django.core.urlresolvers import reverse from django.db import transaction from django.http import HttpResponse diff --git a/orchestra/apps/domains/serializers.py b/orchestra/apps/domains/serializers.py index 1efaf6a1..8e3a0b63 100644 --- a/orchestra/apps/domains/serializers.py +++ b/orchestra/apps/domains/serializers.py @@ -1,4 +1,5 @@ from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers from orchestra.api.serializers import HyperlinkedModelSerializer diff --git a/orchestra/apps/orchestration/admin.py b/orchestra/apps/orchestration/admin.py index b2748d5d..5e95ac57 100644 --- a/orchestra/apps/orchestration/admin.py +++ b/orchestra/apps/orchestration/admin.py @@ -29,6 +29,7 @@ class RouteAdmin(admin.ModelAdmin): ] list_editable = ['host', 'match', 'is_active'] list_filter = ['host', 'is_active', 'backend'] + ordering = ('backend',) BACKEND_HELP_TEXT = { backend: "This backend operates over '%s'" % ServiceBackend.get_backend(backend).model diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index 13ebb8c9..e82b8507 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -161,6 +161,7 @@ class Order(models.Model): elif orders: order = orders.get() order.cancel(commit=commit) + logger.info("CANCELLED order id: {id}".format(id=order.id)) updates.append((order, 'cancelled')) return updates @@ -284,8 +285,6 @@ def cancel_orders(sender, **kwargs): # Account delete will delete all related orders, no need to maintain order consistency # if isinstance(instance, Order.account.field.rel.to): # return - if sender is Order.account.field.rel.to: - return print('delete', sender, instance, instance.pk) if type(instance) in services: for order in Order.objects.by_object(instance).active(): @@ -299,10 +298,10 @@ def cancel_orders(sender, **kwargs): # return print('related', type(related), related, related.pk) # try: -# type(related).objects.get(pk=related.pk) + type(related).objects.get(pk=related.pk) # except related.DoesNotExist: # print('not exists', type(related), related, related.pk) - Order.update_orders(related) + print([(str(a).encode('utf8'), b) for a, b in Order.update_orders(related)]) @receiver(post_save, dispatch_uid="orders.update_orders") def update_orders(sender, **kwargs): diff --git a/orchestra/apps/payments/models.py b/orchestra/apps/payments/models.py index e803e147..aefeec43 100644 --- a/orchestra/apps/payments/models.py +++ b/orchestra/apps/payments/models.py @@ -124,7 +124,7 @@ class Transaction(models.Model): if amount >= self.bill.total: raise ValidationError(_("New transactions can not be allocated for this bill.")) - def check_state(*args): + def check_state(self, *args): if self.state not in args: raise TypeError("Transaction not in %s" % ' or '.join(args)) @@ -176,7 +176,7 @@ class TransactionProcess(models.Model): def __str__(self): return '#%i' % self.id - def check_state(*args): + def check_state(self, *args): if self.state not in args: raise TypeError("Transaction process not in %s" % ' or '.join(args)) diff --git a/orchestra/apps/resources/methods.py b/orchestra/apps/resources/methods.py index bc275089..d11d07a8 100644 --- a/orchestra/apps/resources/methods.py +++ b/orchestra/apps/resources/methods.py @@ -25,7 +25,7 @@ class Last(DataMethod): def filter(self, dataset): try: return dataset.order_by('object_id', '-id').distinct('object_id') - except MonitorData.DoesNotExist: + except dataset.model.DoesNotExist: return dataset.none() def compute_usage(self, dataset): diff --git a/orchestra/apps/saas/admin.py b/orchestra/apps/saas/admin.py index c125f730..9fa0b8c9 100644 --- a/orchestra/apps/saas/admin.py +++ b/orchestra/apps/saas/admin.py @@ -1,6 +1,5 @@ from django.contrib import admin -from django.core.urlresolvers import reverse -from django.utils.translation import ugettext, ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ExtendedModelAdmin, ChangePasswordAdminMixin from orchestra.apps.accounts.admin import AccountAdminMixin diff --git a/orchestra/apps/saas/backends/__init__.py b/orchestra/apps/saas/backends/__init__.py index 0158ce08..6fecde50 100644 --- a/orchestra/apps/saas/backends/__init__.py +++ b/orchestra/apps/saas/backends/__init__.py @@ -1,5 +1,4 @@ import pkgutil -import textwrap class SaaSServiceMixin(object): diff --git a/orchestra/apps/saas/backends/phplist.py b/orchestra/apps/saas/backends/phplist.py index 71b233cc..8834e86f 100644 --- a/orchestra/apps/saas/backends/phplist.py +++ b/orchestra/apps/saas/backends/phplist.py @@ -1,4 +1,3 @@ -import json import re import requests @@ -6,8 +5,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.apps.orchestration import ServiceController -from .. import settings - class PhpListSaaSBackend(ServiceController): verbose_name = _("phpList SaaS") @@ -16,7 +13,6 @@ class PhpListSaaSBackend(ServiceController): block = True def _save(self, saas, server): - base_domain = settings.SAAS_PHPLIST_BASE_DOMAIN admin_link = 'http://%s/admin/' % saas.get_site_domain() admin_content = requests.get(admin_link).content if admin_content.startswith('Cannot connect to Database'): diff --git a/orchestra/apps/saas/services/bscw.py b/orchestra/apps/saas/services/bscw.py index 6ced9863..2bbc29be 100644 --- a/orchestra/apps/saas/services/bscw.py +++ b/orchestra/apps/saas/services/bscw.py @@ -1,5 +1,4 @@ from django import forms -from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ from rest_framework import serializers diff --git a/orchestra/apps/saas/services/phplist.py b/orchestra/apps/saas/services/phplist.py index 10ec6239..78ca7ff2 100644 --- a/orchestra/apps/saas/services/phplist.py +++ b/orchestra/apps/saas/services/phplist.py @@ -1,12 +1,11 @@ from django import forms +from django.core.exceptions import ValidationError from django.core.urlresolvers import reverse from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ -from rest_framework import serializers from orchestra.apps.databases.models import Database, DatabaseUser from orchestra.forms import widgets -from orchestra.plugins.forms import PluginDataForm from .. import settings from .options import SoftwareService, SoftwareServiceForm diff --git a/orchestra/apps/services/admin.py b/orchestra/apps/services/admin.py index 9b2eeb6c..0533da8e 100644 --- a/orchestra/apps/services/admin.py +++ b/orchestra/apps/services/admin.py @@ -8,7 +8,6 @@ from django.utils.translation import ugettext_lazy as _ from orchestra.admin import ChangeViewActionsMixin from orchestra.admin.filters import UsedContentTypeFilter -from orchestra.apps.accounts.admin import AccountAdminMixin from orchestra.core import services from .actions import update_orders, view_help, clone diff --git a/orchestra/apps/services/handlers.py b/orchestra/apps/services/handlers.py index a329e087..67ae0d41 100644 --- a/orchestra/apps/services/handlers.py +++ b/orchestra/apps/services/handlers.py @@ -116,6 +116,8 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): instance._meta.model_name: instance, 'instance': instance, 'math': math, + 'logsteps': lambda n, size=1: \ + round(n/(size*10**int(math.log10(max(n, 1)))))*size*10**int(math.log10(max(n, 1))), 'log10': math.log10, 'Decimal': decimal.Decimal, } diff --git a/orchestra/apps/services/models.py b/orchestra/apps/services/models.py index ade905aa..84c65edf 100644 --- a/orchestra/apps/services/models.py +++ b/orchestra/apps/services/models.py @@ -1,9 +1,7 @@ import decimal from django.contrib.contenttypes.models import ContentType -from django.core.validators import ValidationError from django.db import models -from django.db.models import Q from django.db.models.loading import get_model from django.utils.functional import cached_property from django.utils.module_loading import autodiscover_modules @@ -11,8 +9,6 @@ from django.utils.translation import string_concat, ugettext_lazy as _ from orchestra.core import caches, validators from orchestra.core.translations import ModelTranslation -from orchestra.core.validators import validate_name -from orchestra.models import queryset from orchestra.utils.python import import_class from . import settings diff --git a/orchestra/apps/services/settings.py b/orchestra/apps/services/settings.py index db6dc00e..66323aef 100644 --- a/orchestra/apps/services/settings.py +++ b/orchestra/apps/services/settings.py @@ -1,5 +1,3 @@ -from datetime import timedelta - from django.conf import settings from django.utils.translation import ugettext_lazy as _ diff --git a/orchestra/apps/services/tests/functional_tests/__init__.py b/orchestra/apps/services/tests/functional_tests/__init__.py index 2196813d..e69de29b 100644 --- a/orchestra/apps/services/tests/functional_tests/__init__.py +++ b/orchestra/apps/services/tests/functional_tests/__init__.py @@ -1,9 +0,0 @@ -from orchestra.apps.accounts.models import Account -from orchestra.utils.tests import BaseTestCase, random_ascii - - -# TODO remove this shit -class BaseBillingTest(BaseTestCase): - pass - -# TODO web disk diff --git a/orchestra/apps/systemusers/backends.py b/orchestra/apps/systemusers/backends.py index 40af444a..c17c69e4 100644 --- a/orchestra/apps/systemusers/backends.py +++ b/orchestra/apps/systemusers/backends.py @@ -16,6 +16,8 @@ class SystemUserBackend(ServiceController): def save(self, user): context = self.get_context(user) + if not context['user']: + return groups = ','.join(self.get_groups(user)) context['groups_arg'] = '--groups %s' % groups if groups else '' # TODO userd add will fail if %(user)s group already exists @@ -37,6 +39,8 @@ class SystemUserBackend(ServiceController): def delete(self, user): context = self.get_context(user) + if not context['user']: + return self.append(textwrap.dedent("""\ { sleep 2 && killall -u %(user)s -s KILL; } & killall -u %(user)s || true diff --git a/orchestra/apps/webapps/types/__init__.py b/orchestra/apps/webapps/types/__init__.py index 8ad6bccd..43f40be6 100644 --- a/orchestra/apps/webapps/types/__init__.py +++ b/orchestra/apps/webapps/types/__init__.py @@ -1,4 +1,5 @@ from django.core.exceptions import ValidationError +from django.utils.translation import ugettext_lazy as _ from orchestra import plugins from orchestra.plugins.forms import PluginDataForm @@ -31,7 +32,7 @@ class AppType(plugins.Plugin): def validate(self): """ Unique name validation """ if self.unique_name: - if not self.instance.pk and Webapp.objects.filter(name=self.instance.name, type=self.instance.type).exists(): + if not self.instance.pk and type(self.instance).objects.filter(name=self.instance.name, type=self.instance.type).exists(): raise ValidationError({ 'name': _("A WordPress blog with this name already exists."), }) diff --git a/orchestra/apps/webapps/types/php.py b/orchestra/apps/webapps/types/php.py index d7526c7a..5d046624 100644 --- a/orchestra/apps/webapps/types/php.py +++ b/orchestra/apps/webapps/types/php.py @@ -118,6 +118,7 @@ class PHPApp(AppType): wrapper_path = os.path.normpath(self.FCGID_WRAPPER_PATH % context) return ('fcgid', self.instance.get_path(), wrapper_path) else: + php_version = self.get_php_version() raise ValueError("Unknown directive for php version '%s'" % php_version) def get_php_version(self): diff --git a/orchestra/apps/websites/backends/apache.py b/orchestra/apps/websites/backends/apache.py index a5f42951..46e7b1b2 100644 --- a/orchestra/apps/websites/backends/apache.py +++ b/orchestra/apps/websites/backends/apache.py @@ -208,7 +208,7 @@ class Apache2Backend(ServiceController): proxies = [] for proxy in directives.get('proxy', []): location, target = proxy.split() - location = normurlpath(source) + location = normurlpath(location) proxy = textwrap.dedent("""\ ProxyPass {location}/ {target} ProxyPassReverse {location}/ {target}""".format( diff --git a/orchestra/apps/websites/forms.py b/orchestra/apps/websites/forms.py index b557d664..afbf7110 100644 --- a/orchestra/apps/websites/forms.py +++ b/orchestra/apps/websites/forms.py @@ -1,6 +1,7 @@ from django import forms from django.core.exceptions import ValidationError from django.utils.encoding import force_text +from django.utils.translation import ugettext_lazy as _ from .validators import validate_domain_protocol diff --git a/orchestra/apps/websites/serializers.py b/orchestra/apps/websites/serializers.py index 64e337d7..c98dbf68 100644 --- a/orchestra/apps/websites/serializers.py +++ b/orchestra/apps/websites/serializers.py @@ -1,5 +1,4 @@ from django.core.exceptions import ValidationError -from django.db.models import Q from django.shortcuts import get_object_or_404 from rest_framework import serializers diff --git a/orchestra/bin/orchestra-admin b/orchestra/bin/orchestra-admin index dd0076c4..74be3ae2 100755 --- a/orchestra/bin/orchestra-admin +++ b/orchestra/bin/orchestra-admin @@ -173,6 +173,7 @@ function install_requirements () { xvfbwrapper \ freezegun \ coverage \ + flake8 \ orchestra-orm==dev \ django-debug-toolbar==1.3.0 \ https://github.com/django-nose/django-nose/archive/master.zip \ diff --git a/orchestra/conf/base_settings.py b/orchestra/conf/base_settings.py index 4a411de8..a58cee8b 100644 --- a/orchestra/conf/base_settings.py +++ b/orchestra/conf/base_settings.py @@ -48,11 +48,10 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'orchestra.core.caches.RequestCacheMiddleware', - # also handles transations, ATOMIC REQUESTS does not wrap middlewares + # also handles transations, ATOMIC_REQUESTS does not wrap middlewares 'orchestra.apps.orchestration.middlewares.OperationsMiddleware', # Uncomment the next line for simple clickjacking protection: # 'django.middleware.clickjacking.XFrameOptionsMiddleware', - ) @@ -69,7 +68,6 @@ TEMPLATE_CONTEXT_PROCESSORS =( INSTALLED_APPS = ( # django-orchestra apps - 'orchestra', 'orchestra.apps.accounts', 'orchestra.apps.contacts', diff --git a/orchestra/core/__init__.py b/orchestra/core/__init__.py index 3e9c6172..7772dad5 100644 --- a/orchestra/core/__init__.py +++ b/orchestra/core/__init__.py @@ -23,7 +23,7 @@ class Register(object): def get(self, *args): if args: - return self._registry[arg[0]] + return self._registry[args[0]] return self._registry diff --git a/orchestra/core/middlewares.py b/orchestra/core/middlewares.py deleted file mode 100644 index 7e96a46a..00000000 --- a/orchestra/core/middlewares.py +++ /dev/null @@ -1,47 +0,0 @@ -from django.db import connection, transaction - - -class TransactionMiddleware(object): - """ - Transaction middleware. If this is enabled, each view function will be run - with commit_on_response activated - that way a save() doesn't do a direct - commit, the commit is done when a successful response is created. If an - exception happens, the database is rolled back. - """ - pass -# def process_request(self, request): -# """Enters transaction management""" -# transaction.enter_transaction_management() -# -# def process_exception(self, request, exception): -# """Rolls back the database and leaves transaction management""" -# if transaction.is_dirty(): -# # This rollback might fail because of network failure for example. -# # If rollback isn't possible it is impossible to clean the -# # connection's state. So leave the connection in dirty state and -# # let request_finished signal deal with cleaning the connection. -# transaction.rollback() -# transaction.leave_transaction_management() -# -# def process_response(self, request, response): -# """Commits and leaves transaction management.""" -# if not transaction.get_autocommit(): -# if transaction.is_dirty(): -# # Note: it is possible that the commit fails. If the reason is -# # closed connection or some similar reason, then there is -# # little hope to proceed nicely. However, in some cases ( -# # deferred foreign key checks for exampl) it is still possible -# # to rollback(). -# try: -# transaction.commit() -# except Exception: -# # If the rollback fails, the transaction state will be -# # messed up. It doesn't matter, the connection will be set -# # to clean state after the request finishes. And, we can't -# # clean the state here properly even if we wanted to, the -# # connection is in transaction but we can't rollback... -# transaction.rollback() -# transaction.leave_transaction_management() -# raise -# transaction.leave_transaction_management() -# return response diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index 080bd551..2b473d90 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -1,7 +1,6 @@ import re import crack -import localflavor import phonenumbers from django.core import validators @@ -47,7 +46,7 @@ def validate_ipv6_address(value): def validate_ip_address(value): msg = _("%s is not a valid IP address") % value try: - ip = IP(value) + IP(value) except: raise ValidationError(msg) diff --git a/orchestra/forms/options.py b/orchestra/forms/options.py index feb880e2..d82c13fc 100644 --- a/orchestra/forms/options.py +++ b/orchestra/forms/options.py @@ -1,6 +1,6 @@ from django import forms from django.contrib.auth import forms as auth_forms -from django.utils.translation import ugettext, ugettext_lazy as _ +from django.utils.translation import ugettext_lazy as _ from orchestra.utils.python import random_ascii diff --git a/orchestra/management/commands/staticcheck.py b/orchestra/management/commands/staticcheck.py index 83e4791e..5f75c37b 100644 --- a/orchestra/management/commands/staticcheck.py +++ b/orchestra/management/commands/staticcheck.py @@ -1,102 +1,12 @@ -# Adapted from http://djangosnippets.org/snippets/1762/ - -import ast -import os -import sys - from django.core.management.base import BaseCommand -from pyflakes import checker, messages -from orchestra.utils.paths import get_orchestra_dir - - -# BlackHole, PySyntaxError and checking based on -# https://github.com/patrys/gedit-pyflakes-plugin.git -class BlackHole(object): - write = flush = lambda *args, **kwargs: None - - def __enter__(self): - self.stderr, sys.stderr = sys.stderr, self - - def __exit__(self, *args, **kwargs): - sys.stderr = self.stderr - - -class PySyntaxError(messages.Message): - message = 'syntax error in line %d: %s' - - def __init__(self, filename, e): - super(PySyntaxError, self).__init__(filename, e) - self.message_args = (e.offset, e.text) - - -def check(codeString, filename): - """ - Check the Python source given by C{codeString} for flakes. - - @param codeString: The Python source to check. - @type codeString: C{str} - - @param filename: The name of the file the source came from, used to report errors. - @type filename: C{str} - - @return: The number of warnings emitted. - @rtype: C{int} - """ - try: - with BlackHole(): - tree = ast.parse(codeString, filename) - except SyntaxError as e: - return [PySyntaxError(filename, e)] - else: - # Okay, it's syntactically valid. Now parse it into an ast and check it - w = checker.Checker(tree, filename) - - lines = codeString.split('\n') - # honour pyflakes: ignore comments - messages = [message for message in w.messages - if lines[message.lineno-1].find('pyflakes:ignore') < 0] - messages.sort(lambda a, b: cmp(a.lineno, b.lineno)) - return messages - - -def checkPath(filename): - """ - Check the given path, printing out any warnings detected. - @return: the number of warnings printed - """ - try: - return check(file(filename, 'U').read() + '\n', filename) - except IOError as msg: - return ["%s: %s" % (filename, msg.args[1])] - except TypeError: - pass - - -def checkPaths(filenames): - warnings = [] - for arg in filenames: - if os.path.isdir(arg): - for dirpath, dirnames, filenames in os.walk(arg): - for filename in filenames: - if filename.endswith('.py'): - warnings.extend(checkPath(os.path.join(dirpath, filename))) - else: - warnings.extend(checkPath(arg)) - return warnings -#### pyflakes.scripts.pyflakes ends. +from orchestra.utils.paths import get_orchestra_dir, get_site_dir +from orchestra.utils.system import run class Command(BaseCommand): - help = "Run pyflakes syntax checks." - args = '[filename [filename [...]]]' + help = "Run flake8 syntax checks." def handle(self, *filenames, **options): - if not filenames: - filenames = [get_orchestra_dir(), '.'] - warnings = checkPaths(filenames) - for warning in warnings: - print(warning) - if warnings: - print('Total warnings: %d' % len(warnings)) - raise SystemExit(1) + flake = run('flake8 {%s,%s} | grep -v "W293\|E501"' % (get_orchestra_dir(), get_site_dir())) + print(flake.stdout) diff --git a/orchestra/models/fields.py b/orchestra/models/fields.py index 0a00d4fb..9a6deab7 100644 --- a/orchestra/models/fields.py +++ b/orchestra/models/fields.py @@ -54,7 +54,7 @@ class MultiSelectField(models.CharField, metaclass=models.SubfieldBase): def get_choices_selected(self, arr_choices=''): if not arr_choices: return False - return [ value for value,__ in arr_choices ] + return [value for value, __ in arr_choices] class NullableCharField(models.CharField): diff --git a/orchestra/plugins/admin.py b/orchestra/plugins/admin.py index db1dff1a..e4b80fb1 100644 --- a/orchestra/plugins/admin.py +++ b/orchestra/plugins/admin.py @@ -4,7 +4,6 @@ from django.shortcuts import render, redirect from django.utils.translation import ugettext_lazy as _ from orchestra.admin.utils import wrap_admin_view -from orchestra.utils.functional import cached class SelectPluginAdminMixin(object): diff --git a/orchestra/utils/system.py b/orchestra/utils/system.py index 994f8c2d..c4b89030 100644 --- a/orchestra/utils/system.py +++ b/orchestra/utils/system.py @@ -63,7 +63,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', # Async reading of stdout and sterr while True: stdout = '' - sdterr = '' + stderr = '' # Get complete unicode chunks select.select([p.stdout, p.stderr], [], []) @@ -71,7 +71,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', stderrPiece = read_async(p.stderr) stdout += (stdoutPiece or b'').decode('utf8') - sdterr += (stderrPiece or b'').decode('utf8') + stderr += (stderrPiece or b'').decode('utf8') if display and stdout: sys.stdout.write(stdout) @@ -79,7 +79,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', sys.stderr.write(stderr) state = _Attribute(stdout) - state.stderr = sdterr + state.stderr = stderr state.return_code = p.poll() yield state diff --git a/orchestra/utils/tests.py b/orchestra/utils/tests.py index 89f78d40..87489965 100644 --- a/orchestra/utils/tests.py +++ b/orchestra/utils/tests.py @@ -16,7 +16,6 @@ from orchestra.apps.accounts.models import Account from .python import random_ascii - class AppDependencyMixin(object): DEPENDENCIES = () diff --git a/scripts/container/create.sh b/scripts/container/create.sh index 8d3718de..abc96f56 100755 --- a/scripts/container/create.sh +++ b/scripts/container/create.sh @@ -3,6 +3,7 @@ # This is a helper script for creating a basic LXC container with some convenient packages # ./create.sh [container_name] + set -u NAME=${1:-orchestra} @@ -35,7 +36,7 @@ chroot $CONTAINER locale-gen chroot $CONTAINER apt-get install -y --force-yes \ - nano git screen sudo iputils-ping python2.7 python-pip wget curl dnsutils rsyslog + nano git screen sudo iputils-ping python3 python3-pip wget curl dnsutils rsyslog chroot $CONTAINER apt-get clean diff --git a/scripts/container/deploy.sh b/scripts/container/deploy.sh index b42e12f3..a698cb40 100755 --- a/scripts/container/deploy.sh +++ b/scripts/container/deploy.sh @@ -41,13 +41,13 @@ chown $USER.$USER $HOME run adduser $USER sudo -CURRENT_VERSION=$(python -c "from orchestra import get_version; print get_version();" 2> /dev/null || false) +CURRENT_VERSION=$(python3 -c "from orchestra import get_version; print get_version();" 2> /dev/null || false) if [[ ! $CURRENT_VERSION ]]; then # First Orchestra installation - run "apt-get -y install git python-pip" + run "apt-get -y install git python3-pip" surun "git clone https://github.com/glic3rinu/django-orchestra.git ~/django-orchestra" - echo $HOME/django-orchestra/ | sudo tee /usr/local/lib/python2.7/dist-packages/orchestra.pth + echo $HOME/django-orchestra/ | sudo tee /usr/local/lib/python3*/dist-packages/orchestra.pth run "cp $HOME/django-orchestra/orchestra/bin/orchestra-admin /usr/local/bin/" fi @@ -71,33 +71,33 @@ if [[ ! $(sudo su postgres -c "psql -lqt" | awk {'print $1'} | grep '^orchestra$ /etc/postgresql/${POSTGRES_VERSION}/main/postgresql.conf run "service postgresql restart" - run "python $MANAGE setuppostgres --db_name orchestra --db_user orchestra --db_password orchestra" + run "python3 $MANAGE setuppostgres --db_name orchestra --db_user orchestra --db_password orchestra" # Create database permissions are needed for running tests sudo su postgres -c 'psql -c "ALTER USER orchestra CREATEDB;"' fi if [[ $CURRENT_VERSION ]]; then # Per version upgrade specific operations - run "python $MANAGE postupgradeorchestra --no-restart --from $CURRENT_VERSION" + run "python3 $MANAGE postupgradeorchestra --no-restart --from $CURRENT_VERSION" else - run "python $MANAGE syncdb --noinput" - run "python $MANAGE migrate --noinput" + run "python3 $MANAGE syncdb --noinput" + run "python3 $MANAGE migrate --noinput" fi sudo python $MANAGE setupcelery --username $USER --processes 2 # Install and configure Nginx web server surun "mkdir $BASE_DIR/static" -surun "python $MANAGE collectstatic --noinput" -run "apt-get install -y nginx uwsgi uwsgi-plugin-python" -run "python $MANAGE setupnginx" +surun "python3 $MANAGE collectstatic --noinput" +run "apt-get install -y nginx uwsgi uwsgi-plugin-python3" +run "python3 $MANAGE setupnginx" run "service nginx start" # Apply changes -run "python $MANAGE restartservices" +run "python3 $MANAGE restartservices" # Create a orchestra user -cat <<- EOF | python $MANAGE shell +cat <<- EOF | python3 $MANAGE shell from orchestra.apps.accounts.models import Account if not Account.objects.filter(username="$USER").exists(): print 'Creating orchestra superuser'