From b29c554878ca1ebba2ef3ea68280432258058eed Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Wed, 1 Apr 2015 15:49:21 +0000 Subject: [PATCH] Added orchestrate management command --- ROADMAP.md | 84 ++++++++++--------- TODO.md | 23 +++++ orchestra/apps/contacts/models.py | 2 +- orchestra/apps/domains/models.py | 2 +- .../apps/orchestration/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/orchestrate.py | 56 +++++++++++++ orchestra/apps/orchestration/manager.py | 78 ++++++++++++++++- orchestra/apps/orchestration/methods.py | 2 +- orchestra/apps/orchestration/middlewares.py | 68 ++------------- orchestra/apps/orchestration/models.py | 5 +- orchestra/apps/orders/actions.py | 1 + orchestra/apps/orders/admin.py | 14 ++-- orchestra/apps/orders/models.py | 4 +- orchestra/apps/resources/admin.py | 2 + orchestra/apps/resources/methods.py | 5 +- orchestra/apps/resources/models.py | 11 ++- orchestra/apps/resources/tasks.py | 2 +- orchestra/apps/resources/validators.py | 2 +- orchestra/apps/saas/backends/gitlab.py | 6 +- orchestra/apps/services/admin.py | 32 ++++++- orchestra/apps/services/handlers.py | 12 +-- orchestra/apps/services/models.py | 4 +- .../admin/services/service/change_form.html | 2 +- .../admin/services/service/help.html | 1 - .../tests/functional_tests/test_mailbox.py | 3 +- .../tests/functional_tests/test_traffic.py | 2 +- orchestra/apps/webapps/types/php.py | 6 +- orchestra/core/validators.py | 4 +- orchestra/management/commands/staticcheck.py | 4 +- .../admin/orchestra/generic_confirmation.html | 3 + orchestra/utils/system.py | 4 +- 32 files changed, 290 insertions(+), 154 deletions(-) create mode 100644 orchestra/apps/orchestration/management/__init__.py create mode 100644 orchestra/apps/orchestration/management/commands/__init__.py create mode 100644 orchestra/apps/orchestration/management/commands/orchestrate.py diff --git a/ROADMAP.md b/ROADMAP.md index bb54c273..79696313 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,11 +2,54 @@ Note `*` _for sustancial progress_ -### 1.0a1 Milestone (first alpha release on ~~Oct~~ Nov '14) + +### 2.0 Milestone (unscheduled) + +1. [ ] Integration with third-party service providers, e.g. Gandi +2. [ ] Scheduling of service cancellations and deactivations +1. [ ] Object-level permission system +2. [ ] REST API functionality for superusers +3. [ ] Responsive user interface, based on a JS framework. +4. [ ] Full development documentation +5. [ ] [Ansible](http://www.ansible.com/home) orchestration method, which synchronizes the whole service config everytime instead of incremental changes. + + +### 1.0 Milestone (first stable release on Sep '15) + +1. [ ] Stabilize data model, internal APIs and REST API +3. [ ] Spanish and Catalan translations +1. [ ] Complete documentation for developers + + +### 1.0b1 Milestone (first beta release on ~~Dec '14~~ Jun '15) + +1. [x] Resource allocation and monitoring +1. [x] Order tracking +2. [x] Service definition framework, service plans and pricing +3. [ ] *Billing + 3. [x] Invoice + 3. [x] Membership fee + 3. [ ] *Amendment invoice + 3. [ ] *Amendment fee + 3. [x] Pro Forma + 3. [ ] *Advanced bill handling (move lines, undo billing, ...) +1. [x] Payment methods + 1. [x] SEPA Direct Debit + 2. [x] SEPA Credit Transfer +2. [ ] *Additional services + 2. [ ] *VPS with Proxmox/OpenVZ + 2. [ ] *SaaS (Software as a Service) Redmine/phpList/BSCW/Wordpress/Moodle/Drupal + 2. [ ] *Wordpress/Python webapps + 2. [x] Miscellaneous services +2. [x] Issue tracking system + + + +### 1.0a1 Milestone (first alpha release on ~~Oct '14~~ Apr '15) 1. [x] Automated deployment of the development environment 2. [x] Automated installation and upgrading -2. [ ] Testing framework for running unittests and functional tests with LXC containers +2. ~~[ ] Testing framework for running unittests and functional tests with LXC containers~~ 2. [ ] Continuous integration with Jenkins 2. [x] Admin interface based on django.contrib.admin 3. [x] REST API for users @@ -26,41 +69,4 @@ Note `*` _for sustancial progress_ 1. [ ] Initial documentation -### 1.0b1 Milestone (first beta release on Dec '14) -1. [x] Resource allocation and monitoring -1. [x] Order tracking -2. [x] Service definition framework, service plans and pricing -3. [ ] *Billing - 3. [x] Invoice - 3. [x] Membership fee - 3. [ ] *Amendment invoice - 3. [ ] *Amendment fee - 3. [x] Pro Forma - 3. [ ] *Advanced bill handling (move lines, undo billing, ...) -1. [x] Payment methods - 1. [x] SEPA Direct Debit - 2. [x] SEPA Credit Transfer -2. [ ] *Additional services - 2. [ ] *VPS with Proxmox/OpenVZ - 2. [ ] *SaaS (Software as a Service) Redmine/phpList/BSCW/Wordpress/Moodle/Drupal - 2. [x] Miscellaneous services -2. [x] Issue tracking system - - -### 1.0 Milestone (first stable release on Apr '15) - -1. [ ] Stabilize data model, internal APIs and REST API -3. [ ] Spanish and Catalan translations -1. [ ] Complete documentation for developers - - -### 2.0 Milestone - -1. [ ] Integration with third-party service providers, e.g. Gandi -2. [ ] Scheduling of service cancellations and deactivations -1. [ ] Object-level permission system -2. [ ] REST API functionality for superusers -3. [ ] Responsive user interface, based on a JS framework. -4. [ ] Full documentation -5. [ ] [Ansible](http://www.ansible.com/home) orchestration method, which synchronizes the whole service config everytime instead of incremental changes. diff --git a/TODO.md b/TODO.md index e692f69c..73826208 100644 --- a/TODO.md +++ b/TODO.md @@ -243,6 +243,10 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * line 513: change threshold and one time service metric change should update last value if not billed, only record for recurring invoicing. postpay services should store the last metric for pricing period. * add ini, end dates on bill lines and breakup quanity into size(defaut:1) and metric * threshold for significative metric accountancy on services.handler + * http://orchestra.pangea.org/admin/orders/order/6418/ + * http://orchestra.pangea.org/admin/orders/order/6495/bill_selected_orders/ + * >>> round(float(decimal.Decimal('2.63'))/0.5)*0.5 + * >>> round(float(str(decimal.Decimal('2.99')).split('.')[0]))/1*1 * move normurlpath to orchestra.utils from websites.utils @@ -261,6 +265,12 @@ require_once(‘/etc/moodles/’.$moodle_host.‘config.php’);``` moodle/drupl * Base price: domini propi (all domains) + extra for other domains +* prepend ORCHESTRA_ to orchestra/settings.py + + +* rename backends with generic names to concrete services.. eg VsFTPdTraffic, UNIXSystemUser + + Translation ----------- @@ -290,3 +300,16 @@ xxxxx -- 0 20M 22M 7 200 300 * saas validate_creation generic approach, for all backends. standard output * html code x: × + + +* cleanup backendlogs, monitor data and metricstorage +* create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help + +* uwsgi --max-requests=5000 \ # respawn processes after serving 5000 requests and +celery max-tasks-per-child + +* generate settings.py more like django (installed_apps, middlewares, etc,,,) + +* postupgradeorchestra send signals in order to hook custom stuff + +* make base home for systemusers that ara homed into main account systemuser diff --git a/orchestra/apps/contacts/models.py b/orchestra/apps/contacts/models.py index 7a4d1494..0584b73e 100644 --- a/orchestra/apps/contacts/models.py +++ b/orchestra/apps/contacts/models.py @@ -74,7 +74,7 @@ class Contact(models.Model): elif self.zipcode and self.country: try: validators.validate_zipcode(self.zipcode, self.country) - except ValidationError, error: + except ValidationError as error: errors['zipcode'] = error if errors: raise ValidationError(errors) diff --git a/orchestra/apps/domains/models.py b/orchestra/apps/domains/models.py index dd700997..4dda4944 100644 --- a/orchestra/apps/domains/models.py +++ b/orchestra/apps/domains/models.py @@ -247,7 +247,7 @@ class Record(models.Model): } try: choices[self.type](self.value) - except ValidationError, error: + except ValidationError as error: raise ValidationError({'value': error}) def get_ttl(self): diff --git a/orchestra/apps/orchestration/management/__init__.py b/orchestra/apps/orchestration/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/orchestration/management/commands/__init__.py b/orchestra/apps/orchestration/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orchestra/apps/orchestration/management/commands/orchestrate.py b/orchestra/apps/orchestration/management/commands/orchestrate.py new file mode 100644 index 00000000..bcbcfb13 --- /dev/null +++ b/orchestra/apps/orchestration/management/commands/orchestrate.py @@ -0,0 +1,56 @@ +import sys + +from django.core.management.base import BaseCommand +from django.db.models.loading import get_model +from django.utils.six.moves import input + +from orchestra.apps.orchestration import manager +from orchestra.apps.orchestration.models import BackendOperation as Operation + + +class Command(BaseCommand): + help = 'Runs orchestration backends.' + option_list = BaseCommand.option_list + args = "[app_label] [filter]" + + def handle(self, *args, **options): + model_label = args[0] + model = get_model(*model_label.split('.')) + # TODO options + action = options.get('action', 'save') + interactive = options.get('interactive', True) + kwargs = {} + for comp in args[1:]: + comps = iter(comp.split('=')) + for arg in comps: + kwargs[arg] = next(comps).strip().rstrip(',') + operations = [] + operations = set() + route_cache = {} + for instance in model.objects.filter(**kwargs): + manager.collect(instance, action, operations=operations, route_cache=route_cache) + scripts, block = manager.generate(operations) + servers = [] + # Print scripts + for key, value in scripts.iteritems(): + server, __ = key + backend, operations = value + servers.append(server.name) + sys.stdout.write('# Execute on %s\n' % server.name) + for method, commands in backend.scripts: + sys.stdout.write('\n'.join(commands) + '\n') + if interactive: + context = { + 'servers': ', '.join(servers), + } + msg = ("\n\nAre your sure to execute the previous scripts on %(servers)s (yes/no)? " % context) + confirm = input(msg) + while 1: + if confirm not in ('yes', 'no'): + confirm = input('Please enter either "yes" or "no": ') + continue + if confirm == 'no': + return + break +# manager.execute(scripts, block=block) + diff --git a/orchestra/apps/orchestration/manager.py b/orchestra/apps/orchestration/manager.py index bb53da4e..28a6150c 100644 --- a/orchestra/apps/orchestration/manager.py +++ b/orchestra/apps/orchestration/manager.py @@ -9,8 +9,9 @@ from django.core.mail import mail_admins from orchestra.utils.python import import_class from . import settings +from .backends import ServiceBackend from .helpers import send_report -from .models import BackendLog +from .models import BackendLog, BackendOperation as Operation from .signals import pre_action, post_action @@ -55,8 +56,7 @@ def close_connection(execute): return wrapper -def execute(operations, async=False): - """ generates and executes the operations on the servers """ +def generate(operations): scripts = OrderedDict() cache = {} block = False @@ -86,13 +86,20 @@ def execute(operations, async=False): post_action.send(**kwargs) if backend.block: block = True + for value in scripts.itervalues(): + backend, operations = value + backend.commit() + return scripts, block + + +def execute(scripts, block=False, async=False): + """ executes the operations on the servers """ # Execute scripts on each server threads = [] executions = [] for key, value in scripts.iteritems(): server, __ = key backend, operations = value - backend.commit() execute = as_task(backend.execute) logger.debug('%s is going to be executed on %s' % (backend, server)) if block: @@ -125,3 +132,66 @@ def execute(operations, async=False): mocked_log = BackendLog(state=BackendLog.EXCEPTION) logs.append(mocked_log) return logs + + +def collect(instance, action, **kwargs): + """ collect operations """ + operations = kwargs.get('operations', set()) + route_cache = kwargs.get('route_cache', {}) + for backend_cls in ServiceBackend.get_backends(): + # Check if there exists a related instance to be executed for this backend + instances = [] + if backend_cls.is_main(instance): + instances = [(instance, action)] + else: + candidate = backend_cls.get_related(instance) + if candidate: + if candidate.__class__.__name__ == 'ManyRelatedManager': + if 'pk_set' in kwargs: + # m2m_changed signal + candidates = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) + else: + candidates = candidate.all() + else: + candidates = [candidate] + for candidate in candidates: + # Check if a delete for candidate is in operations + delete_mock = Operation.create(backend_cls, candidate, Operation.DELETE) + if delete_mock not in operations: + # related objects with backend.model trigger save() + instances.append((candidate, Operation.SAVE)) + for selected, iaction in instances: + # Maintain consistent state of operations based on save/delete behaviour + # Prevent creating a deleted selected by deleting existing saves + if iaction == Operation.DELETE: + save_mock = Operation.create(backend_cls, selected, Operation.SAVE) + try: + operations.remove(save_mock) + except KeyError: + pass + else: + update_fields = kwargs.get('update_fields', None) + if update_fields is not None: + # "update_fileds=[]" is a convention for explicitly executing backend + # i.e. account.disable() + if update_fields != []: + execute = False + for field in update_fields: + if field not in backend_cls.ignore_fields: + execute = True + break + if not execute: + continue + operation = Operation.create(backend_cls, selected, iaction) + # Only schedule operations if the router gives servers to execute into + servers = router.get_servers(operation, cache=route_cache) + if servers: + operation.servers = servers + if iaction != Operation.DELETE: + # usually we expect to be using last object state, + # except when we are deleting it + operations.discard(operation) + elif iaction == Operation.DELETE: + operation.preload_context() + operations.add(operation) + return operations diff --git a/orchestra/apps/orchestration/methods.py b/orchestra/apps/orchestration/methods.py index 36f6f9e2..d2ad1606 100644 --- a/orchestra/apps/orchestration/methods.py +++ b/orchestra/apps/orchestration/methods.py @@ -51,7 +51,7 @@ def SSH(backend, log, server, cmds, async=False): key = settings.ORCHESTRATION_SSH_KEY_PATH try: ssh.connect(addr, username='root', key_filename=key, timeout=10) - except socket.error, e: + except socket.error as e: logger.error('%s timed out on %s' % (backend, addr)) log.state = log.TIMEOUT log.stderr = str(e) diff --git a/orchestra/apps/orchestration/middlewares.py b/orchestra/apps/orchestration/middlewares.py index c65561ea..35fe0bca 100644 --- a/orchestra/apps/orchestration/middlewares.py +++ b/orchestra/apps/orchestration/middlewares.py @@ -7,11 +7,9 @@ from django.http.response import HttpResponseServerError from orchestra.utils.python import OrderedSet -from .backends import ServiceBackend +from . import manager from .helpers import message_user -from .manager import router -from .models import BackendLog -from .models import BackendOperation as Operation +from .models import BackendLog, BackendOperation as Operation @receiver(post_save, dispatch_uid='orchestration.post_save_collector') @@ -68,64 +66,10 @@ class OperationsMiddleware(object): request = getattr(cls.thread_locals, 'request', None) if request is None: return - pending_operations = cls.get_pending_operations() - route_cache = cls.get_route_cache() - for backend_cls in ServiceBackend.get_backends(): - # Check if there exists a related instance to be executed for this backend - instances = [] - if backend_cls.is_main(kwargs['instance']): - instances = [(kwargs['instance'], action)] - else: - candidate = backend_cls.get_related(kwargs['instance']) - if candidate: - if candidate.__class__.__name__ == 'ManyRelatedManager': - if 'pk_set' in kwargs: - # m2m_changed signal - candidates = kwargs['model'].objects.filter(pk__in=kwargs['pk_set']) - else: - candidates = candidate.all() - else: - candidates = [candidate] - for candidate in candidates: - # Check if a delete for candidate is in pending_operations - delete_mock = Operation.create(backend_cls, candidate, Operation.DELETE) - if delete_mock not in pending_operations: - # related objects with backend.model trigger save() - instances.append((candidate, Operation.SAVE)) - for instance, iaction in instances: - # Maintain consistent state of pending_operations based on save/delete behaviour - # Prevent creating a deleted instance by deleting existing saves - if iaction == Operation.DELETE: - save_mock = Operation.create(backend_cls, instance, Operation.SAVE) - try: - pending_operations.remove(save_mock) - except KeyError: - pass - else: - update_fields = kwargs.get('update_fields', None) - if update_fields is not None: - # "update_fileds=[]" is a convention for explicitly executing backend - # i.e. account.disable() - if update_fields != []: - execute = False - for field in update_fields: - if field not in backend_cls.ignore_fields: - execute = True - break - if not execute: - continue - operation = Operation.create(backend_cls, instance, iaction) - # Only schedule operations if the router gives servers to execute into - servers = router.get_servers(operation, cache=route_cache) - if servers: - operation.servers = servers - if iaction != Operation.DELETE: - # usually we expect to be using last object state, - # except when we are deleting it - pending_operations.discard(operation) - elif iaction == Operation.DELETE: - operation.preload_context() - pending_operations.add(operation) + kwargs['operations'] = cls.get_pending_operations() + kwargs['route_cache'] = cls.get_route_cache() + instance = kwargs.pop('instance') + manager.collect(instance, action, **kwargs) def process_request(self, request): """ Store request on a thread local variable """ diff --git a/orchestra/apps/orchestration/models.py b/orchestra/apps/orchestration/models.py index dfe3ea0f..4385a2b6 100644 --- a/orchestra/apps/orchestration/models.py +++ b/orchestra/apps/orchestration/models.py @@ -141,7 +141,8 @@ class BackendOperation(models.Model): @classmethod def execute(cls, operations, async=False): from . import manager - return manager.execute(operations, async=async) + scripts, block = manager.generate(operations) + return manager.execute(scripts, block=block, async=async) @classmethod def execute_action(cls, instance, action): @@ -224,7 +225,7 @@ class Route(models.Model): return try: bool(self.matches(obj)) - except Exception, exception: + except Exception as exception: name = type(exception).__name__ message = exception.message raise ValidationError(': '.join((name, message))) diff --git a/orchestra/apps/orders/actions.py b/orchestra/apps/orders/actions.py index bec49b99..13ee2580 100644 --- a/orchestra/apps/orders/actions.py +++ b/orchestra/apps/orders/actions.py @@ -13,6 +13,7 @@ from .forms import BillSelectedOptionsForm, BillSelectConfirmationForm, BillSele class BillSelectedOrders(object): """ Form wizard for billing orders admin action """ short_description = _("Bill selected orders") + verbose_name = _("Bill") template = 'admin/orders/order/bill_selected_options.html' __name__ = 'bill_selected_orders' diff --git a/orchestra/apps/orders/admin.py b/orchestra/apps/orders/admin.py index 9e84e529..3c956216 100644 --- a/orchestra/apps/orders/admin.py +++ b/orchestra/apps/orders/admin.py @@ -59,6 +59,8 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): inlines = (MetricStorageInline,) add_inlines = () search_fields = ('account__username', 'description') + list_prefetch_related = ('metrics', 'content_object') + list_select_related = ('account', 'service') service_link = admin_link('service') content_object_link = admin_link('content_object', order=False) @@ -78,13 +80,13 @@ class OrderAdmin(AccountAdminMixin, ExtendedModelAdmin): display_billed_until.admin_order_field = 'billed_until' def display_metric(self, order): - metric = order.metrics.latest() - return metric.value if metric else '' + """ dispalys latest metric value, don't uses latest() because not loosing prefetch_related """ + try: + metric = order.metrics.all()[0] + except IndexError: + return '' + return metric.value display_metric.short_description = _("Metric") - - def get_queryset(self, request): - qs = super(OrderAdmin, self).get_queryset(request) - return qs.select_related('service').prefetch_related('content_object') class MetricStorageAdmin(admin.ModelAdmin): diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index 20308b34..dc795f02 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -258,8 +258,8 @@ class MetricStorage(models.Model): except cls.DoesNotExist: cls.objects.create(order=order, value=value, updated_on=now) else: - error = decimal.Decimal(settings.ORDERS_METRIC_ERROR) - if last.value*(1+error) > value or last.value*error < value: + error = decimal.Decimal(str(settings.ORDERS_METRIC_ERROR)) + if value > last.value+error or value < last.value-error: cls.objects.create(order=order, value=value, updated_on=now) else: last.updated_on = now diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 47824fd5..e1bd6793 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -44,6 +44,7 @@ class ResourceAdmin(ExtendedModelAdmin): change_view_actions = actions change_readonly_fields = ('name', 'content_type') prepopulated_fields = {'name': ('verbose_name',)} + list_select_related = ('content_type', 'crontab',) def change_view(self, request, object_id, form_url='', extra_context=None): """ Remaind user when monitor routes are not configured """ @@ -243,6 +244,7 @@ def resource_inline_factory(resources): return '%s %s %s' % (data.used, data.resource.unit, update_link) return _("Unknonw %s") % update_link display_used.short_description = _("Used") + display_used.allow_tags = True def has_add_permission(self, *args, **kwargs): """ Hidde add another """ diff --git a/orchestra/apps/resources/methods.py b/orchestra/apps/resources/methods.py index 78b518e5..e7fc7070 100644 --- a/orchestra/apps/resources/methods.py +++ b/orchestra/apps/resources/methods.py @@ -1,4 +1,5 @@ import datetime +import decimal from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -21,7 +22,7 @@ class DataMethod(plugins.Plugin): class Last(DataMethod): name = 'last' - verbose_name = _("Last") + verbose_name = _("Last value") def filter(self, dataset): try: @@ -71,7 +72,7 @@ class MonthlyAvg(MonthlySum): result = 0 for data in dataset: slot = (data.created_at-ini).total_seconds() - result += data.value * slot/total + result += data.value * decimal.Decimal(str(slot/total)) ini = data.created_at return result diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index 90903f94..64d75eb4 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -199,25 +199,24 @@ class ResourceData(models.Model): content_type=ct, object_id=obj.pk, resource=resource - ) + ), False except cls.DoesNotExist: return cls.objects.create( content_object=obj, resource=resource, allocated=resource.default_allocation - ) + ), True @property def unit(self): return self.resource.unit def get_used(self): - resource = data.resource + resource = self.resource total = 0 has_result = False - today = datetime.date.today() - for dataset in data.get_monitor_datasets(): - usage = data.method_instance.compute_usage(dataset) + for dataset in self.get_monitor_datasets(): + usage = resource.method_instance.compute_usage(dataset) if usage is not None: has_result = True total += usage diff --git a/orchestra/apps/resources/tasks.py b/orchestra/apps/resources/tasks.py index 6ea9a4e5..de89fb32 100644 --- a/orchestra/apps/resources/tasks.py +++ b/orchestra/apps/resources/tasks.py @@ -38,7 +38,7 @@ def monitor(resource_id, ids=None, async=True): triggers = [] model = resource.content_type.model_class() for obj in model.objects.filter(**kwargs): - data = ResourceData.get_or_create(obj, resource) + data, __ = ResourceData.get_or_create(obj, resource) data.update() if not resource.disable_trigger: a = data.used diff --git a/orchestra/apps/resources/validators.py b/orchestra/apps/resources/validators.py index 803f29ec..2d0db904 100644 --- a/orchestra/apps/resources/validators.py +++ b/orchestra/apps/resources/validators.py @@ -5,7 +5,7 @@ from django.utils.translation import ugettext_lazy as _ def validate_scale(value): try: int(eval(value)) - except Exception, e: + except Exception as e: raise ValidationError( _("'%s' is not a valid scale expression. (%s)") % (value, str(e)) ) diff --git a/orchestra/apps/saas/backends/gitlab.py b/orchestra/apps/saas/backends/gitlab.py index 3e935c5b..2925504a 100644 --- a/orchestra/apps/saas/backends/gitlab.py +++ b/orchestra/apps/saas/backends/gitlab.py @@ -54,7 +54,7 @@ class GitLabSaaSBackend(ServiceController): saas.data['user_id'] = user['id'] # Using queryset update to avoid triggering backends with the post_save signal type(saas).objects.filter(pk=saas.pk).update(data=saas.data) - print json.dumps(user, indent=4) + print(json.dumps(user, indent=4)) def change_password(self, saas, server): self.authenticate() @@ -65,7 +65,7 @@ class GitLabSaaSBackend(ServiceController): user['password'] = saas.password response = requests.put(user_url, data=user, headers=self.headers) user = self.validate_response(response, 200) - print json.dumps(user, indent=4) + print(json.dumps(user, indent=4)) def set_state(self, saas, server): # TODO http://feedback.gitlab.com/forums/176466-general/suggestions/4098632-add-administrative-api-call-to-block-users @@ -77,7 +77,7 @@ class GitLabSaaSBackend(ServiceController): user['state'] = 'active' if saas.active else 'blocked', response = requests.patch(user_url, data=user, headers=self.headers) user = self.validate_response(response, 200) - print json.dumps(user, indent=4) + print(json.dumps(user, indent=4)) def delete_user(self, saas, server): self.authenticate() diff --git a/orchestra/apps/services/admin.py b/orchestra/apps/services/admin.py index 5329eabc..9b2eeb6c 100644 --- a/orchestra/apps/services/admin.py +++ b/orchestra/apps/services/admin.py @@ -1,6 +1,8 @@ from django import forms +from django.conf.urls import patterns, url from django.contrib import admin from django.core.urlresolvers import reverse +from django.template.response import TemplateResponse from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -35,10 +37,22 @@ class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): 'on_cancel', 'payment_style', 'tax', 'nominal_price') }), ) - actions = [update_orders, clone] - change_view_actions = actions + [view_help] + actions = (update_orders, clone) + change_view_actions = actions + (view_help,) change_form_template = 'admin/services/service/change_form.html' + def get_urls(self): + """Returns the additional urls for the change view links""" + urls = super(ServiceAdmin, self).get_urls() + admin_site = self.admin_site + opts = self.model._meta + return patterns('', + url('^add/help/$', + admin_site.admin_view(self.help_view), + name='%s_%s_help' % (opts.app_label, opts.model_name) + ) + ) + urls + def formfield_for_dbfield(self, db_field, **kwargs): """ Improve performance of account field and filter by account """ if db_field.name == 'content_type': @@ -72,6 +86,20 @@ class ServiceAdmin(ChangeViewActionsMixin, admin.ModelAdmin): ) }) return qs + + def help_view(self, request, *args): + opts = self.model._meta + context = { + 'add': True, + 'title': _("Need some help?"), + 'opts': opts, + 'obj': args[0].get() if args else None, + 'action_name': _("help"), + 'app_label': opts.app_label, + } + return TemplateResponse(request, 'admin/services/service/help.html', context) + help_view.url_name = 'help' + help_view.verbose_name = _("Help") admin.site.register(Service, ServiceAdmin) diff --git a/orchestra/apps/services/handlers.py b/orchestra/apps/services/handlers.py index 4acb890c..0ef71930 100644 --- a/orchestra/apps/services/handlers.py +++ b/orchestra/apps/services/handlers.py @@ -52,7 +52,7 @@ class ServiceHandler(plugins.Plugin): return try: bool(self.matches(obj)) - except Exception, exception: + except Exception as exception: name = type(exception).__name__ message = exception.message raise ValidationError(': '.join((name, message))) @@ -64,7 +64,7 @@ class ServiceHandler(plugins.Plugin): return try: bool(self.get_metric(obj)) - except Exception, exception: + except Exception as exception: name = type(exception).__name__ message = exception.message raise ValidationError(': '.join((name, message))) @@ -187,17 +187,17 @@ class ServiceHandler(plugins.Plugin): size = rdelta.years * 12 size += rdelta.months days = calendar.monthrange(end.year, end.month)[1] - size += decimal.Decimal(rdelta.days)/days + size += decimal.Decimal(str(rdelta.days))/days elif self.billing_period == self.ANUAL: size = rdelta.years - size += decimal.Decimal(rdelta.months)/12 + size += decimal.Decimal(str(rdelta.months))/12 days = 366 if calendar.isleap(end.year) else 365 - size += decimal.Decimal(rdelta.days)/days + size += decimal.Decimal(str(rdelta.days))/days elif self.billing_period == self.NEVER: size = 1 else: raise NotImplementedError - return decimal.Decimal(size) + return decimal.Decimal(str(size)) def get_pricing_slots(self, ini, end): day = 1 diff --git a/orchestra/apps/services/models.py b/orchestra/apps/services/models.py index d0269bc4..66bb013b 100644 --- a/orchestra/apps/services/models.py +++ b/orchestra/apps/services/models.py @@ -211,14 +211,14 @@ class Service(models.Model): if counter >= metric: counter = metric accumulated += (counter - ant_counter) * rate['price'] - return decimal.Decimal(accumulated) + return decimal.Decimal(str(accumulated)) ant_counter = counter accumulated += rate['price'] * rate['quantity'] else: for rate in rates: counter += rate['quantity'] if counter >= position: - return decimal.Decimal(rate['price']) + return decimal.Decimal(str(rate['price'])) def get_rates(self, account, cache=True): # rates are cached per account diff --git a/orchestra/apps/services/templates/admin/services/service/change_form.html b/orchestra/apps/services/templates/admin/services/service/change_form.html index 8869433f..5f512dcd 100644 --- a/orchestra/apps/services/templates/admin/services/service/change_form.html +++ b/orchestra/apps/services/templates/admin/services/service/change_form.html @@ -44,7 +44,7 @@ payment_style=PREPAY">Database
  • - {% trans "Help" %} + {% trans "Help" %}
    • {% endif %} diff --git a/orchestra/apps/services/templates/admin/services/service/help.html b/orchestra/apps/services/templates/admin/services/service/help.html index 069252b9..5308eddc 100644 --- a/orchestra/apps/services/templates/admin/services/service/help.html +++ b/orchestra/apps/services/templates/admin/services/service/help.html @@ -7,7 +7,6 @@ {% block content %}
    - Enjoy my friend.
    diff --git a/orchestra/apps/services/tests/functional_tests/test_mailbox.py b/orchestra/apps/services/tests/functional_tests/test_mailbox.py index bb3f0bf3..761fe0cb 100644 --- a/orchestra/apps/services/tests/functional_tests/test_mailbox.py +++ b/orchestra/apps/services/tests/functional_tests/test_mailbox.py @@ -69,8 +69,7 @@ class MailboxBillingTest(BaseBillingTest): return self.resource def allocate_disk(self, mailbox, value): - # TODO get_or_Create return created - data = ResourceData.get_or_create(mailbox, self.resource) + data, __ = ResourceData.get_or_create(mailbox, self.resource) data.allocated = value data.save() diff --git a/orchestra/apps/services/tests/functional_tests/test_traffic.py b/orchestra/apps/services/tests/functional_tests/test_traffic.py index 8602ba5b..b3885a5e 100644 --- a/orchestra/apps/services/tests/functional_tests/test_traffic.py +++ b/orchestra/apps/services/tests/functional_tests/test_traffic.py @@ -52,7 +52,7 @@ class BaseTrafficBillingTest(BaseBillingTest): def report_traffic(self, account, value): MonitorData.objects.create(monitor='FTPTraffic', content_object=account.systemusers.get(), value=value) - data = ResourceData.get_or_create(account, self.resource) + data, __ = ResourceData.get_or_create(account, self.resource) data.update() diff --git a/orchestra/apps/webapps/types/php.py b/orchestra/apps/webapps/types/php.py index 97760da9..2bd11b59 100644 --- a/orchestra/apps/webapps/types/php.py +++ b/orchestra/apps/webapps/types/php.py @@ -123,7 +123,9 @@ class PHPApp(AppType): def get_php_version_number(self): php_version = self.get_php_version() - number = re.findall(r'[0-9]+\.?[0-9]+', php_version) + number = re.findall(r'[0-9]+\.?[0-9]?', php_version) + if not number: + raise ValueError("No version number matches for '%s'" % php_version) if len(number) > 1: - raise ValueError("Multiple version number matches for '%'" % php_version) + raise ValueError("Multiple version number matches for '%s'" % php_version) return number[0] diff --git a/orchestra/core/validators.py b/orchestra/core/validators.py index 052a3a85..7914a8bf 100644 --- a/orchestra/core/validators.py +++ b/orchestra/core/validators.py @@ -18,7 +18,7 @@ def all_valid(kwargs): for field, validator in kwargs.iteritems(): try: validator[0](*validator[1:]) - except ValidationError, error: + except ValidationError as error: errors[field] = error if errors: raise ValidationError(errors) @@ -91,7 +91,7 @@ def validate_username(value): def validate_password(value): try: crack.VeryFascistCheck(value) - except ValueError, message: + except ValueError as message: raise ValidationError("Password %s." % str(message)[3:]) diff --git a/orchestra/management/commands/staticcheck.py b/orchestra/management/commands/staticcheck.py index 9585ff93..05c74398 100644 --- a/orchestra/management/commands/staticcheck.py +++ b/orchestra/management/commands/staticcheck.py @@ -46,7 +46,7 @@ def check(codeString, filename): try: with BlackHole(): tree = ast.parse(codeString, filename) - except SyntaxError, e: + except SyntaxError as e: return [PySyntaxError(filename, e)] else: # Okay, it's syntactically valid. Now parse it into an ast and check it @@ -67,7 +67,7 @@ def checkPath(filename): """ try: return check(file(filename, 'U').read() + '\n', filename) - except IOError, msg: + except IOError as msg: return ["%s: %s" % (filename, msg.args[1])] except TypeError: pass diff --git a/orchestra/templates/admin/orchestra/generic_confirmation.html b/orchestra/templates/admin/orchestra/generic_confirmation.html index 0b21e4d1..5a99b0dd 100644 --- a/orchestra/templates/admin/orchestra/generic_confirmation.html +++ b/orchestra/templates/admin/orchestra/generic_confirmation.html @@ -17,6 +17,9 @@ {% if obj %} › {{ obj }} › {{ action_name }} +{% elif add %} +› {% trans "Add" %} {{ opts.verbose_name }} +› {{ action_name }} {% else %} › {{ action_name }} multiple objects {% endif %} diff --git a/orchestra/utils/system.py b/orchestra/utils/system.py index fc6264b9..f35bd437 100644 --- a/orchestra/utils/system.py +++ b/orchestra/utils/system.py @@ -38,7 +38,7 @@ def read_async(fd): """ try: return fd.read() - except IOError, e: + except IOError as e: if e.errno != errno.EAGAIN: raise e else: @@ -74,7 +74,7 @@ def runiterator(command, display=False, error_codes=[0], silent=False, stdin='', try: stdout += unicode(stdoutPiece.decode("utf8")) if force_unicode else stdoutPiece sdterr += unicode(stderrPiece.decode("utf8")) if force_unicode else stderrPiece - except UnicodeDecodeError, e: + except UnicodeDecodeError as e: pass else: break