diff --git a/TODO.md b/TODO.md index 90528bdc..ddd23f8e 100644 --- a/TODO.md +++ b/TODO.md @@ -362,3 +362,8 @@ resorce monitoring more efficient, less mem an better queries for calc current d # autoresponses on mailboxes, not addresses or remove them # Async particular actions? + + +apt-get install cython3 +export CYTHON='cython3' +pip3 install https://github.com/fantix/gevent/archive/master.zip diff --git a/orchestra/contrib/resources/models.py b/orchestra/contrib/resources/models.py index 72e9697c..4ad8a8f4 100644 --- a/orchestra/contrib/resources/models.py +++ b/orchestra/contrib/resources/models.py @@ -5,12 +5,13 @@ from django.db import models from django.utils import timezone from django.utils.functional import cached_property from django.utils.translation import ugettext_lazy as _ +from djcelery.models import PeriodicTask from orchestra.core import validators from orchestra.models import queryset, fields from orchestra.models.utils import get_model_field_path -from . import tasks, settings +from . import tasks from .backends import ServiceMonitor from .aggregations import Aggregation from .validators import validate_scale @@ -37,9 +38,9 @@ class Resource(models.Model): _related = set() # keeps track of related models for resource cleanup name = models.CharField(_("name"), max_length=32, - help_text=_("Required. 32 characters or fewer. Lowercase letters, " - "digits and hyphen only."), - validators=[validators.validate_name]) + help_text=_("Required. 32 characters or fewer. Lowercase letters, " + "digits and hyphen only."), + validators=[validators.validate_name]) verbose_name = models.CharField(_("verbose name"), max_length=256) content_type = models.ForeignKey(ContentType, help_text=_("Model where this resource will be hooked.")) @@ -118,18 +119,32 @@ class Resource(models.Model): created = not self.pk super(Resource, self).save(*args, **kwargs) self.sync_periodic_task() - # This only work on tests (multiprocessing used on real deployments) + # This only works on tests (multiprocessing used on real deployments) apps.get_app_config('resources').reload_relations() def delete(self, *args, **kwargs): super(Resource, self).delete(*args, **kwargs) - name = 'monitor.%s' % str(self) self.sync_periodic_task() def sync_periodic_task(self): name = 'monitor.%s' % str(self) - sync = import_class(settings.RESOURCES_TASK_BACKEND) - return sync(self, name) + if resource.pk and resource.crontab: + try: + task = PeriodicTask.objects.get(name=name) + except PeriodicTask.DoesNotExist: + if resource.is_active: + PeriodicTask.objects.create( + name=name, + task='resources.Monitor', + args=[resource.pk], + crontab=resource.crontab + ) + else: + if task.crontab != resource.crontab: + task.crontab = resource.crontab + task.save(update_fields=['crontab']) + else: + PeriodicTask.objects.filter(name=name).delete() def get_model_path(self, monitor): """ returns a model path between self.content_type and monitor.model """ diff --git a/orchestra/contrib/resources/settings.py b/orchestra/contrib/resources/settings.py deleted file mode 100644 index 571df6aa..00000000 --- a/orchestra/contrib/resources/settings.py +++ /dev/null @@ -1,6 +0,0 @@ -from orchestra.contrib.settings import Setting - - -RESOURCES_TASK_BACKEND = Setting('RESOURCES_TASK_BACKEND', - 'orchestra.contrib.resources.utils.cron_sync' -) diff --git a/orchestra/contrib/resources/utils.py b/orchestra/contrib/resources/utils.py deleted file mode 100644 index 61d17a9a..00000000 --- a/orchestra/contrib/resources/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -from orchestra.contrib.crons.utils import apply_local - -from . import settings - - -def celery_sync(resource, name): - from djcelery.models import PeriodicTask - if resource.pk and resource.crontab: - try: - task = PeriodicTask.objects.get(name=name) - except PeriodicTask.DoesNotExist: - if resource.is_active: - PeriodicTask.objects.create( - name=name, - task='resources.Monitor', - args=[resource.pk], - crontab=resource.crontab - ) - else: - if task.crontab != resource.crontab: - task.crontab = resource.crontab - task.save(update_fields=['crontab']) - else: - PeriodicTask.objects.filter( - name=name, - ).delete() - - -def cron_sync(resource, name): - if resource.pk and resource.crontab: - context = { - 'manager': os.path.join(paths.get_project_dir(), 'manage.py'), - 'id': resource.pk, - } - apply_local(resource.crontab, - 'python3 %(manager)s runmethod orchestra.contrib.resources.tasks.monitor %(id)s', - 'orchestra', # TODO - name - ) - else: - apply_local(resource.crontab, '', 'orchestra', name, action='delete') diff --git a/orchestra/contrib/services/tests/functional_tests/test_domain.py b/orchestra/contrib/services/tests/functional_tests/test_domain.py index eb050f0a..188a7334 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_domain.py +++ b/orchestra/contrib/services/tests/functional_tests/test_domain.py @@ -2,14 +2,12 @@ from django.contrib.contenttypes.models import ContentType from orchestra.contrib.miscellaneous.models import MiscService, Miscellaneous from orchestra.contrib.plans.models import Plan -from orchestra.utils.tests import random_ascii +from orchestra.utils.tests import random_ascii, BaseTestCase from ...models import Service -from . import BaseBillingTest - -class DomainBillingTest(BaseBillingTest): +class DomainBillingTest(BaseTestCase): def create_domain_service(self): service = Service.objects.create( description="Domain .ES", diff --git a/orchestra/contrib/services/tests/functional_tests/test_ftp.py b/orchestra/contrib/services/tests/functional_tests/test_ftp.py index 93c07c15..9a2b6def 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_ftp.py +++ b/orchestra/contrib/services/tests/functional_tests/test_ftp.py @@ -6,15 +6,13 @@ from django.contrib.contenttypes.models import ContentType from django.utils import timezone from orchestra.contrib.systemusers.models import SystemUser -from orchestra.utils.tests import random_ascii +from orchestra.utils.tests import random_ascii, BaseTestCase from ... import settings from ...models import Service -from . import BaseBillingTest - -class FTPBillingTest(BaseBillingTest): +class FTPBillingTest(BaseTestCase): def create_ftp_service(self): return Service.objects.create( description="FTP Account", diff --git a/orchestra/contrib/services/tests/functional_tests/test_job.py b/orchestra/contrib/services/tests/functional_tests/test_job.py index a208bea9..5d9e3dbf 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_job.py +++ b/orchestra/contrib/services/tests/functional_tests/test_job.py @@ -2,14 +2,12 @@ from django.contrib.contenttypes.models import ContentType from orchestra.contrib.miscellaneous.models import MiscService, Miscellaneous from orchestra.contrib.plans.models import Plan -from orchestra.utils.tests import random_ascii +from orchestra.utils.tests import random_ascii, BaseTestCase from ...models import Service -from . import BaseBillingTest - -class JobBillingTest(BaseBillingTest): +class JobBillingTest(BaseTestCase): def create_job_service(self): service = Service.objects.create( description="Random job", diff --git a/orchestra/contrib/services/tests/functional_tests/test_mailbox.py b/orchestra/contrib/services/tests/functional_tests/test_mailbox.py index 6d2c76c5..162227ed 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_mailbox.py +++ b/orchestra/contrib/services/tests/functional_tests/test_mailbox.py @@ -6,14 +6,12 @@ from freezegun import freeze_time from orchestra.contrib.mailboxes.models import Mailbox from orchestra.contrib.plans.models import Plan from orchestra.contrib.resources.models import Resource, ResourceData -from orchestra.utils.tests import random_ascii +from orchestra.utils.tests import random_ascii, BaseTestCase from ...models import Service -from . import BaseBillingTest - -class MailboxBillingTest(BaseBillingTest): +class MailboxBillingTest(BaseTestCase): def create_mailbox_service(self): service = Service.objects.create( description="Mailbox", @@ -59,7 +57,7 @@ class MailboxBillingTest(BaseBillingTest): self.resource = Resource.objects.create( name='disk', content_type=ContentType.objects.get_for_model(Mailbox), - period=Resource.LAST, + aggregation='last', verbose_name='Mailbox disk', unit='GB', scale=10**9, diff --git a/orchestra/contrib/services/tests/functional_tests/test_plan.py b/orchestra/contrib/services/tests/functional_tests/test_plan.py index 7ffffdf1..19a7a61c 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_plan.py +++ b/orchestra/contrib/services/tests/functional_tests/test_plan.py @@ -1,13 +1,12 @@ from django.contrib.contenttypes.models import ContentType from orchestra.contrib.plans.models import Plan, ContractedPlan +from orchestra.utils.tests import BaseTestCase from ...models import Service -from . import BaseBillingTest - -class PlanBillingTest(BaseBillingTest): +class PlanBillingTest(BaseTestCase): def create_plan_service(self): service = Service.objects.create( description="Association membership fee", diff --git a/orchestra/contrib/services/tests/functional_tests/test_traffic.py b/orchestra/contrib/services/tests/functional_tests/test_traffic.py index db357f66..0cb13ecc 100644 --- a/orchestra/contrib/services/tests/functional_tests/test_traffic.py +++ b/orchestra/contrib/services/tests/functional_tests/test_traffic.py @@ -7,13 +7,12 @@ from orchestra.contrib.accounts.models import Account from orchestra.contrib.miscellaneous.models import MiscService, Miscellaneous from orchestra.contrib.plans.models import Plan from orchestra.contrib.resources.models import Resource, ResourceData, MonitorData +from orchestra.utils.tests import BaseTestCase from ...models import Service -from . import BaseBillingTest - -class BaseTrafficBillingTest(BaseBillingTest): +class BaseTrafficBillingTest(BaseTestCase): TRAFFIC_METRIC = 'account.resources.traffic.used' def create_traffic_service(self): @@ -41,7 +40,7 @@ class BaseTrafficBillingTest(BaseBillingTest): self.resource = Resource.objects.create( name='traffic', content_type=ContentType.objects.get_for_model(Account), - period=Resource.MONTHLY_SUM, + aggregation='monthly-sum', verbose_name='Account Traffic', unit='GB', scale='10**9', diff --git a/orchestra/contrib/systemusers/backends.py b/orchestra/contrib/systemusers/backends.py index 67f89408..2be5847d 100644 --- a/orchestra/contrib/systemusers/backends.py +++ b/orchestra/contrib/systemusers/backends.py @@ -35,11 +35,14 @@ class UNIXUserBackend(ServiceController): usermod %(user)s --home %(home)s --password '%(password)s' --shell %(shell)s %(groups_arg)s else useradd %(user)s --home %(home)s --password '%(password)s' --shell %(shell)s %(groups_arg)s || { + useradd_code=$? # User is logged in, kill and retry - if [[ $? -eq 8 ]]; then + if [[ $useradd_code -eq 8 ]]; then pkill -u %(user)s; sleep 2 pkill -9 -u %(user)s; sleep 1 useradd %(user)s --home %(home)s --password '%(password)s' --shell %(shell)s %(groups_arg)s + else + exit $useradd_code fi } fi