From 2beed2677aae985629948a9a768b5772f2974398 Mon Sep 17 00:00:00 2001 From: Marc Aymerich Date: Thu, 10 Sep 2015 10:34:07 +0000 Subject: [PATCH] Added metricstorage cleanup periodic task --- TODO.md | 4 -- orchestra/contrib/domains/models.py | 2 +- orchestra/contrib/orders/settings.py | 6 +++ orchestra/contrib/orders/tasks.py | 50 +++++++++++++++++++ .../contrib/saas/backends/wordpressmu.py | 30 +++++------ orchestra/contrib/saas/settings.py | 2 +- orchestra/contrib/services/handlers.py | 14 ++++-- 7 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 orchestra/contrib/orders/tasks.py diff --git a/TODO.md b/TODO.md index fa24b064..374f2010 100644 --- a/TODO.md +++ b/TODO.md @@ -182,7 +182,6 @@ ugettext("Description") * saas validate_creation generic approach, for all backends. standard output -* periodic task to cleanup metricstorage # create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help * postupgradeorchestra send signals in order to hook custom stuff @@ -394,6 +393,3 @@ Case # Don't enforce one contact per account? remove account.email in favour of contacts? # Mailer: mark as sent - -# Pending filter filter out orders zero metric from pending - diff --git a/orchestra/contrib/domains/models.py b/orchestra/contrib/domains/models.py index 8781cffc..92c8c615 100644 --- a/orchestra/contrib/domains/models.py +++ b/orchestra/contrib/domains/models.py @@ -33,7 +33,7 @@ class Domain(models.Model): account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), blank=True, related_name='domains', help_text=_("Automatically selected for subdomains.")) top = models.ForeignKey('domains.Domain', null=True, related_name='subdomain_set', - editable=False) + editable=False, verbose_name=_("top domain")) serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, editable=False, help_text=_("A revision number that changes whenever this domain is updated.")) refresh = models.CharField(_("refresh"), max_length=16, blank=True, diff --git a/orchestra/contrib/orders/settings.py b/orchestra/contrib/orders/settings.py index 6a59f0e3..91593b5e 100644 --- a/orchestra/contrib/orders/settings.py +++ b/orchestra/contrib/orders/settings.py @@ -36,3 +36,9 @@ ORDERS_METRIC_ERROR = Setting('ORDERS_METRIC_ERROR', help_text=("Only account for significative changes.
" "metric_storage new value: lastvalue*(1+threshold) > currentvalue or lastvalue*threshold < currentvalue."), ) + + +ORDERS_BILLED_METRIC_CLEANUP_DAYS = Setting('ORDERS_BILLED_METRIC_CLEANUP_DAYS', + 40, + help_text=("Number of days after a billed stored metric is deleted."), +) diff --git a/orchestra/contrib/orders/tasks.py b/orchestra/contrib/orders/tasks.py new file mode 100644 index 00000000..6089e652 --- /dev/null +++ b/orchestra/contrib/orders/tasks.py @@ -0,0 +1,50 @@ +import datetime + +from celery.task.schedules import crontab +from django.apps import apps + +from orchestra.contrib.tasks import periodic_task + +from . import settings + + +@periodic_task(run_every=crontab(hour=4, minute=30), name='orders.cleanup_metrics') +def cleanup_metrics(): + from .models import MetricStorage, Order + Service = apps.get_model(settings.ORDERS_SERVICE_MODEL) + + # General cleaning: order.billed_on-delta + general = 0 + delta = datetime.timedelta(days=settings.ORDERS_BILLED_METRIC_CLEANUP_DAYS) + for order in Order.objects.filter(billed_on__isnull=False): + epoch = order.billed_on-delta + try: + latest = order.metrics.filter(updated_on__lt=epoch).latest('updated_on') + except MetricStorage.DoesNotExist: + pass + else: + general += order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).count() + order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).only('id').delete() + + # Reduce monthly metrics to latest + monthly = 0 + monthly_services = Service.objects.exclude(metric='').filter( + billing_period=Service.MONTHLY, pricing_period=Service.BILLING_PERIOD + ) + for service in monthly_services: + for order in Order.objects.filter(service=service): + dates = order.metrics.values_list('created_on', flat=True) + months = set((date.year, date.month) for date in dates) + for year, month in months: + metrics = order.metrics.filter( + created_on__year=year, created_on__month=month, + updated_on__year=year, updated_on__month=month) + try: + latest = metrics.latest('updated_on') + except MetricStorage.DoesNotExist: + pass + else: + monthly += metrics.exclude(pk=latest.pk).count() + metrics.exclude(pk=latest.pk).only('id').delete() + + return (general, monthly) diff --git a/orchestra/contrib/saas/backends/wordpressmu.py b/orchestra/contrib/saas/backends/wordpressmu.py index 24daae1b..be8e8def 100644 --- a/orchestra/contrib/saas/backends/wordpressmu.py +++ b/orchestra/contrib/saas/backends/wordpressmu.py @@ -40,18 +40,18 @@ class WordpressMuBackend(ServiceController): errors = re.findall(r'\n\t

(.*)

', response.content.decode('utf8')) raise RuntimeError(errors[0] if errors else 'Unknown %i error' % response.status_code) - def get_id(self, session, webapp): + def get_id(self, session, saas): search = self.get_base_url() - search += '/wp-admin/network/sites.php?s=%s&action=blogs' % webapp.name + search += '/wp-admin/network/sites.php?s=%s&action=blogs' % saas.name regex = re.compile( '%s' % webapp.name + 'class="edit">%s' % saas.name ) content = session.get(search).content.decode('utf8') # Get id ids = regex.search(content) if not ids: - raise RuntimeError("Blog '%s' not found" % webapp.name) + raise RuntimeError("Blog '%s' not found" % saas.name) ids = ids.groups() if len(ids) > 1: raise ValueError("Multiple matches") @@ -60,13 +60,13 @@ class WordpressMuBackend(ServiceController): wpnonce = re.search(r'_wpnonce=([^"]*)"', wpnonce).groups()[0] return int(ids[0]), wpnonce - def create_blog(self, webapp, server): + def create_blog(self, saas, server): session = requests.Session() self.login(session) # Check if blog already exists try: - self.get_id(session, webapp) + self.get_id(session, saas) except RuntimeError: url = self.get_base_url() url += '/wp-admin/network/site-new.php' @@ -77,9 +77,9 @@ class WordpressMuBackend(ServiceController): url += '?action=add-site' data = { - 'blog[domain]': webapp.name, - 'blog[title]': webapp.name, - 'blog[email]': webapp.account.email, + 'blog[domain]': saas.name, + 'blog[title]': saas.name, + 'blog[email]': saas.account.email, '_wpnonce_add-blog': wpnonce, } @@ -87,12 +87,12 @@ class WordpressMuBackend(ServiceController): response = session.post(url, data=data) self.validate_response(response) - def delete_blog(self, webapp, server): + def delete_blog(self, saas, server): session = requests.Session() self.login(session) try: - id, wpnonce = self.get_id(session, webapp) + id, wpnonce = self.get_id(session, saas) except RuntimeError: pass else: @@ -114,8 +114,8 @@ class WordpressMuBackend(ServiceController): response = session.post(delete, data=data) self.validate_response(response) - def save(self, webapp): - self.append(self.create_blog, webapp) + def save(self, saas): + self.append(self.create_blog, saas) - def delete(self, webapp): - self.append(self.delete_blog, webapp) + def delete(self, saas): + self.append(self.delete_blog, saas) diff --git a/orchestra/contrib/saas/settings.py b/orchestra/contrib/saas/settings.py index 6feeb5d6..2105bcf5 100644 --- a/orchestra/contrib/saas/settings.py +++ b/orchestra/contrib/saas/settings.py @@ -23,7 +23,7 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES', ) -SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESSMU_ADMIN_PASSWORD', +SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD', 'secret' ) diff --git a/orchestra/contrib/services/handlers.py b/orchestra/contrib/services/handlers.py index 7d5af6f5..d36c881e 100644 --- a/orchestra/contrib/services/handlers.py +++ b/orchestra/contrib/services/handlers.py @@ -25,6 +25,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): """ _PLAN = 'plan' _COMPENSATION = 'compensation' + _PREPAY = 'prepay' model = None @@ -275,14 +276,15 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): 'metric': metric, 'discounts': [], }) + if subtotal > price: plan_discount = price-subtotal self.generate_discount(line, self._PLAN, plan_discount) subtotal += plan_discount for dtype, dprice in discounts: subtotal += dprice - # Prevent compensations to refund money - if dtype == self._COMPENSATION and subtotal < 0: + # Prevent compensations/prepays to refund money + if dtype in (self._COMPENSATION, self._PREPAY) and subtotal < 0: dprice -= subtotal if dprice: self.generate_discount(line, dtype, dprice) @@ -527,7 +529,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if discount > 0: price -= discount discounts = ( - ('prepay', -discount), + (self._PREPAY, -discount), ) # Don't overdload bills with lots of lines if price > 0: @@ -535,6 +537,8 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): if prepay_discount < 0: # User has prepaid less than the actual consumption for order, price, cini, cend, metric, discounts in recharges: + if discounts: + price -= discounts[0][1] line = self.generate_line(order, price, cini, cend, metric=metric, computed=True, discounts=discounts) lines.append(line) @@ -561,7 +565,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): # price -= discount # prepay_discount -= discount # discounts = ( -# ('prepay', -discount), +# (self._PREPAY', -discount), # ) if metric > 0: line = self.generate_line(order, price, cini, cend, metric=metric, @@ -580,7 +584,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount): # price -= discount # prepay_discount -= discount # discounts = ( -# ('prepay', -discount), +# (self._PREPAY, -discount), # ) if metric > 0: line = self.generate_line(order, price, cini, cend, metric=metric,