Added metricstorage cleanup periodic task
This commit is contained in:
Marc Aymerich
parent
332ad49b64
commit
2beed2677a
4
TODO.md
4
TODO.md
|
|
@ -182,7 +182,6 @@ ugettext("Description")
|
|||
|
||||
* saas validate_creation generic approach, for all backends. standard output
|
||||
|
||||
* periodic task to cleanup metricstorage
|
||||
# create orchestrate databases.Database pk=1 -n --dry-run | --noinput --action save (default)|delete --backend name (limit to this backend) --help
|
||||
|
||||
* postupgradeorchestra send signals in order to hook custom stuff
|
||||
|
|
@ -394,6 +393,3 @@ Case
|
|||
# Don't enforce one contact per account? remove account.email in favour of contacts?
|
||||
|
||||
# Mailer: mark as sent
|
||||
|
||||
# Pending filter filter out orders zero metric from pending
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class Domain(models.Model):
|
|||
account = models.ForeignKey('accounts.Account', verbose_name=_("Account"), blank=True,
|
||||
related_name='domains', help_text=_("Automatically selected for subdomains."))
|
||||
top = models.ForeignKey('domains.Domain', null=True, related_name='subdomain_set',
|
||||
editable=False)
|
||||
editable=False, verbose_name=_("top domain"))
|
||||
serial = models.IntegerField(_("serial"), default=utils.generate_zone_serial, editable=False,
|
||||
help_text=_("A revision number that changes whenever this domain is updated."))
|
||||
refresh = models.CharField(_("refresh"), max_length=16, blank=True,
|
||||
|
|
|
|||
|
|
@ -36,3 +36,9 @@ ORDERS_METRIC_ERROR = Setting('ORDERS_METRIC_ERROR',
|
|||
help_text=("Only account for significative changes.<br>"
|
||||
"metric_storage new value: <tt>lastvalue*(1+threshold) > currentvalue or lastvalue*threshold < currentvalue</tt>."),
|
||||
)
|
||||
|
||||
|
||||
ORDERS_BILLED_METRIC_CLEANUP_DAYS = Setting('ORDERS_BILLED_METRIC_CLEANUP_DAYS',
|
||||
40,
|
||||
help_text=("Number of days after a billed stored metric is deleted."),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
import datetime
|
||||
|
||||
from celery.task.schedules import crontab
|
||||
from django.apps import apps
|
||||
|
||||
from orchestra.contrib.tasks import periodic_task
|
||||
|
||||
from . import settings
|
||||
|
||||
|
||||
@periodic_task(run_every=crontab(hour=4, minute=30), name='orders.cleanup_metrics')
|
||||
def cleanup_metrics():
|
||||
from .models import MetricStorage, Order
|
||||
Service = apps.get_model(settings.ORDERS_SERVICE_MODEL)
|
||||
|
||||
# General cleaning: order.billed_on-delta
|
||||
general = 0
|
||||
delta = datetime.timedelta(days=settings.ORDERS_BILLED_METRIC_CLEANUP_DAYS)
|
||||
for order in Order.objects.filter(billed_on__isnull=False):
|
||||
epoch = order.billed_on-delta
|
||||
try:
|
||||
latest = order.metrics.filter(updated_on__lt=epoch).latest('updated_on')
|
||||
except MetricStorage.DoesNotExist:
|
||||
pass
|
||||
else:
|
||||
general += order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).count()
|
||||
order.metrics.exclude(pk=latest.pk).filter(updated_on__lt=epoch).only('id').delete()
|
||||
|
||||
# Reduce monthly metrics to latest
|
||||
monthly = 0
|
||||
monthly_services = Service.objects.exclude(metric='').filter(
|
||||
billing_period=Service.MONTHLY, pricing_period=Service.BILLING_PERIOD
|
||||
)
|
||||
for service in monthly_services:
|
||||
for order in Order.objects.filter(service=service):
|
||||
dates = order.metrics.values_list('created_on', flat=True)
|
||||
months = set((date.year, date.month) for date in dates)
|
||||
for year, month in months:
|
||||
metrics = order.metrics.filter(
|
||||
created_on__year=year, created_on__month=month,
|
||||
updated_on__year=year, updated_on__month=month)
|
||||
try:
|
||||
latest = metrics.latest('updated_on')
|
||||
except MetricStorage.DoesNotExist:
|
||||
pass
|
||||
else:
|
||||
monthly += metrics.exclude(pk=latest.pk).count()
|
||||
metrics.exclude(pk=latest.pk).only('id').delete()
|
||||
|
||||
return (general, monthly)
|
||||
|
|
@ -40,18 +40,18 @@ class WordpressMuBackend(ServiceController):
|
|||
errors = re.findall(r'<body id="error-page">\n\t<p>(.*)</p></body>', response.content.decode('utf8'))
|
||||
raise RuntimeError(errors[0] if errors else 'Unknown %i error' % response.status_code)
|
||||
|
||||
def get_id(self, session, webapp):
|
||||
def get_id(self, session, saas):
|
||||
search = self.get_base_url()
|
||||
search += '/wp-admin/network/sites.php?s=%s&action=blogs' % webapp.name
|
||||
search += '/wp-admin/network/sites.php?s=%s&action=blogs' % saas.name
|
||||
regex = re.compile(
|
||||
'<a href="http://[\.\-\w]+/wp-admin/network/site-info\.php\?id=([0-9]+)"\s+'
|
||||
'class="edit">%s</a>' % webapp.name
|
||||
'class="edit">%s</a>' % saas.name
|
||||
)
|
||||
content = session.get(search).content.decode('utf8')
|
||||
# Get id
|
||||
ids = regex.search(content)
|
||||
if not ids:
|
||||
raise RuntimeError("Blog '%s' not found" % webapp.name)
|
||||
raise RuntimeError("Blog '%s' not found" % saas.name)
|
||||
ids = ids.groups()
|
||||
if len(ids) > 1:
|
||||
raise ValueError("Multiple matches")
|
||||
|
|
@ -60,13 +60,13 @@ class WordpressMuBackend(ServiceController):
|
|||
wpnonce = re.search(r'_wpnonce=([^"]*)"', wpnonce).groups()[0]
|
||||
return int(ids[0]), wpnonce
|
||||
|
||||
def create_blog(self, webapp, server):
|
||||
def create_blog(self, saas, server):
|
||||
session = requests.Session()
|
||||
self.login(session)
|
||||
|
||||
# Check if blog already exists
|
||||
try:
|
||||
self.get_id(session, webapp)
|
||||
self.get_id(session, saas)
|
||||
except RuntimeError:
|
||||
url = self.get_base_url()
|
||||
url += '/wp-admin/network/site-new.php'
|
||||
|
|
@ -77,9 +77,9 @@ class WordpressMuBackend(ServiceController):
|
|||
|
||||
url += '?action=add-site'
|
||||
data = {
|
||||
'blog[domain]': webapp.name,
|
||||
'blog[title]': webapp.name,
|
||||
'blog[email]': webapp.account.email,
|
||||
'blog[domain]': saas.name,
|
||||
'blog[title]': saas.name,
|
||||
'blog[email]': saas.account.email,
|
||||
'_wpnonce_add-blog': wpnonce,
|
||||
}
|
||||
|
||||
|
|
@ -87,12 +87,12 @@ class WordpressMuBackend(ServiceController):
|
|||
response = session.post(url, data=data)
|
||||
self.validate_response(response)
|
||||
|
||||
def delete_blog(self, webapp, server):
|
||||
def delete_blog(self, saas, server):
|
||||
session = requests.Session()
|
||||
self.login(session)
|
||||
|
||||
try:
|
||||
id, wpnonce = self.get_id(session, webapp)
|
||||
id, wpnonce = self.get_id(session, saas)
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
|
|
@ -114,8 +114,8 @@ class WordpressMuBackend(ServiceController):
|
|||
response = session.post(delete, data=data)
|
||||
self.validate_response(response)
|
||||
|
||||
def save(self, webapp):
|
||||
self.append(self.create_blog, webapp)
|
||||
def save(self, saas):
|
||||
self.append(self.create_blog, saas)
|
||||
|
||||
def delete(self, webapp):
|
||||
self.append(self.delete_blog, webapp)
|
||||
def delete(self, saas):
|
||||
self.append(self.delete_blog, saas)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ SAAS_ENABLED_SERVICES = Setting('SAAS_ENABLED_SERVICES',
|
|||
)
|
||||
|
||||
|
||||
SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESSMU_ADMIN_PASSWORD',
|
||||
SAAS_WORDPRESS_ADMIN_PASSWORD = Setting('SAAS_WORDPRESS_ADMIN_PASSWORD',
|
||||
'secret'
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
|
|||
"""
|
||||
_PLAN = 'plan'
|
||||
_COMPENSATION = 'compensation'
|
||||
_PREPAY = 'prepay'
|
||||
|
||||
model = None
|
||||
|
||||
|
|
@ -275,14 +276,15 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
|
|||
'metric': metric,
|
||||
'discounts': [],
|
||||
})
|
||||
|
||||
if subtotal > price:
|
||||
plan_discount = price-subtotal
|
||||
self.generate_discount(line, self._PLAN, plan_discount)
|
||||
subtotal += plan_discount
|
||||
for dtype, dprice in discounts:
|
||||
subtotal += dprice
|
||||
# Prevent compensations to refund money
|
||||
if dtype == self._COMPENSATION and subtotal < 0:
|
||||
# Prevent compensations/prepays to refund money
|
||||
if dtype in (self._COMPENSATION, self._PREPAY) and subtotal < 0:
|
||||
dprice -= subtotal
|
||||
if dprice:
|
||||
self.generate_discount(line, dtype, dprice)
|
||||
|
|
@ -527,7 +529,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
|
|||
if discount > 0:
|
||||
price -= discount
|
||||
discounts = (
|
||||
('prepay', -discount),
|
||||
(self._PREPAY, -discount),
|
||||
)
|
||||
# Don't overdload bills with lots of lines
|
||||
if price > 0:
|
||||
|
|
@ -535,6 +537,8 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
|
|||
if prepay_discount < 0:
|
||||
# User has prepaid less than the actual consumption
|
||||
for order, price, cini, cend, metric, discounts in recharges:
|
||||
if discounts:
|
||||
price -= discounts[0][1]
|
||||
line = self.generate_line(order, price, cini, cend, metric=metric,
|
||||
computed=True, discounts=discounts)
|
||||
lines.append(line)
|
||||
|
|
@ -561,7 +565,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
|
|||
# price -= discount
|
||||
# prepay_discount -= discount
|
||||
# discounts = (
|
||||
# ('prepay', -discount),
|
||||
# (self._PREPAY', -discount),
|
||||
# )
|
||||
if metric > 0:
|
||||
line = self.generate_line(order, price, cini, cend, metric=metric,
|
||||
|
|
@ -580,7 +584,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
|
|||
# price -= discount
|
||||
# prepay_discount -= discount
|
||||
# discounts = (
|
||||
# ('prepay', -discount),
|
||||
# (self._PREPAY, -discount),
|
||||
# )
|
||||
if metric > 0:
|
||||
line = self.generate_line(order, price, cini, cend, metric=metric,
|
||||
|
|
|
|||
Loading…
Reference in New Issue