Fixed ransom bugs

This commit is contained in:
Marc Aymerich 2015-04-14 14:29:22 +00:00
parent 711951d1bd
commit 5606b14e28
24 changed files with 307 additions and 76 deletions

View file

@ -186,7 +186,6 @@ require_once(/etc/moodles/.$moodle_host.config.php);``` moodle/drupl
* use server.name | server.address on python backends, like gitlab instead of settings?
* TODO raise404, here and everywhere
# display subline links on billlines, to show that they exists.
* update service orders on a celery task? because it take alot
# billline quantity eval('10x100') instead of miningless description '(10*100)' line.verbose_quantity
@ -246,7 +245,6 @@ celery max-tasks-per-child
* autoscale celery workers http://docs.celeryproject.org/en/latest/userguide/workers.html#autoscaling
* webapp has_website list filter
glic3rinu's django-fluent-dashboard
* gevent is not ported to python3 :'(
@ -294,3 +292,6 @@ https://code.djangoproject.com/ticket/24576
* fpm reload starts new pools?
* rename resource.monitors to resource.backends ?
* abstract model classes enabling overriding?
# Ignore superusers & co on billing
# bill.totals make it 100% computed?

View file

@ -36,7 +36,7 @@ def get_modeladmin(model, import_module=True):
def insertattr(model, name, value):
""" Inserts attribute to a modeladmin """
modeladmin = None
if models.Model in model.__mro__:
if isinstance(model, models.Model)
modeladmin = get_modeladmin(model)
modeladmin_class = type(modeladmin)
elif not inspect.isclass(model):

View file

@ -4,6 +4,8 @@ from django.contrib import admin
from django.contrib.admin.utils import unquote
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import F, Sum
from django.db.models.functions import Coalesce
from django.templatetags.static import static
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext_lazy as _
@ -58,7 +60,9 @@ class ClosedBillLineInline(BillLineInline):
# TODO reimplement as nested inlines when upstream
# https://code.djangoproject.com/ticket/9025
fields = ('display_description', 'rate', 'quantity', 'tax', 'display_subtotal', 'display_total')
fields = (
'display_description', 'rate', 'quantity', 'tax', 'display_subtotal', 'display_total'
)
readonly_fields = fields
def display_description(self, line):
@ -77,6 +81,11 @@ class ClosedBillLineInline(BillLineInline):
display_subtotal.short_description = _("Subtotal")
display_subtotal.allow_tags = True
def display_total(self, line):
return line.get_total()
display_total.short_description = _("Total")
display_total.allow_tags = True
def has_add_permission(self, request):
return False
@ -134,6 +143,7 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
change_view_actions = [
actions.view_bill, actions.download_bills, actions.send_bills, actions.close_bills
]
search_fields = ('number', 'account__username', 'comments')
actions = [actions.download_bills, actions.close_bills, actions.send_bills]
change_readonly_fields = ('account_link', 'type', 'is_open')
readonly_fields = ('number', 'display_total', 'is_sent', 'display_payment_state')
@ -147,10 +157,10 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
num_lines.short_description = _("lines")
def display_total(self, bill):
return "%s &%s;" % (bill.total, settings.BILLS_CURRENCY.lower())
return "%s &%s;" % (round(bill.totals, 2), settings.BILLS_CURRENCY.lower())
display_total.allow_tags = True
display_total.short_description = _("total")
display_total.admin_order_field = 'total'
display_total.admin_order_field = 'totals'
def type_link(self, bill):
bill_type = bill.type.lower()
@ -210,8 +220,8 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
def get_inline_instances(self, request, obj=None):
inlines = super(BillAdmin, self).get_inline_instances(request, obj)
if obj and not obj.is_open:
return [inline for inline in inlines if not isinstance(inline, BillLineInline)]
return [inline for inline in inlines if not isinstance(inline, ClosedBillLineInline)]
return [inline for inline in inlines if type(inline) is not BillLineInline]
return [inline for inline in inlines if type(inline) is not ClosedBillLineInline]
def formfield_for_dbfield(self, db_field, **kwargs):
""" Make value input widget bigger """
@ -223,8 +233,13 @@ class BillAdmin(AccountAdminMixin, ExtendedModelAdmin):
def get_queryset(self, request):
qs = super(BillAdmin, self).get_queryset(request)
qs = qs.annotate(models.Count('lines'))
qs = qs.prefetch_related('lines', 'lines__sublines', 'transactions')
qs = qs.annotate(
models.Count('lines'),
totals=Sum(
(F('lines__subtotal') + Coalesce(F('lines__sublines__total'), 0)) * (1+F('lines__tax')/100)
),
)
qs = qs.prefetch_related('transactions')
return qs
def change_view(self, request, object_id, **kwargs):

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -2,6 +2,8 @@ from dateutil.relativedelta import relativedelta
from django.core.validators import ValidationError, RegexValidator
from django.db import models
from django.db.models import F, Sum
from django.db.models.functions import Coalesce
from django.template import loader, Context
from django.utils import timezone, translation
from django.utils.encoding import force_text
@ -89,6 +91,7 @@ class Bill(models.Model):
is_sent = models.BooleanField(_("sent"), default=False)
due_on = models.DateField(_("due on"), null=True, blank=True)
updated_on = models.DateField(_("updated on"), auto_now=True)
# TODO allways compute total or what?
total = models.DecimalField(max_digits=12, decimal_places=2, default=0)
comments = models.TextField(_("comments"), blank=True)
html = models.TextField(_("HTML"), blank=True)
@ -227,18 +230,17 @@ class Bill(models.Model):
def get_subtotals(self):
subtotals = {}
for line in self.lines.all():
subtotal, taxes = subtotals.get(line.tax, (0, 0))
subtotal += line.get_total()
subtotals[line.tax] = (subtotal, (line.tax/100)*subtotal)
lines = self.lines.annotate(totals=(F('subtotal') + Coalesce(F('sublines__total'), 0)))
for tax, total in lines.values_list('tax', 'totals'):
subtotal, taxes = subtotals.get(tax, (0, 0))
subtotal += total
subtotals[tax] = (subtotal, round(tax/100*subtotal, 2))
return subtotals
def get_total(self):
total = 0
for tax, subtotal in self.get_subtotals().items():
subtotal, taxes = subtotal
total += subtotal + taxes
return total
totals = self.lines.annotate(
totals=(F('subtotal') + Coalesce(F('sublines__total'), 0)) * (1+F('tax')/100))
return round(totals.aggregate(Sum('totals'))['totals__sum'], 2)
class Invoice(Bill):
@ -272,12 +274,12 @@ class BillLine(models.Model):
description = models.CharField(_("description"), max_length=256)
rate = models.DecimalField(_("rate"), blank=True, null=True, max_digits=12, decimal_places=2)
quantity = models.DecimalField(_("quantity"), max_digits=12, decimal_places=2)
verbose_quantity = models.CharField(_("Verbose quantity"), max_length=16)
subtotal = models.DecimalField(_("subtotal"), max_digits=12, decimal_places=2)
tax = models.DecimalField(_("tax"), max_digits=2, decimal_places=2)
tax = models.DecimalField(_("tax"), max_digits=4, decimal_places=2)
# Undo
# initial = models.DateTimeField(null=True)
# end = models.DateTimeField(null=True)
order = models.ForeignKey(settings.BILLS_ORDER_MODEL, null=True, blank=True,
help_text=_("Informative link back to the order"), on_delete=models.SET_NULL)
order_billed_on = models.DateField(_("order billed"), null=True, blank=True)
@ -297,10 +299,11 @@ class BillLine(models.Model):
def get_total(self):
""" Computes subline discounts """
total = self.subtotal
for subline in self.sublines.all():
total += subline.total
return total
if self.pk:
return self.subtotal + sum(self.sublines.values_list('total', flat=True))
def get_verbose_quantity(self):
return self.verbose_quantity or self.quantity
def undo(self):
# TODO warn user that undoing bills with compensations lead to compensation lost
@ -313,12 +316,11 @@ class BillLine(models.Model):
self.order.billed_on = self.order_billed_on
self.delete()
def save(self, *args, **kwargs):
# TODO cost and consistency of this shit
super(BillLine, self).save(*args, **kwargs)
if self.bill.is_open:
self.bill.total = self.bill.get_total()
self.bill.save(update_fields=['total'])
# def save(self, *args, **kwargs):
# super(BillLine, self).save(*args, **kwargs)
# if self.bill.is_open:
# self.bill.total = self.bill.get_total()
# self.bill.save(update_fields=['total'])
class BillSubline(models.Model):
@ -339,12 +341,12 @@ class BillSubline(models.Model):
total = models.DecimalField(max_digits=12, decimal_places=2)
type = models.CharField(_("type"), max_length=16, choices=TYPES, default=OTHER)
def save(self, *args, **kwargs):
# TODO cost of this shit
super(BillSubline, self).save(*args, **kwargs)
if self.line.bill.is_open:
self.line.bill.total = self.line.bill.get_total()
self.line.bill.save(update_fields=['total'])
# def save(self, *args, **kwargs):
# # TODO cost of this shit
# super(BillSubline, self).save(*args, **kwargs)
# if self.line.bill.is_open:
# self.line.bill.total = self.line.bill.get_total()
# self.line.bill.save(update_fields=['total'])
accounts.register(Bill)

View file

@ -79,7 +79,7 @@
{% with sublines=line.sublines.all %}
<span class="{% if not sublines %}last {% endif %}column-id">{{ line.id }}</span>
<span class="{% if not sublines %}last {% endif %}column-description">{{ line.description }}</span>
<span class="{% if not sublines %}last {% endif %}column-quantity">{{ line.quantity|default:"&nbsp;" }}</span>
<span class="{% if not sublines %}last {% endif %}column-quantity">{{ line.get_verbose_quantity|default:"&nbsp;"|safe }}</span>
<span class="{% if not sublines %}last {% endif %}column-rate">{% if line.rate %}{{ line.rate }} &{{ currency.lower }};{% else %}&nbsp;{% endif %}</span>
<span class="{% if not sublines %}last {% endif %}column-subtotal">{{ line.subtotal }} &{{ currency.lower }};</span>
<br>
@ -96,7 +96,7 @@
</div>
<div id="totals">
<br>&nbsp;<br>
{% for tax, subtotal in bill.get_subtotals.iteritems %}
{% for tax, subtotal in bill.get_subtotals.items %}
<span class="subtotal column-title">subtotal {{ tax }}% {% trans "VAT" %}</span>
<span class="subtotal column-value">{{ subtotal | first }} &{{ currency.lower }};</span>
<br>

View file

@ -37,7 +37,7 @@ class RouteAdmin(admin.ModelAdmin):
for backend, __ in ServiceBackend.get_choices()
}
DEFAULT_MATCH = {
backend.get_name(): backend.default_route_match for backend in ServiceBackend.get_backends(active=False)
backend.get_name(): backend.default_route_match for backend in ServiceBackend.get_backends()
}
def display_model(self, route):

View file

@ -99,15 +99,12 @@ class ServiceBackend(plugins.Plugin, metaclass=ServiceMount):
return None
@classmethod
def get_backends(cls, instance=None, action=None, active=True):
from .models import Route
def get_backends(cls, instance=None, action=None, active=None):
backends = cls.get_plugins()
included = []
if active:
active_backends = Route.objects.filter(is_active=True).values_list('backend', flat=True)
# Filter for instance or action
for backend in backends:
if active and backend.get_name() not in active_backends:
if active is not None and backend.get_name() not in active:
continue
include = True
if instance:
@ -208,5 +205,5 @@ class ServiceController(ServiceBackend):
""" filter controller classes """
backends = super(ServiceController, cls).get_backends()
return [
backend for backend in backends if ServiceController in backend.__mro__
backend for backend in backends if isinstance(backend, ServiceController)
]

View file

@ -141,7 +141,8 @@ def collect(instance, action, **kwargs):
""" collect operations """
operations = kwargs.get('operations', set())
route_cache = kwargs.get('route_cache', {})
for backend_cls in ServiceBackend.get_backends():
active_backends = kwargs.get('active_backends', None)
for backend_cls in ServiceBackend.get_backends(active=active_backends):
# Check if there exists a related instance to be executed for this backend and action
instances = []
if action in backend_cls.actions:

View file

@ -6,13 +6,16 @@ from django.db.models.signals import pre_delete, post_save, m2m_changed
from django.dispatch import receiver
from django.http.response import HttpResponseServerError
from orchestra.utils.python import OrderedSet
from orchestra.utils.python import OrderedSet, import_class
from . import manager, Operation
from . import manager, Operation, settings
from .helpers import message_user
from .models import BackendLog
router = import_class(settings.ORCHESTRATION_ROUTER)
@receiver(post_save, dispatch_uid='orchestration.post_save_collector')
def post_save_collector(sender, *args, **kwargs):
if sender not in [BackendLog, Operation]:
@ -63,6 +66,16 @@ class OperationsMiddleware(object):
return request.route_cache
return {}
@classmethod
def get_active_cache(cls):
""" chache the routes to save sql queries """
if hasattr(cls.thread_locals, 'request'):
request = cls.thread_locals.request
if not hasattr(request, 'active_cache'):
request.active_cache = router.get_active_backends()
return request.active_cache
return router.get_active_backends()
@classmethod
def collect(cls, action, **kwargs):
""" Collects all pending operations derived from model signals """
@ -71,6 +84,7 @@ class OperationsMiddleware(object):
return
kwargs['operations'] = cls.get_pending_operations()
kwargs['route_cache'] = cls.get_route_cache()
kwargs['active_backends'] = cls.get_active_cache()
instance = kwargs.pop('instance')
manager.collect(instance, action, **kwargs)

View file

@ -169,6 +169,10 @@ class Route(models.Model):
servers.append(route.host)
return servers
@classmethod
def get_active_backends(cls):
return cls.objects.filter(is_active=True).values_list('backend', flat=True)
def clean(self):
if not self.match:
self.match = 'True'

View file

@ -33,18 +33,20 @@ class BillsBackend(object):
bill = Invoice.objects.create(account=account, is_open=True)
bills.append(bill)
# Create bill line
billine = bill.lines.create(
quantity = line.metric*line.size
if quantity != 0:
billine = bill.lines.create(
rate=service.nominal_price,
quantity=line.metric*line.size,
verbose_quantity=self.get_verbose_quantity(line),
subtotal=line.subtotal,
tax=service.tax,
description=self.get_line_description(line),
order=line.order,
order_billed_on=line.order.old_billed_on,
order_billed_until=line.order.old_billed_until
)
self.create_sublines(billine, line.discounts)
)
self.create_sublines(billine, line.discounts)
return bills
def format_period(self, ini, end):
@ -61,12 +63,24 @@ class BillsBackend(object):
description = line.order.description
if service.billing_period != service.NEVER:
description += " %s" % self.format_period(line.ini, line.end)
if service.metric and service.billing_period != service.NEVER and service.pricing_period == service.NEVER:
metric = format(line.metric, '.2f').rstrip('0').rstrip('.')
size = format(line.size, '.2f').rstrip('0').rstrip('.')
description += " (%sx%s)" % (metric, size)
return description
def get_verbose_quantity(self, line):
# service = line.order.service
# if service.metric and service.billing_period != service.NEVER and service.pricing_period == service.NEVER:
metric = format(line.metric, '.2f').rstrip('0').rstrip('.')
if metric.endswith('.00'):
metric = metric.split('.')[0]
size = format(line.size, '.2f').rstrip('0').rstrip('.')
if size.endswith('.00'):
size = metric.split('.')[0]
if metric == '1':
return size
if size == '1':
return metric
return "%s&times;%s" % (metric, size)
# return ''
def create_sublines(self, line, discounts):
for discount in discounts:
line.sublines.create(

View file

@ -30,8 +30,6 @@ class Last(Aggregation):
return dataset.none()
def compute_usage(self, dataset):
# FIXME Aggregation of 0s returns None! django bug?
# value = dataset.aggregate(models.Sum('value'))['value__sum']
values = dataset.values_list('value', flat=True)
if values:
return sum(values)

View file

@ -21,7 +21,7 @@ class ServiceMonitor(ServiceBackend):
def get_plugins(cls):
""" filter controller classes """
return [
plugin for plugin in cls.plugins if ServiceMonitor in plugin.__mro__
plugin for plugin in cls.plugins if isinstance(plugin, ServiceMonitor)
]
@classmethod

View file

@ -11,7 +11,7 @@ from django.utils.translation import ugettext, ugettext_lazy as _
from orchestra import plugins
from orchestra.utils.humanize import text2int
from orchestra.utils.python import AttrDict
from orchestra.utils.python import AttrDict, cmp_to_key
from . import settings, helpers
@ -399,6 +399,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
pend = order.billed_until or order.registered_on
pini = pend - rdelta
metric = self.get_register_or_renew_events(porders, pini, pend)
position = min(position, metric)
price = self.get_price(account, metric, position=position, rates=rates)
ini = order.billed_until or order.registered_on
end = order.new_billed_until
@ -445,8 +446,8 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
if self.payment_style == self.PREPAY and self.on_cancel == self.COMPENSATE:
# Get orders pending for compensation
givers = list(related_orders.givers(ini, end))
givers.sort(cmp=helpers.cmp_billed_until_or_registered_on)
orders.sort(cmp=helpers.cmp_billed_until_or_registered_on)
givers = sorted(givers, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on))
orders = sorted(orders, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on))
self.assign_compensations(givers, orders, **options)
rates = self.get_rates(account)
@ -459,7 +460,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
ini -= rdelta
porders = related_orders.pricing_orders(ini, end)
porders = list(set(orders).union(set(porders)))
porders.sort(cmp=helpers.cmp_billed_until_or_registered_on)
porders = sorted(porders, key=cmp_to_key(helpers.cmp_billed_until_or_registered_on))
if concurrent:
# Periodic billing with no pricing period
lines = self.bill_concurrent_orders(account, porders, rates, ini, end)

View file

@ -214,11 +214,13 @@ class Service(models.Model):
ant_counter = counter
accumulated += rate['price'] * rate['quantity']
else:
if metric < position:
raise ValueError("Metric can not be less than the position.")
for rate in rates:
counter += rate['quantity']
if counter >= position:
return decimal.Decimal(str(rate['price']))
def get_rates(self, account, cache=True):
# rates are cached per account
if not cache:

View file

@ -118,12 +118,11 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
super(PHPBackend, self).commit()
def get_fpm_config(self, webapp, context):
merge = settings.WEBAPPS_MERGE_PHP_WEBAPPS
options = webapp.get_options(merge=self.MERGE)
context.update({
'init_vars': webapp.type_instance.get_php_init_vars(merge=self.MERGE),
'max_children': webapp.get_options().get('processes',
settings.WEBAPPS_FPM_DEFAULT_MAX_CHILDREN),
'request_terminate_timeout': webapp.get_options().get('timeout', False),
'max_children': options.get('processes', settings.WEBAPPS_FPM_DEFAULT_MAX_CHILDREN),
'request_terminate_timeout': options.get('timeout', False),
})
context['fpm_listen'] = webapp.type_instance.FPM_LISTEN % context
fpm_config = Template(textwrap.dedent("""\
@ -139,7 +138,7 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
pm.max_requests = {{ max_requests }}
pm.max_children = {{ max_children }}
{% if request_terminate_timeout %}request_terminate_timeout = {{ request_terminate_timeout }}{% endif %}
{% for name, value in init_vars.iteritems %}
{% for name, value in init_vars.items %}
php_admin_value[{{ name | safe }}] = {{ value | safe }}{% endfor %}
"""
))
@ -168,9 +167,10 @@ class PHPBackend(WebAppServiceMixin, ServiceController):
exec %(php_binary_path)s %(php_init_vars)s""") % context
def get_fcgid_cmd_options(self, webapp, context):
options = webapp.get_options(merge=self.MERGE)
maps = {
'MaxProcesses': webapp.get_options().get('processes', None),
'IOTimeout': webapp.get_options().get('timeout', None),
'MaxProcesses': options.get('processes', None),
'IOTimeout': options.get('timeout', None),
}
cmd_options = []
for directive, value in maps.items():

View file

@ -39,7 +39,6 @@ class WordPressBackend(WebAppServiceMixin, ServiceController):
exc('wget http://wordpress.org/latest.tar.gz -O - --no-check-certificate | tar -xzvf - -C %(app_path)s --strip-components=1');
exc('mkdir %(app_path)s/wp-content/uploads');
exc('chmod 750 %(app_path)s/wp-content/uploads');
exc('chown -R %(user)s:%(group)s %(app_path)s');
$config_file = file('%(app_path)s/' . 'wp-config-sample.php');
$secret_keys = file_get_contents('https://api.wordpress.org/secret-key/1.1/salt/');
@ -70,6 +69,8 @@ class WordPressBackend(WebAppServiceMixin, ServiceController):
foreach ( $config_file as $line_num => $line ) {
fwrite($fw, $line);
}
exc('chown -R %(user)s:%(group)s %(app_path)s');
define('WP_CONTENT_DIR', 'wp-content/');
define('WP_LANG_DIR', WP_CONTENT_DIR . '/languages' );
define('WP_USE_THEMES', true);

View file

@ -56,8 +56,17 @@ class WebApp(models.Model):
self.data = apptype.clean_data()
@cached
def get_options(self):
return OrderedDict((opt.name, opt.value) for opt in self.options.all().order_by('name'))
def get_options(self, merge=False):
if merge:
options = OrderedDict()
qs = WebAppOption.objects.filter(webapp__account=self.account, webapp__type=self.type)
for name, value in qs.values_list('name', 'value').order_by('name'):
if name in options:
options[name] = max(options[name], value)
else:
options[name] = value
return options
return OrderedDict(self.options.values_list('name', 'value').order_by('name'))
def get_directive(self):
return self.type_instance.get_directive()

View file

@ -24,6 +24,7 @@ WEBAPPS_PHPFPM_POOL_PATH = getattr(settings, 'WEBAPPS_PHPFPM_POOL_PATH',
WEBAPPS_FCGID_WRAPPER_PATH = getattr(settings, 'WEBAPPS_FCGID_WRAPPER_PATH',
# Inside SuExec Document root
# Make sure all account wrappers are in the same DIR
'/home/httpd/fcgi-bin.d/%(user)s/%(app_name)s-wrapper'
)

View file

@ -192,10 +192,11 @@ class Apache2Backend(ServiceController):
'wrapper_name': os.path.basename(wrapper_path),
})
directives = ''
# This Alias trick is used instead of FcgidWrapper because we don't want to define
# This Action trick is used instead of FcgidWrapper because we don't want to define
# a new fcgid process class each time an app is mounted (num proc limits enforcement).
if 'wrapper_dir' not in context:
# fcgi-bin only needs to be defined once per vhots
# We assume that all account wrapper paths will share the same dir
context['wrapper_dir'] = os.path.dirname(wrapper_path)
directives = textwrap.dedent("""\
Alias /fcgi-bin/ %(wrapper_dir)s/

View file

@ -89,3 +89,24 @@ class CaptureStdout(list):
def __exit__(self, *args):
self.extend(self._stringio.getvalue().splitlines())
sys.stdout = self._stdout
def cmp_to_key(mycmp):
'Convert a cmp= function into a key= function'
class K(object):
def __init__(self, obj, *args):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj) < 0
def __gt__(self, other):
return mycmp(self.obj, other.obj) > 0
def __eq__(self, other):
return mycmp(self.obj, other.obj) == 0
def __le__(self, other):
return mycmp(self.obj, other.obj) <= 0
def __ge__(self, other):
return mycmp(self.obj, other.obj) >= 0
def __ne__(self, other):
return mycmp(self.obj, other.obj) != 0
return K