From 5fa54adf24d625e4185fe0d571cb2a6638cec798 Mon Sep 17 00:00:00 2001 From: Marc Date: Tue, 16 Sep 2014 14:35:00 +0000 Subject: [PATCH] Preliminar implementation of billing machinery --- TODO.md | 1 + orchestra/apps/orders/billing.py | 1 - orchestra/apps/orders/handlers.py | 305 +++++++++++------- orchestra/apps/orders/helpers.py | 81 ++--- orchestra/apps/orders/models.py | 11 +- orchestra/apps/orders/rating.py | 19 +- .../orders/tests/functional_tests/tests.py | 249 +++++++------- orchestra/apps/payments/actions.py | 2 +- orchestra/apps/resources/admin.py | 2 +- orchestra/apps/resources/models.py | 2 +- orchestra/apps/resources/serializers.py | 2 +- orchestra/core/caches.py | 8 +- orchestra/models/queryset.py | 50 +-- 13 files changed, 397 insertions(+), 336 deletions(-) diff --git a/TODO.md b/TODO.md index a318bc48..1f2bc070 100644 --- a/TODO.md +++ b/TODO.md @@ -101,3 +101,4 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon * bill.bad_debt() -> transaction.ABORTED * transaction.ABORTED -> bill.bad_debt - Issue new transaction when current transaction is ABORTED +* underescore *every* private function diff --git a/orchestra/apps/orders/billing.py b/orchestra/apps/orders/billing.py index f9a5c4a0..21d3acb2 100644 --- a/orchestra/apps/orders/billing.py +++ b/orchestra/apps/orders/billing.py @@ -39,7 +39,6 @@ class BillsBackend(object): description=self.get_line_description(line), ) self.create_sublines(billine, line.discounts) - print bills return bills def format_period(self, ini, end): diff --git a/orchestra/apps/orders/handlers.py b/orchestra/apps/orders/handlers.py index 2e358b0f..5c9a7453 100644 --- a/orchestra/apps/orders/handlers.py +++ b/orchestra/apps/orders/handlers.py @@ -1,5 +1,6 @@ import calendar import datetime +import decimal from dateutil import relativedelta from django.contrib.contenttypes.models import ContentType @@ -71,7 +72,7 @@ class ServiceHandler(plugins.Plugin): else: raise NotImplementedError(msg) bp = datetime.datetime(year=date.year, month=date.month, day=day, - tzinfo=timezone.get_current_timezone()) + tzinfo=timezone.get_current_timezone()).date() elif self.billing_period == self.ANUAL: if self.billing_point == self.ON_REGISTER: month = order.registered_on.month @@ -87,7 +88,7 @@ class ServiceHandler(plugins.Plugin): if bp.month >= month: year = bp.year + 1 bp = datetime.datetime(year=year, month=month, day=day, - tzinfo=timezone.get_current_timezone()) + tzinfo=timezone.get_current_timezone()).date() elif self.billing_period == self.NEVER: bp = order.registered_on else: @@ -96,21 +97,23 @@ class ServiceHandler(plugins.Plugin): return order.cancelled_on return bp - def get_pricing_size(self, ini, end): + def get_price_size(self, ini, end): rdelta = relativedelta.relativedelta(end, ini) - if self.get_pricing_period() == self.MONTHLY: - size = rdelta.months + if self.billing_period == self.MONTHLY: + size = rdelta.years * 12 + size += rdelta.months days = calendar.monthrange(end.year, end.month)[1] - size += float(rdelta.days)/days - elif self.get_pricing_period() == self.ANUAL: + size += decimal.Decimal(rdelta.days)/days + elif self.billing_period == self.ANUAL: size = rdelta.years + size += decimal.Decimal(rdelta.months)/12 days = 366 if calendar.isleap(end.year) else 365 - size += float((end-ini).days)/days - elif self.get_pricing_period() == self.NEVER: + size += decimal.Decimal(rdelta.days)/days + elif self.billing_period == self.NEVER: size = 1 else: raise NotImplementedError - return size + return decimal.Decimal(size) def get_pricing_slots(self, ini, end): period = self.get_pricing_period() @@ -131,56 +134,139 @@ class ServiceHandler(plugins.Plugin): yield ini, next ini = next - def get_price_with_orders(self, order, size, ini, end): - porders = self.orders.filter(account=order.account).filter( - Q(cancelled_on__isnull=True) | Q(cancelled_on__gt=ini) - ).filter(registered_on__lt=end).order_by('registered_on') - price = 0 - if self.orders_effect == self.REGISTER_OR_RENEW: - events = helpers.get_register_or_renew_events(porders, order, ini, end) - elif self.orders_effect == self.CONCURRENT: - events = helpers.get_register_or_cancel_events(porders, order, ini, end) - else: - raise NotImplementedError - for metric, position, ratio in events: - price += self.get_price(order, metric, position=position) * size * ratio - return price + def generate_discount(self, line, dtype, price): + line.discounts.append(AttributeDict(**{ + 'type': dtype, + 'total': price, + })) - def get_price_with_metric(self, order, size, ini, end): - metric = order.get_metric(ini, end) - price = self.get_price(order, metric) * size - return price - - def generate_line(self, order, price, size, ini, end): - subtotal = float(self.nominal_price) * size - discounts = [] - if subtotal > price: - discounts.append(AttributeDict(**{ - 'type': 'volume', - 'total': price-subtotal - })) - elif subtotal < price: - raise ValueError("Something is wrong!") - return AttributeDict(**{ + def generate_line(self, order, price, size, ini, end, discounts=[]): + subtotal = self.nominal_price * size + line = AttributeDict(**{ 'order': order, 'subtotal': subtotal, 'size': size, 'ini': ini, 'end': end, - 'discounts': discounts, + 'discounts': [], }) + discounted = 0 + for dtype, dprice in discounts: + self.generate_discount(line, dtype, dprice) + discounted += dprice + subtotal -= discounted + if subtotal > price: + self.generate_discount(line, 'volume', price-subtotal) + return line + + def compensate(self, givers, receivers, commit=True): + compensations = [] + for order in givers: + if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until: + interval = helpers.Interval(order.cancelled_on, order.billed_until, order) + compensations.append[interval] + for order in receivers: + if not order.billed_until or order.billed_until < order.new_billed_until: + # receiver + ini = order.billed_until or order.registered_on + end = order.cancelled_on or datetime.date.max + order_interval = helpers.Interval(ini, order.new_billed_until) + compensations, used_compensations = helpers.compensate(order_interval, compensations) + order._compensations = used_compensations + for comp in used_compensations: + comp.order.new_billed_until = min(comp.order.billed_until, comp.end) + if commit: + for order in givers: + if hasattr(order, 'new_billed_until'): + order.billed_until = order.new_billed_until + order.save() - def _generate_bill_lines(self, orders, account, **options): + def get_register_or_renew_events(self, porders, ini, end): + # TODO count intermediat billing points too + counter = 0 + for order in porders: + bu = getattr(order, 'new_billed_until', order.billed_until) + if bu: + if order.register >= ini and order.register < end: + counter += 1 + if order.register != bu and bu >= ini and bu < end: + counter += 1 + if order.billed_until and order.billed_until != bu: + if order.register != order.billed_until and order.billed_until >= ini and order.billed_until < end: + counter += 1 + return counter + + def bill_concurrent_orders(self, account, porders, rates, ini, end, commit=True): + # Concurrent + # Get pricing orders + priced = {} + for ini, end, orders in helpers.get_chunks(porders, ini, end): + size = self.get_price_size(ini, end) + metric = len(orders) + interval = helpers.Interval(ini=ini, end=end) + for position, order in enumerate(orders): + csize = 0 + compensations = getattr(order, '_compensations', []) + for comp in compensations: + intersect = comp.intersect(interval) + if intersect: + csize += self.get_price_size(intersect.ini, intersect.end) + price = self.get_price(account, metric, position=position, rates=rates) + price = price * size + cprice = price * (size-csize) + if order in prices: + priced[order][0] += price + priced[order][1] += cprice + else: + priced[order] = (price, cprice) + lines = [] + for order, prices in priced.iteritems(): + # Generate lines and discounts from order.nominal_price + price, cprice = prices + if cprice: + discounts = (('compensation', cprice),) + line = self.generate_line(order, price, size, ini, end, discounts=discounts) + lines.append(line) + if commit: + order.billed_until = order.new_billed_until + order.save() + return lines + + def bill_registered_or_renew_events(self, account, porders, rates, ini, end, commit=True): + # Before registration + lines = [] + perido = self.get_pricing_period() + if period == self.MONTHLY: + rdelta = relativedelta.relativedelta(months=1) + elif period == self.ANUAL: + rdelta = relativedelta.relativedelta(years=1) + elif period == self.NEVER: + raise NotImplementedError("Rates with no pricing period?") + ini -= rdelta + for position, order in enumerate(porders): + if hasattr(order, 'new_billed_until'): + cend = order.billed_until or order.registered_on + cini = cend - rdelta + metric = self.get_register_or_renew_events(porders, cini, cend) + size = self.get_price_size(ini, end) + price = self.get_price(account, metric, position=position, rates=rates) + price = price * size + line = self.generate_line(order, price, size, ini, end) + lines.append(line) + if commit: + order.billed_until = order.new_billed_until + order.save() + + def bill_with_orders(self, orders, account, **options): # For the "boundary conditions" just think that: # date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0) # In most cases: # ini >= registered_date, end < registered_date - bp = None lines = [] commit = options.get('commit', True) ini = datetime.date.max - end = datetime.date.ini + end = datetime.date.min # boundary lookup for order in orders: cini = order.registered_on @@ -189,87 +275,42 @@ class ServiceHandler(plugins.Plugin): bp = self.get_billing_point(order, bp=bp, **options) order.new_billed_until = bp ini = min(ini, cini) - end = max(end, bp) # TODO if all bp are the same ... - + end = max(end, bp) + from .models import Order related_orders = Order.objects.filter(service=self.service, account=account) if self.on_cancel == self.COMPENSATE: # Get orders pending for compensation givers = related_orders.filter_givers(ini, end) givers.sort(cmp=helpers.cmp_billed_until_or_registered_on) orders.sort(cmp=helpers.cmp_billed_until_or_registered_on) - self.compensate(givers, orders) + self.compensate(givers, orders, commit=commit) - rates = 'TODO' + rates = self.get_rates(account) if rates: - # Get pricing orders porders = related_orders.filter_pricing_orders(ini, end) - porders = set(orders).union(set(porders)) - for ini, end, orders in self.get_chunks(porders, ini, end): - if self.pricing_period == self.ANUAL: - pass - elif self.pricing_period == self.MONTHLY: - pass - else: - raise NotImplementedError - metric = len(orders) - for position, order in enumerate(orders): - # TODO position +1? - price = self.get_price(order, metric, position=position) - price *= size + porders = list(set(orders).union(set(porders))) + porders.sort(cmp=helpers.cmp_billed_until_or_registered_on) + if self.billing_period != self.NEVER and self.get_pricing_period != self.NEVER: + liens = self.bill_concurrent_orders(account, porders, rates, ini, end, commit=commit) + else: + lines = self.bill_registered_or_renew_events(account, porders, rates, ini, end, commit=commit) else: - pass - - def compensate(self, givers, receivers): - compensations = [] - for order in givers: - if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until: - compensations.append[Interval(order.cancelled_on, order.billed_until, order)] - for order in receivers: - if not order.billed_until or order.billed_until < order.new_billed_until: - # receiver + lines = [] + price = self.nominal_price + # Calculate nominal price + for order in orders: ini = order.billed_until or order.registered_on - end = order.cancelled_on or datetime.date.max - order_interval = helpers.Interval(ini, order.new_billed_until) # TODO beyond interval? - compensations, used_compensations = helpers.compensate(order_interval, compensations) - order._compensations = used_compensations - for comp in used_compensations: - comp.order.billed_until = min(comp.order.billed_until, comp.end) + end = order.new_billed_until + size = self.get_price_size(ini, end) + order.nominal_price = price * size + line = self.generate_line(order, price*size, size, ini, end) + lines.append(line) + if commit: + order.billed_until = order.new_billed_until + order.save() + return lines - def get_chunks(self, porders, ini, end, ix=0): - if ix >= len(porders): - return [[ini, end, []]] - order = porders[ix] - ix += 1 - bu = getattr(order, 'new_billed_until', order.billed_until) - if not bu or bu <= ini or order.registered_on >= end: - return self.get_chunks(porders, ini, end, ix=ix) - result = [] - if order.registered_on < end and order.registered_on > ini: - ro = order.registered_on - result = self.get_chunks(porders, ini, ro, ix=ix) - ini = ro - if bu < end: - result += self.get_chunks(porders, bu, end, ix=ix) - end = bu - chunks = self.get_chunks(porders, ini, end, ix=ix) - for chunk in chunks: - chunk[2].insert(0, order) - result.append(chunk) - return result - - def generate_bill_lines(self, orders, **options): - # For the "boundary conditions" just think that: - # date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0) - # In most cases: - # ini >= registered_date, end < registered_date - - # TODO Perform compensations on cancelled services - if self.on_cancel in (self.COMPENSATE, self.REFOUND): - pass - # TODO compensations with commit=False, fuck commit or just fuck the transaction? - # compensate(orders, **options) - # TODO create discount per compensation - bp = None + def bill_with_metric(self, orders, account, **options): lines = [] commit = options.get('commit', True) for order in orders: @@ -277,18 +318,32 @@ class ServiceHandler(plugins.Plugin): ini = order.billed_until or order.registered_on if bp <= ini: continue - if not self.metric: - # Number of orders metric; bill line per order - size = self.get_pricing_size(ini, bp) - price = self.get_price_with_orders(order, size, ini, bp) - lines.append(self.generate_line(order, price, size, ini, bp)) - else: - # weighted metric; bill line per pricing period - for ini, end in self.get_pricing_slots(ini, bp): - size = self.get_pricing_size(ini, end) - price = self.get_price_with_metric(order, size, ini, end) - lines.append(self.generate_line(order, price, size, ini, end)) - order.billed_until = bp + order.new_billed_until = bp + # weighted metric; bill line per pricing period + prev = None + lines_info = [] + for ini, end in self.get_pricing_slots(ini, bp): + size = self.get_price_size(ini, end) + metric = order.get_metric(ini, end) + price = self.get_price(order, metric) + current = AttributeDict(price=price, size=size, ini=ini, end=end) + if prev and prev.metric == current.metric and prev.end == current.end: + prev.end = current.end + prev.size += current.size + prev.price += current.price + else: + lines_info.append(current) + prev = current + for line in lines_info: + lines.append(self.generate_line(order, price, size, ini, end)) if commit: + order.billed_until = order.new_billed_until order.save() return lines + + def generate_bill_lines(self, orders, account, **options): + if not self.metric: + lines = self.bill_with_orders(orders, account, **options) + else: + lines = self.bill_with_metric(orders, account, **options) + return lines diff --git a/orchestra/apps/orders/helpers.py b/orchestra/apps/orders/helpers.py index ef549e4f..75412cb7 100644 --- a/orchestra/apps/orders/helpers.py +++ b/orchestra/apps/orders/helpers.py @@ -39,57 +39,27 @@ def get_related_objects(origin, max_depth=2): queue.append(new_models) -def get_register_or_cancel_events(porders, order, ini, end): - assert ini <= end, "ini > end" - CANCEL = 'cancel' - REGISTER = 'register' - changes = {} - counter = 0 - for num, porder in enumerate(porders.order_by('registered_on'), start=1): - if porder == order: - position = num - if porder.cancelled_on: - cancel = porder.cancelled_on - if porder.billed_until and porder.cancelled_on < porder.billed_until: - cancel = porder.billed_until - if cancel > ini and cancel < end: - changes.setdefault(cancel, []) - changes[cancel].append((CANCEL, num)) - if porder.registered_on <= ini: - counter += 1 - elif porder.registered_on < end: - changes.setdefault(porder.registered_on, []) - changes[porder.registered_on].append((REGISTER, num)) - pointer = ini - total = float((end-ini).days) - for date in sorted(changes.keys()): - yield counter, position, (date-pointer).days/total - for change, num in changes[date]: - if change is CANCEL: - counter -= 1 - if num < position: - position -= 1 - else: - counter += 1 - pointer = date - yield counter, position, (end-pointer).days/total - - -def get_register_or_renew_events(handler, porders, order, ini, end): - total = float((end-ini).days) - for sini, send in handler.get_pricing_slots(ini, end): - counter = 0 - position = -1 - for porder in porders.order_by('registered_on'): - if porder == order: - position = abs(position) - elif position < 0: - position -= 1 - if porder.registered_on >= sini and porder.registered_on < send: - counter += 1 - elif porder.billed_until > send or porder.cancelled_on > send: - counter += 1 - yield counter, position, (send-sini)/total +def get_chunks(porders, ini, end, ix=0): + if ix >= len(porders): + return [[ini, end, []]] + order = porders[ix] + ix += 1 + bu = getattr(order, 'new_billed_until', order.billed_until) + if not bu or bu <= ini or order.registered_on >= end: + return get_chunks(porders, ini, end, ix=ix) + result = [] + if order.registered_on < end and order.registered_on > ini: + ro = order.registered_on + result = get_chunks(porders, ini, ro, ix=ix) + ini = ro + if bu < end: + result += get_chunks(porders, bu, end, ix=ix) + end = bu + chunks = get_chunks(porders, ini, end, ix=ix) + for chunk in chunks: + chunk[2].insert(0, order) + result.append(chunk) + return result def cmp_billed_until_or_registered_on(a, b): @@ -145,7 +115,7 @@ class Interval(object): if intersection: intersections.append(intersection) return intersections - + def get_intersections(order_intervals, compensations): intersections = [] @@ -158,8 +128,9 @@ def get_intersections(order_intervals, compensations): intersections.sort() return intersections -# Intervals should not overlap + def intersect(compensation, order_intervals): + # Intervals should not overlap compensated = [] not_compensated = [] unused_compensation = [] @@ -167,14 +138,16 @@ def intersect(compensation, order_intervals): compensated.append(compensation.intersect(interval, unused_compensation, not_compensated)) return (compensated, not_compensated, unused_compensation) + def apply_compensation(order, compensation): remaining_order = [] remaining_compensation = [] applied_compensation = compensation.intersect_set(order, remaining_compensation, remaining_order) return applied_compensation, remaining_order, remaining_compensation -# TODO can be optimized + def update_intersections(not_compensated, compensations): + # TODO can be optimized compensation_intervals = [] for __, compensation in compensations: compensation_intervals.append(compensation) diff --git a/orchestra/apps/orders/models.py b/orchestra/apps/orders/models.py index 8aecc648..a4966ac9 100644 --- a/orchestra/apps/orders/models.py +++ b/orchestra/apps/orders/models.py @@ -49,7 +49,7 @@ class RateQuerySet(models.QuerySet): return self.filter( Q(plan__is_default=True) | Q(plan__contracts__account=account) - ).order_by('plan', 'quantity').select_related('plan').distinct() + ).order_by('plan', 'quantity').select_related('plan') class Rate(models.Model): @@ -257,12 +257,13 @@ class Service(models.Model): return self.billing_period return self.pricing_period - def get_price(self, order, metric, rates=None, position=None): + def get_price(self, account, metric, rates=None, position=None): """ if position is provided an specific price for that position is returned, accumulated price is returned otherwise """ - rates = self.get_rates(order.account) + if rates is None: + rates = self.get_rates(account) if not rates: rates = [{ 'quantity': metric, @@ -288,7 +289,7 @@ class Service(models.Model): if counter >= position: return float(rate['price']) - def get_rates(self, account, cache=False): + def get_rates(self, account, cache=True): # rates are cached per account if not cache: return self.rates.by_account(account) @@ -310,7 +311,7 @@ class OrderQuerySet(models.QuerySet): bill_backend = Order.get_bill_backend() qs = self.select_related('account', 'service') commit = options.get('commit', True) - for account, services in qs.group_by('account', 'service'): + for account, services in qs.group_by('account', 'service').iteritems(): bill_lines = [] for service, orders in services: lines = service.handler.generate_bill_lines(orders, account, **options) diff --git a/orchestra/apps/orders/rating.py b/orchestra/apps/orders/rating.py index cc172548..0e9d4e56 100644 --- a/orchestra/apps/orders/rating.py +++ b/orchestra/apps/orders/rating.py @@ -7,14 +7,25 @@ def _compute(rates, metric): value = 0 num = len(rates) accumulated = 0 + barrier = 1 + next_barrier = None end = False ix = 0 steps = [] while ix < num and not end: + fold = 1 + # Multiple contractions + while ix < num-1 and rates[ix] == rates[ix+1]: + ix += 1 + fold += 1 if ix+1 == num: quantity = metric - accumulated + next_barrier = quantity else: quantity = rates[ix+1].quantity - rates[ix].quantity + next_barrier = quantity + if rates[ix+1].price > rates[ix].price: + quantity *= fold if accumulated+quantity > metric: quantity = metric - accumulated end = True @@ -22,9 +33,10 @@ def _compute(rates, metric): steps.append(AttributeDict(**{ 'quantity': quantity, 'price': price, - 'barrier': accumulated+1, + 'barrier': barrier, })) accumulated += quantity + barrier += next_barrier value += quantity*price ix += 1 return value, steps @@ -32,9 +44,10 @@ def _compute(rates, metric): def step_price(rates, metric): # Step price + # TODO allow multiple plans group = [] minimal = (sys.maxint, []) - for plan, rates in rates.group_by('plan'): + for plan, rates in rates.group_by('plan').iteritems(): value, steps = _compute(rates, metric) if plan.is_combinable: group.append(steps) @@ -90,7 +103,7 @@ def match_price(rates, metric): candidates = [] selected = False prev = None - for rate in rates: + for rate in rates.distinct(): if prev and prev.plan != rate.plan: if not selected and prev.quantity <= metric: candidates.append(prev) diff --git a/orchestra/apps/orders/tests/functional_tests/tests.py b/orchestra/apps/orders/tests/functional_tests/tests.py index 4c421182..4a91fa7b 100644 --- a/orchestra/apps/orders/tests/functional_tests/tests.py +++ b/orchestra/apps/orders/tests/functional_tests/tests.py @@ -1,4 +1,6 @@ import datetime +import decimal +import sys from dateutil import relativedelta from django.contrib.contenttypes.models import ContentType @@ -9,7 +11,7 @@ from orchestra.apps.users.models import User from orchestra.utils.tests import BaseTestCase, random_ascii from ... import settings, helpers -from ...models import Service, Order +from ...models import Plan, Service, Order class OrderTests(BaseTestCase): @@ -26,122 +28,114 @@ class OrderTests(BaseTestCase): account.save() return account - def create_service(self): + def create_ftp_service(self): service = Service.objects.create( description="FTP Account", content_type=ContentType.objects.get_for_model(User), match='not user.is_main and user.has_posix()', billing_period=Service.ANUAL, billing_point=Service.FIXED_DATE, -# delayed_billing=Service.NEVER, is_fee=False, metric='', pricing_period=Service.BILLING_PERIOD, rate_algorithm=Service.STEP_PRICE, -# orders_effect=Service.CONCURRENT, on_cancel=Service.DISCOUNT, payment_style=Service.PREPAY, -# trial_period=Service.NEVER, -# refound_period=Service.NEVER, - tax=21, + tax=0, nominal_price=10, ) return service -# def test_ftp_account_1_year_fiexed(self): -# service = self.create_service() -# bp = timezone.now().date() + relativedelta.relativedelta(years=1) -# bills = service.orders.bill(billing_point=bp, fixed_point=True) -# self.assertEqual(20, bills[0].get_total()) - - def create_ftp(self): + def create_ftp(self, account=None): username = '%s_ftp' % random_ascii(10) - account = self.create_account() + if not account: + account = self.create_account() user = User.objects.create_user(username=username, account=account) POSIX = user._meta.get_field_by_name('posix')[0].model POSIX.objects.create(user=user) return user - def atest_get_chunks(self): - service = self.create_service() + def test_get_chunks(self): + service = self.create_ftp_service() handler = service.handler porders = [] now = timezone.now().date() ct = ContentType.objects.get_for_model(User) + account = self.create_account() - ftp = self.create_ftp() + ftp = self.create_ftp(account=account) order = Order.objects.get(content_type=ct, object_id=ftp.pk) porders.append(order) - end = handler.get_billing_point(order).date() - chunks = handler.get_chunks(porders, now, end) + end = handler.get_billing_point(order) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(1, len(chunks)) self.assertIn([now, end, []], chunks) - ftp = self.create_ftp() + ftp = self.create_ftp(account=account) order1 = Order.objects.get(content_type=ct, object_id=ftp.pk) order1.billed_until = now+datetime.timedelta(days=2) porders.append(order1) - chunks = handler.get_chunks(porders, now, end) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(2, len(chunks)) self.assertIn([order1.registered_on, order1.billed_until, [order1]], chunks) self.assertIn([order1.billed_until, end, []], chunks) - - ftp = self.create_ftp() + + ftp = self.create_ftp(account=account) order2 = Order.objects.get(content_type=ct, object_id=ftp.pk) order2.billed_until = now+datetime.timedelta(days=700) porders.append(order2) - chunks = handler.get_chunks(porders, now, end) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(2, len(chunks)) self.assertIn([order.registered_on, order1.billed_until, [order1, order2]], chunks) self.assertIn([order1.billed_until, end, [order2]], chunks) - - ftp = self.create_ftp() + + ftp = self.create_ftp(account=account) order3 = Order.objects.get(content_type=ct, object_id=ftp.pk) order3.billed_until = now+datetime.timedelta(days=700) porders.append(order3) - chunks = handler.get_chunks(porders, now, end) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(2, len(chunks)) self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) self.assertIn([order1.billed_until, end, [order2, order3]], chunks) - ftp = self.create_ftp() + ftp = self.create_ftp(account=account) order4 = Order.objects.get(content_type=ct, object_id=ftp.pk) order4.registered_on = now+datetime.timedelta(days=5) order4.billed_until = now+datetime.timedelta(days=10) porders.append(order4) - chunks = handler.get_chunks(porders, now, end) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(4, len(chunks)) self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks) self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks) self.assertIn([order4.billed_until, end, [order2, order3]], chunks) - - ftp = self.create_ftp() + + ftp = self.create_ftp(account=account) order5 = Order.objects.get(content_type=ct, object_id=ftp.pk) order5.registered_on = now+datetime.timedelta(days=700) order5.billed_until = now+datetime.timedelta(days=780) porders.append(order5) - chunks = handler.get_chunks(porders, now, end) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(4, len(chunks)) self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks) self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks) self.assertIn([order4.billed_until, end, [order2, order3]], chunks) - - ftp = self.create_ftp() + + ftp = self.create_ftp(account=account) order6 = Order.objects.get(content_type=ct, object_id=ftp.pk) order6.registered_on = now-datetime.timedelta(days=780) order6.billed_until = now-datetime.timedelta(days=700) porders.append(order6) - chunks = handler.get_chunks(porders, now, end) + chunks = helpers.get_chunks(porders, now, end) self.assertEqual(4, len(chunks)) self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks) self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks) self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks) self.assertIn([order4.billed_until, end, [order2, order3]], chunks) - def atest_sort_billed_until_or_registered_on(self): - service = self.create_service() + def test_sort_billed_until_or_registered_on(self): + service = self.create_ftp_service() now = timezone.now() order = Order( service=service, @@ -171,7 +165,7 @@ class OrderTests(BaseTestCase): orders = [order3, order, order1, order2, order4, order5, order6] self.assertEqual(orders, sorted(orders, cmp=helpers.cmp_billed_until_or_registered_on)) - def atest_compensation(self): + def test_compensation(self): now = timezone.now() order = Order( description='0', @@ -219,7 +213,7 @@ class OrderTests(BaseTestCase): ]) porders = [order3, order, order1, order2, order4, order5, order6] porders = sorted(porders, cmp=helpers.cmp_billed_until_or_registered_on) - service = self.create_service() + service = self.create_ftp_service() compensations = [] receivers = [] for order in porders: @@ -237,29 +231,22 @@ class OrderTests(BaseTestCase): self.assertEqual(test_line[1], compensation.end) self.assertEqual(test_line[2], compensation.order) - def get_rates(self, account): - rates = self.rates.filter(Q(plan__is_default=True) | Q(plan__contracts__account=account)).order_by('plan', 'quantity').select_related('plan').disctinct() - def test_rates(self): - from ...models import Plan - import sys - from decimal import Decimal - service = self.create_service() - + service = self.create_ftp_service() + account = self.create_account() superplan = Plan.objects.create(name='SUPER', allow_multiple=False, is_combinable=True) service.rates.create(plan=superplan, quantity=1, price=0) service.rates.create(plan=superplan, quantity=3, price=10) service.rates.create(plan=superplan, quantity=4, price=9) service.rates.create(plan=superplan, quantity=10, price=1) - account = self.create_account() account.plans.create(plan=superplan) - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) rates = [ - {'price': Decimal('0.00'), 'quantity': 2}, - {'price': Decimal('10.00'), 'quantity': 1}, - {'price': Decimal('9.00'), 'quantity': 6}, - {'price': Decimal('1.00'), 'quantity': 21} + {'price': decimal.Decimal('0.00'), 'quantity': 2}, + {'price': decimal.Decimal('10.00'), 'quantity': 1}, + {'price': decimal.Decimal('9.00'), 'quantity': 6}, + {'price': decimal.Decimal('1.00'), 'quantity': 21} ] for rate, result in zip(rates, results): self.assertEqual(rate['price'], result.price) @@ -268,44 +255,44 @@ class OrderTests(BaseTestCase): dupeplan = Plan.objects.create(name='DUPE', allow_multiple=True, is_combinable=True) service.rates.create(plan=dupeplan, quantity=1, price=0) service.rates.create(plan=dupeplan, quantity=3, price=9) - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) for rate, result in zip(rates, results): self.assertEqual(rate['price'], result.price) self.assertEqual(rate['quantity'], result.quantity) account.plans.create(plan=dupeplan) - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) rates = [ - {'price': Decimal('0.00'), 'quantity': 4}, - {'price': Decimal('9.00'), 'quantity': 5}, - {'price': Decimal('1.00'), 'quantity': 21}, + {'price': decimal.Decimal('0.00'), 'quantity': 4}, + {'price': decimal.Decimal('9.00'), 'quantity': 5}, + {'price': decimal.Decimal('1.00'), 'quantity': 21}, ] for rate, result in zip(rates, results): self.assertEqual(rate['price'], result.price) self.assertEqual(rate['quantity'], result.quantity) - hyperplan = Plan.objects.create(name='HYPER', allow_multiple=True, is_combinable=False) + hyperplan = Plan.objects.create(name='HYPER', allow_multiple=False, is_combinable=False) service.rates.create(plan=hyperplan, quantity=1, price=0) service.rates.create(plan=hyperplan, quantity=20, price=5) account.plans.create(plan=hyperplan) - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) rates = [ - {'price': Decimal('0.00'), 'quantity': 19}, - {'price': Decimal('5.00'), 'quantity': 11} + {'price': decimal.Decimal('0.00'), 'quantity': 19}, + {'price': decimal.Decimal('5.00'), 'quantity': 11} ] for rate, result in zip(rates, results): self.assertEqual(rate['price'], result.price) self.assertEqual(rate['quantity'], result.quantity) hyperplan.is_combinable = True hyperplan.save() - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) rates = [ - {'price': Decimal('0.00'), 'quantity': 23}, - {'price': Decimal('1.00'), 'quantity': 7} + {'price': decimal.Decimal('0.00'), 'quantity': 23}, + {'price': decimal.Decimal('1.00'), 'quantity': 7} ] for rate, result in zip(rates, results): self.assertEqual(rate['price'], result.price) @@ -313,67 +300,99 @@ class OrderTests(BaseTestCase): service.rate_algorithm = service.MATCH_PRICE service.save() - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) self.assertEqual(1, len(results)) - self.assertEqual(Decimal('1.00'), results[0].price) + self.assertEqual(decimal.Decimal('1.00'), results[0].price) self.assertEqual(30, results[0].quantity) hyperplan.delete() - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 8) self.assertEqual(1, len(results)) - self.assertEqual(Decimal('9.00'), results[0].price) + self.assertEqual(decimal.Decimal('9.00'), results[0].price) self.assertEqual(8, results[0].quantity) superplan.delete() - results = service.get_rates(account) + results = service.get_rates(account, cache=False) results = service.rate_method(results, 30) self.assertEqual(1, len(results)) - self.assertEqual(Decimal('9.00'), results[0].price) + self.assertEqual(decimal.Decimal('9.00'), results[0].price) self.assertEqual(30, results[0].quantity) + + def test_rates_allow_multiple(self): + service = self.create_ftp_service() + account = self.create_account() + dupeplan = Plan.objects.create(name='DUPE', allow_multiple=True, is_combinable=True) + account.plans.create(plan=dupeplan) + service.rates.create(plan=dupeplan, quantity=1, price=0) + service.rates.create(plan=dupeplan, quantity=3, price=9) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 2}, + {'price': decimal.Decimal('9.00'), 'quantity': 28}, + ] + for rate, result in zip(rates, results): + self.assertEqual(rate['price'], result.price) + self.assertEqual(rate['quantity'], result.quantity) -# def test_ftp_account_1_year_fiexed(self): -# service = self.create_service() -# now = timezone.now().date()etb -# month = settings.ORDERS_SERVICE_ANUAL_BILLING_MONTH -# ini = datetime.datetime(year=now.year, month=month, -# day=1, tzinfo=timezone.get_current_timezone()) -# order = service.orders.all()[0] -# order.registered_on = ini -# order.save() -# bp = ini -# bills = service.orders.bill(billing_point=bp, fixed_point=False, commit=False) -# print bills[0][1][0].subtotal -# print bills -# bp = ini + relativedelta.relativedelta(months=12) -# bills = service.orders.bill(billing_point=bp, fixed_point=False, commit=False) -# print bills[0][1][0].subtotal -# print bills -# def test_ftp_account_2_year_fiexed(self): -# service = self.create_service() -# bp = timezone.now().date() + relativedelta.relativedelta(years=2) -# bills = service.orders.bill(billing_point=bp, fixed_point=True) -# self.assertEqual(40, bills[0].get_total()) -# -# def test_ftp_account_6_month_fixed(self): -# service = self.create_service() -# bp = timezone.now().date() + relativedelta.relativedelta(months=6) -# bills = service.orders.bill(billing_point=bp, fixed_point=True) -# self.assertEqual(6, bills[0].get_total()) -# -# def test_ftp_account_next_billing_point(self): -# service = self.create_service() -# now = timezone.now().date() -# bp_month = settings.ORDERS_SERVICE_ANUAL_BILLING_MONTH -# if date.month > bp_month: -# bp = datetime.datetime(year=now.year+1, month=bp_month, -# day=1, tzinfo=timezone.get_current_timezone()) -# else: -# bp = datetime.datetime(year=now.year, month=bp_month, -# day=1, tzinfo=timezone.get_current_timezone()) -# -# days = (bp - now).days -# bills = service.orders.bill(billing_point=bp, fixed_point=False) -# self.assertEqual(40, bills[0].get_total()) + account.plans.create(plan=dupeplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 4}, + {'price': decimal.Decimal('9.00'), 'quantity': 26}, + ] + for rate, result in zip(rates, results): + self.assertEqual(rate['price'], result.price) + self.assertEqual(rate['quantity'], result.quantity) + account.plans.create(plan=dupeplan) + results = service.get_rates(account, cache=False) + results = service.rate_method(results, 30) + rates = [ + {'price': decimal.Decimal('0.00'), 'quantity': 6}, + {'price': decimal.Decimal('9.00'), 'quantity': 24}, + ] + for rate, result in zip(rates, results): + self.assertEqual(rate['price'], result.price) + self.assertEqual(rate['quantity'], result.quantity) + + def test_ftp_account_1_year_fiexed(self): + service = self.create_ftp_service() + user = self.create_ftp() + bp = timezone.now().date() + relativedelta.relativedelta(years=1) + bills = service.orders.bill(billing_point=bp, fixed_point=True) + self.assertEqual(10, bills[0].get_total()) + + def test_ftp_account_2_year_fiexed(self): + service = self.create_ftp_service() + user = self.create_ftp() + bp = timezone.now().date() + relativedelta.relativedelta(years=2) + bills = service.orders.bill(billing_point=bp, fixed_point=True) + self.assertEqual(20, bills[0].get_total()) + + def test_ftp_account_6_month_fixed(self): + service = self.create_ftp_service() + self.create_ftp() + bp = timezone.now().date() + relativedelta.relativedelta(months=6) + bills = service.orders.bill(billing_point=bp, fixed_point=True) + self.assertEqual(5, bills[0].get_total()) + + def test_ftp_account_next_billing_point(self): + service = self.create_ftp_service() + self.create_ftp() + now = timezone.now() + bp_month = settings.ORDERS_SERVICE_ANUAL_BILLING_MONTH + if now.month > bp_month: + bp = datetime.datetime(year=now.year+1, month=bp_month, + day=1, tzinfo=timezone.get_current_timezone()) + else: + bp = datetime.datetime(year=now.year, month=bp_month, + day=1, tzinfo=timezone.get_current_timezone()) + bills = service.orders.bill(billing_point=now, fixed_point=False) + size = decimal.Decimal((bp - now).days)/365 + error = decimal.Decimal(0.05) + self.assertGreater(10*size+error*(10*size), bills[0].get_total()) + self.assertLess(10*size-error*(10*size), bills[0].get_total()) diff --git a/orchestra/apps/payments/actions.py b/orchestra/apps/payments/actions.py index 365714bc..d696b843 100644 --- a/orchestra/apps/payments/actions.py +++ b/orchestra/apps/payments/actions.py @@ -11,7 +11,7 @@ def process_transactions(modeladmin, request, queryset): msg = _("Selected transactions must be on '{state}' state") messages.error(request, msg.format(state=Transaction.WAITTING_PROCESSING)) return - for method, transactions in queryset.group_by('source__method'): + for method, transactions in queryset.group_by('source__method').iteritems(): if method is not None: method = PaymentMethod.get_plugin(method) procs = method.process(transactions) diff --git a/orchestra/apps/resources/admin.py b/orchestra/apps/resources/admin.py index 84b6a747..2914d265 100644 --- a/orchestra/apps/resources/admin.py +++ b/orchestra/apps/resources/admin.py @@ -137,7 +137,7 @@ def resource_inline_factory(resources): if not running_syncdb(): # not run during syncdb - for ct, resources in Resource.objects.group_by('content_type'): + for ct, resources in Resource.objects.group_by('content_type').iteritems(): inline = resource_inline_factory(resources) model = ct.model_class() insertattr(model, 'inlines', inline) diff --git a/orchestra/apps/resources/models.py b/orchestra/apps/resources/models.py index 2e674198..49f224a5 100644 --- a/orchestra/apps/resources/models.py +++ b/orchestra/apps/resources/models.py @@ -178,7 +178,7 @@ def create_resource_relation(): return self relation = GenericRelation('resources.ResourceData') - for ct, resources in Resource.objects.group_by('content_type'): + for ct, resources in Resource.objects.group_by('content_type').iteritems(): model = ct.model_class() model.add_to_class('resource_set', relation) model.resources = ResourceHandler() diff --git a/orchestra/apps/resources/serializers.py b/orchestra/apps/resources/serializers.py index 5890ece2..64fbaa4d 100644 --- a/orchestra/apps/resources/serializers.py +++ b/orchestra/apps/resources/serializers.py @@ -26,7 +26,7 @@ class ResourceSerializer(serializers.ModelSerializer): if not running_syncdb(): # TODO why this is even loaded during syncdb? # Create nested serializers on target models - for ct, resources in Resource.objects.group_by('content_type'): + for ct, resources in Resource.objects.group_by('content_type').iteritems(): model = ct.model_class() try: router.insert(model, 'resources', ResourceSerializer, required=False, many=True, source='resource_set') diff --git a/orchestra/core/caches.py b/orchestra/core/caches.py index 478d7dd9..4550d827 100644 --- a/orchestra/core/caches.py +++ b/orchestra/core/caches.py @@ -1,5 +1,6 @@ from threading import currentThread +from django.core.cache.backends.dummy import DummyCache from django.core.cache.backends.locmem import LocMemCache @@ -16,14 +17,12 @@ class RequestCache(LocMemCache): def get_request_cache(): """ Returns per-request cache when running RequestCacheMiddleware otherwise a - new LocMemCache instance (when running periodic tasks or shell) + DummyCache instance (when running periodic tasks, tests or shell) """ try: return _request_cache[currentThread()] except KeyError: - cache = RequestCache() - _request_cache[currentThread()] = cache - return cache + return DummyCache('dummy', {}) class RequestCacheMiddleware(object): @@ -33,7 +32,6 @@ class RequestCacheMiddleware(object): cache.clear() def process_response(self, request, response): - # TODO not sure if this actually saves memory, remove otherwise if currentThread() in _request_cache: _request_cache[currentThread()].clear() return response diff --git a/orchestra/models/queryset.py b/orchestra/models/queryset.py index 812b7dd5..aff7a77c 100644 --- a/orchestra/models/queryset.py +++ b/orchestra/models/queryset.py @@ -1,28 +1,30 @@ +from collections import OrderedDict + from .utils import get_field_value -def group_by(qset, *fields, **kwargs): - """ group_by iterator with support for multiple nested fields """ - ix = kwargs.get('ix', 0) - if ix is 0: - qset = qset.order_by(*fields) - group = [] - first = True +def group_by(qset, *fields): + """ 100% in python in order to preserve original order_by """ + first = OrderedDict() + num = len(fields) for obj in qset: - try: - current = get_field_value(obj, fields[ix]) - except AttributeError: - # Intermediary relation does not exists - current = None - if first or current == previous: - group.append(obj) - else: - if ix < len(fields)-1: - group = group_by(group, *fields, ix=ix+1) - yield previous, group - group = [obj] - previous = current - first = False - if ix < len(fields)-1: - group = group_by(group, *fields, ix=ix+1) - yield previous, group + ix = 0 + group = first + while ix < num: + try: + current = get_field_value(obj, fields[ix]) + except AttributeError: + # Intermediary relation does not exists + current = None + if ix < num-1: + try: + group = group[current] + except KeyError: + group[current] = OrderedDict() + else: + try: + group[current].append(obj) + except KeyError: + group[current] = [obj] + ix += 1 + return first