Preliminar implementation of billing machinery

TODO.md

@@ -101,3 +101,4 @@ at + clock time, midnight, noon- At 3:30 p.m., At 4:01, At noon
     * bill.bad_debt() -> transaction.ABORTED
     * transaction.ABORTED -> bill.bad_debt
     - Issue new transaction when current transaction is ABORTED
+* underescore *every* private function

orchestra/apps/orders/billing.py

@@ -39,7 +39,6 @@ class BillsBackend(object):
                     description=self.get_line_description(line),
             )
             self.create_sublines(billine, line.discounts)
-        print bills
         return bills
     
     def format_period(self, ini, end):

orchestra/apps/orders/handlers.py

@@ -1,5 +1,6 @@
 import calendar
 import datetime
+import decimal
 
 from dateutil import relativedelta
 from django.contrib.contenttypes.models import ContentType
@@ -71,7 +72,7 @@ class ServiceHandler(plugins.Plugin):
                     else:
                         raise NotImplementedError(msg)
                     bp = datetime.datetime(year=date.year, month=date.month, day=day,
-                        tzinfo=timezone.get_current_timezone())
+                        tzinfo=timezone.get_current_timezone()).date()
                 elif self.billing_period == self.ANUAL:
                     if self.billing_point == self.ON_REGISTER:
                         month = order.registered_on.month
@@ -87,7 +88,7 @@ class ServiceHandler(plugins.Plugin):
                     if bp.month >= month:
                         year = bp.year + 1
                     bp = datetime.datetime(year=year, month=month, day=day,
-                        tzinfo=timezone.get_current_timezone())
+                        tzinfo=timezone.get_current_timezone()).date()
                 elif self.billing_period == self.NEVER:
                     bp = order.registered_on
                 else:
@@ -96,21 +97,23 @@ class ServiceHandler(plugins.Plugin):
             return order.cancelled_on
         return bp
     
-    def get_pricing_size(self, ini, end):
+    def get_price_size(self, ini, end):
         rdelta = relativedelta.relativedelta(end, ini)
-        if self.get_pricing_period() == self.MONTHLY:
+        if self.billing_period == self.MONTHLY:
-            size = rdelta.months
+            size = rdelta.years * 12
+            size += rdelta.months
             days = calendar.monthrange(end.year, end.month)[1]
-            size += float(rdelta.days)/days
+            size += decimal.Decimal(rdelta.days)/days
-        elif self.get_pricing_period() == self.ANUAL:
+        elif self.billing_period == self.ANUAL:
             size = rdelta.years
+            size += decimal.Decimal(rdelta.months)/12
             days = 366 if calendar.isleap(end.year) else 365
-            size += float((end-ini).days)/days
+            size += decimal.Decimal(rdelta.days)/days
-        elif self.get_pricing_period() == self.NEVER:
+        elif self.billing_period == self.NEVER:
             size = 1
         else:
             raise NotImplementedError
-        return size
+        return decimal.Decimal(size)
     
     def get_pricing_slots(self, ini, end):
         period = self.get_pricing_period()
@@ -131,56 +134,139 @@ class ServiceHandler(plugins.Plugin):
             yield ini, next
             ini = next
     
-    def get_price_with_orders(self, order, size, ini, end):
+    def generate_discount(self, line, dtype, price):
-        porders = self.orders.filter(account=order.account).filter(
+        line.discounts.append(AttributeDict(**{
-            Q(cancelled_on__isnull=True) | Q(cancelled_on__gt=ini)
+            'type': dtype,
-            ).filter(registered_on__lt=end).order_by('registered_on')
+            'total': price,
-        price = 0
-        if self.orders_effect == self.REGISTER_OR_RENEW:
-            events = helpers.get_register_or_renew_events(porders, order, ini, end)
-        elif self.orders_effect == self.CONCURRENT:
-            events = helpers.get_register_or_cancel_events(porders, order, ini, end)
-        else:
-            raise NotImplementedError
-        for metric, position, ratio in events:
-            price += self.get_price(order, metric, position=position) * size * ratio
-        return price
-    
-    def get_price_with_metric(self, order, size, ini, end):
-        metric = order.get_metric(ini, end)
-        price = self.get_price(order, metric) * size
-        return price
-    
-    def generate_line(self, order, price, size, ini, end):
-        subtotal = float(self.nominal_price) * size
-        discounts = []
-        if subtotal > price:
-            discounts.append(AttributeDict(**{
-                'type': 'volume',
-                'total': price-subtotal
         }))
-        elif subtotal < price:
+    
-            raise ValueError("Something is wrong!")
+    def generate_line(self, order, price, size, ini, end, discounts=[]):
-        return AttributeDict(**{
+        subtotal = self.nominal_price * size
+        line = AttributeDict(**{
             'order': order,
             'subtotal': subtotal,
             'size': size,
             'ini': ini,
             'end': end,
-            'discounts': discounts,
+            'discounts': [],
         })
+        discounted = 0
+        for dtype, dprice in discounts:
+            self.generate_discount(line, dtype, dprice)
+            discounted += dprice
+        subtotal -= discounted
+        if subtotal > price:
+            self.generate_discount(line, 'volume', price-subtotal)
+        return line
 
-    def _generate_bill_lines(self, orders, account, **options):
+    def compensate(self, givers, receivers, commit=True):
+        compensations = []
+        for order in givers:
+            if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until:
+                interval = helpers.Interval(order.cancelled_on, order.billed_until, order)
+                compensations.append[interval]
+        for order in receivers:
+            if not order.billed_until or order.billed_until < order.new_billed_until:
+                # receiver
+                ini = order.billed_until or order.registered_on
+                end = order.cancelled_on or datetime.date.max
+                order_interval = helpers.Interval(ini, order.new_billed_until)
+                compensations, used_compensations = helpers.compensate(order_interval, compensations)
+                order._compensations = used_compensations
+                for comp in used_compensations:
+                    comp.order.new_billed_until = min(comp.order.billed_until, comp.end)
+        if commit:
+            for order in givers:
+                if hasattr(order, 'new_billed_until'):
+                    order.billed_until = order.new_billed_until
+                    order.save()
+    
+    def get_register_or_renew_events(self, porders, ini, end):
+        # TODO count intermediat billing points too
+        counter = 0
+        for order in porders:
+            bu = getattr(order, 'new_billed_until', order.billed_until)
+            if bu:
+                if order.register >= ini and order.register < end:
+                    counter += 1
+                if order.register != bu and bu >= ini and bu < end:
+                    counter += 1
+                if order.billed_until and order.billed_until != bu:
+                    if order.register != order.billed_until and order.billed_until >= ini and order.billed_until < end:
+                        counter += 1
+        return counter
+    
+    def bill_concurrent_orders(self, account, porders, rates, ini, end, commit=True):
+        # Concurrent
+        # Get pricing orders
+        priced = {}
+        for ini, end, orders in helpers.get_chunks(porders, ini, end):
+            size = self.get_price_size(ini, end)
+            metric = len(orders)
+            interval = helpers.Interval(ini=ini, end=end)
+            for position, order in enumerate(orders):
+                csize = 0
+                compensations = getattr(order, '_compensations', [])
+                for comp in compensations:
+                    intersect = comp.intersect(interval)
+                    if intersect:
+                        csize += self.get_price_size(intersect.ini, intersect.end)
+                price = self.get_price(account, metric, position=position, rates=rates)
+                price = price * size
+                cprice = price * (size-csize)
+                if order in prices:
+                    priced[order][0] += price
+                    priced[order][1] += cprice
+                else:
+                    priced[order] = (price, cprice)
+        lines = []
+        for order, prices in priced.iteritems():
+            # Generate lines and discounts from order.nominal_price
+            price, cprice = prices
+            if cprice:
+                discounts = (('compensation', cprice),)
+            line = self.generate_line(order, price, size, ini, end, discounts=discounts)
+            lines.append(line)
+            if commit:
+                order.billed_until = order.new_billed_until
+                order.save()
+        return lines
+    
+    def bill_registered_or_renew_events(self, account, porders, rates, ini, end, commit=True):
+        # Before registration
+        lines = []
+        perido = self.get_pricing_period()
+        if period == self.MONTHLY:
+            rdelta = relativedelta.relativedelta(months=1)
+        elif period == self.ANUAL:
+            rdelta = relativedelta.relativedelta(years=1)
+        elif period == self.NEVER:
+            raise NotImplementedError("Rates with no pricing period?")
+        ini -= rdelta
+        for position, order in enumerate(porders):
+            if hasattr(order, 'new_billed_until'):
+                cend = order.billed_until or order.registered_on
+                cini = cend - rdelta
+                metric = self.get_register_or_renew_events(porders, cini, cend)
+                size = self.get_price_size(ini, end)
+                price = self.get_price(account, metric, position=position, rates=rates)
+                price = price * size
+                line = self.generate_line(order, price, size, ini, end)
+                lines.append(line)
+                if commit:
+                    order.billed_until = order.new_billed_until
+                    order.save()
+    
+    def bill_with_orders(self, orders, account, **options):
         # For the "boundary conditions" just think that:
         #   date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0)
         #   In most cases:
         #       ini >= registered_date, end < registered_date
-        
         bp = None
         lines = []
         commit = options.get('commit', True)
         ini = datetime.date.max
-        end = datetime.date.ini
+        end = datetime.date.min
         # boundary lookup
         for order in orders:
             cini = order.registered_on
@@ -189,87 +275,42 @@ class ServiceHandler(plugins.Plugin):
             bp = self.get_billing_point(order, bp=bp, **options)
             order.new_billed_until = bp
             ini = min(ini, cini)
-            end = max(end, bp) # TODO if all bp are the same ...
+            end = max(end, bp)
-        
+        from .models import Order
         related_orders = Order.objects.filter(service=self.service, account=account)
         if self.on_cancel == self.COMPENSATE:
             # Get orders pending for compensation
             givers = related_orders.filter_givers(ini, end)
             givers.sort(cmp=helpers.cmp_billed_until_or_registered_on)
             orders.sort(cmp=helpers.cmp_billed_until_or_registered_on)
-            self.compensate(givers, orders)
+            self.compensate(givers, orders, commit=commit)
         
-        rates = 'TODO'
+        rates = self.get_rates(account)
         if rates:
-            # Get pricing orders
             porders = related_orders.filter_pricing_orders(ini, end)
-            porders = set(orders).union(set(porders))
+            porders = list(set(orders).union(set(porders)))
-            for ini, end, orders in self.get_chunks(porders, ini, end):
+            porders.sort(cmp=helpers.cmp_billed_until_or_registered_on)
-                if self.pricing_period == self.ANUAL:
+            if self.billing_period != self.NEVER and self.get_pricing_period != self.NEVER:
-                    pass
+                liens = self.bill_concurrent_orders(account, porders, rates, ini, end, commit=commit)
-                elif self.pricing_period == self.MONTHLY:
-                    pass
             else:
-                    raise NotImplementedError
+                lines = self.bill_registered_or_renew_events(account, porders, rates, ini, end, commit=commit)
-                metric = len(orders)
-                for position, order in enumerate(orders):
-                    # TODO position +1?
-                    price = self.get_price(order, metric, position=position)
-                    price *= size
         else:
-            pass
+            lines = []
-    
+            price = self.nominal_price
-    def compensate(self, givers, receivers):
+            # Calculate nominal price
-        compensations = []
+            for order in orders:
-        for order in givers:
-            if order.billed_until and order.cancelled_on and order.cancelled_on < order.billed_until:
-                compensations.append[Interval(order.cancelled_on, order.billed_until, order)]
-        for order in receivers:
-            if not order.billed_until or order.billed_until < order.new_billed_until:
-                # receiver
                 ini = order.billed_until or order.registered_on
-                end = order.cancelled_on or datetime.date.max
+                end = order.new_billed_until
-                order_interval = helpers.Interval(ini, order.new_billed_until) # TODO beyond interval?
+                size = self.get_price_size(ini, end)
-                compensations, used_compensations = helpers.compensate(order_interval, compensations)
+                order.nominal_price = price * size
-                order._compensations = used_compensations
+                line = self.generate_line(order, price*size, size, ini, end)
-                for comp in used_compensations:
+                lines.append(line)
-                    comp.order.billed_until = min(comp.order.billed_until, comp.end)
+                if commit:
+                    order.billed_until = order.new_billed_until
+                    order.save()
+        return lines
     
-    def get_chunks(self, porders, ini, end, ix=0):
+    def bill_with_metric(self, orders, account, **options):
-        if ix >= len(porders):
-            return [[ini, end, []]]
-        order = porders[ix]
-        ix += 1
-        bu = getattr(order, 'new_billed_until', order.billed_until)
-        if not bu or bu <= ini or order.registered_on >= end:
-            return self.get_chunks(porders, ini, end, ix=ix)
-        result = []
-        if order.registered_on < end and order.registered_on > ini:
-            ro = order.registered_on
-            result = self.get_chunks(porders, ini, ro, ix=ix)
-            ini = ro
-        if bu < end:
-            result += self.get_chunks(porders, bu, end, ix=ix)
-            end = bu
-        chunks = self.get_chunks(porders, ini, end, ix=ix)
-        for chunk in chunks:
-            chunk[2].insert(0, order)
-            result.append(chunk)
-        return result
-    
-    def generate_bill_lines(self, orders, **options):
-        # For the "boundary conditions" just think that:
-        #   date(2011, 1, 1) is equivalent to datetime(2011, 1, 1, 0, 0, 0)
-        #   In most cases:
-        #       ini >= registered_date, end < registered_date
-        
-        # TODO Perform compensations on cancelled services
-        if self.on_cancel in (self.COMPENSATE, self.REFOUND):
-            pass
-            # TODO compensations with commit=False, fuck commit or just fuck the transaction?
-            # compensate(orders, **options)
-            # TODO create discount per compensation
-        bp = None
         lines = []
         commit = options.get('commit', True)
         for order in orders:
@@ -277,18 +318,32 @@ class ServiceHandler(plugins.Plugin):
             ini = order.billed_until or order.registered_on
             if bp <= ini:
                 continue
-            if not self.metric:
+            order.new_billed_until = bp
-                # Number of orders metric; bill line per order
-                size = self.get_pricing_size(ini, bp)
-                price = self.get_price_with_orders(order, size, ini, bp)
-                lines.append(self.generate_line(order, price, size, ini, bp))
-            else:
             # weighted metric; bill line per pricing period
+            prev = None
+            lines_info = []
             for ini, end in self.get_pricing_slots(ini, bp):
-                    size = self.get_pricing_size(ini, end)
+                size = self.get_price_size(ini, end)
-                    price = self.get_price_with_metric(order, size, ini, end)
+                metric = order.get_metric(ini, end)
+                price = self.get_price(order, metric)
+                current = AttributeDict(price=price, size=size, ini=ini, end=end)
+                if prev and prev.metric == current.metric and prev.end == current.end:
+                    prev.end = current.end
+                    prev.size += current.size
+                    prev.price += current.price
+                else:
+                    lines_info.append(current)
+                prev = current
+            for line in lines_info:
                 lines.append(self.generate_line(order, price, size, ini, end))
-            order.billed_until = bp
             if commit:
+                order.billed_until = order.new_billed_until
                 order.save()
         return lines
+    
+    def generate_bill_lines(self, orders, account, **options):
+        if not self.metric:
+            lines = self.bill_with_orders(orders, account, **options)
+        else:
+            lines = self.bill_with_metric(orders, account, **options)
+        return lines

orchestra/apps/orders/helpers.py

@@ -39,57 +39,27 @@ def get_related_objects(origin, max_depth=2):
                 queue.append(new_models)
 
 
-def get_register_or_cancel_events(porders, order, ini, end):
+def get_chunks(porders, ini, end, ix=0):
-    assert ini <= end, "ini > end"
+    if ix >= len(porders):
-    CANCEL = 'cancel'
+        return [[ini, end, []]]
-    REGISTER = 'register'
+    order = porders[ix]
-    changes = {}
+    ix += 1
-    counter = 0
+    bu = getattr(order, 'new_billed_until', order.billed_until)
-    for num, porder in enumerate(porders.order_by('registered_on'), start=1):
+    if not bu or bu <= ini or order.registered_on >= end:
-        if porder == order:
+        return get_chunks(porders, ini, end, ix=ix)
-            position = num
+    result = []
-        if porder.cancelled_on:
+    if order.registered_on < end and order.registered_on > ini:
-            cancel = porder.cancelled_on
+        ro = order.registered_on
-            if porder.billed_until and porder.cancelled_on < porder.billed_until:
+        result = get_chunks(porders, ini, ro, ix=ix)
-                cancel = porder.billed_until
+        ini = ro
-            if cancel > ini and cancel < end:
+    if bu < end:
-                changes.setdefault(cancel, [])
+        result += get_chunks(porders, bu, end, ix=ix)
-                changes[cancel].append((CANCEL, num))
+        end = bu
-        if porder.registered_on <= ini:
+    chunks = get_chunks(porders, ini, end, ix=ix)
-            counter += 1
+    for chunk in chunks:
-        elif porder.registered_on < end:
+        chunk[2].insert(0, order)
-            changes.setdefault(porder.registered_on, [])
+        result.append(chunk)
-            changes[porder.registered_on].append((REGISTER, num))
+    return result
-    pointer = ini
-    total = float((end-ini).days)
-    for date in sorted(changes.keys()):
-        yield counter, position, (date-pointer).days/total
-        for change, num in changes[date]:
-            if change is CANCEL:
-                counter -= 1
-                if num < position:
-                    position -= 1
-            else:
-                counter += 1
-        pointer = date
-    yield counter, position, (end-pointer).days/total
-
-
-def get_register_or_renew_events(handler, porders, order, ini, end):
-    total = float((end-ini).days)
-    for sini, send in handler.get_pricing_slots(ini, end):
-        counter = 0
-        position = -1
-        for porder in porders.order_by('registered_on'):
-            if porder == order:
-                position = abs(position)
-            elif position < 0:
-                position -= 1
-            if porder.registered_on >= sini and porder.registered_on < send:
-                counter += 1
-            elif porder.billed_until > send or porder.cancelled_on > send:
-                counter += 1
-        yield counter, position, (send-sini)/total
 
 
 def cmp_billed_until_or_registered_on(a, b):
@@ -158,8 +128,9 @@ def get_intersections(order_intervals, compensations):
     intersections.sort()
     return intersections
 
-# Intervals should not overlap
+
 def intersect(compensation, order_intervals):
+    # Intervals should not overlap
     compensated = []
     not_compensated = []
     unused_compensation = []
@@ -167,14 +138,16 @@ def intersect(compensation, order_intervals):
         compensated.append(compensation.intersect(interval, unused_compensation, not_compensated))
     return (compensated, not_compensated, unused_compensation)
 
+
 def apply_compensation(order, compensation):
     remaining_order = []
     remaining_compensation = []
     applied_compensation = compensation.intersect_set(order, remaining_compensation, remaining_order)
     return applied_compensation, remaining_order, remaining_compensation
 
-# TODO can be optimized
+
 def update_intersections(not_compensated, compensations):
+    # TODO can be optimized
     compensation_intervals = []
     for __, compensation in compensations:
         compensation_intervals.append(compensation)

orchestra/apps/orders/models.py

@@ -49,7 +49,7 @@ class RateQuerySet(models.QuerySet):
         return self.filter(
             Q(plan__is_default=True) |
             Q(plan__contracts__account=account)
-        ).order_by('plan', 'quantity').select_related('plan').distinct()
+        ).order_by('plan', 'quantity').select_related('plan')
 
 
 class Rate(models.Model):
@@ -257,12 +257,13 @@ class Service(models.Model):
             return self.billing_period
         return self.pricing_period
     
-    def get_price(self, order, metric, rates=None, position=None):
+    def get_price(self, account, metric, rates=None, position=None):
         """
         if position is provided an specific price for that position is returned,
         accumulated price is returned otherwise
         """
-        rates = self.get_rates(order.account)
+        if rates is None:
+            rates = self.get_rates(account)
         if not rates:
             rates = [{
                 'quantity': metric,
@@ -288,7 +289,7 @@ class Service(models.Model):
                 if counter >= position:
                     return float(rate['price'])
     
-    def get_rates(self, account, cache=False):
+    def get_rates(self, account, cache=True):
         # rates are cached per account
         if not cache:
             return self.rates.by_account(account)
@@ -310,7 +311,7 @@ class OrderQuerySet(models.QuerySet):
         bill_backend = Order.get_bill_backend()
         qs = self.select_related('account', 'service')
         commit = options.get('commit', True)
-        for account, services in qs.group_by('account', 'service'):
+        for account, services in qs.group_by('account', 'service').iteritems():
             bill_lines = []
             for service, orders in services:
                 lines = service.handler.generate_bill_lines(orders, account, **options)

orchestra/apps/orders/rating.py

@@ -7,14 +7,25 @@ def _compute(rates, metric):
     value = 0
     num = len(rates)
     accumulated = 0
+    barrier = 1
+    next_barrier = None
     end = False
     ix = 0
     steps = []
     while ix < num and not end:
+        fold = 1
+        # Multiple contractions
+        while ix < num-1 and rates[ix] == rates[ix+1]:
+            ix += 1
+            fold += 1
         if ix+1 == num:
             quantity = metric - accumulated
+            next_barrier = quantity
         else:
             quantity = rates[ix+1].quantity - rates[ix].quantity
+            next_barrier = quantity
+            if rates[ix+1].price > rates[ix].price:
+                quantity *= fold
             if accumulated+quantity > metric:
                 quantity = metric - accumulated
                 end = True
@@ -22,9 +33,10 @@ def _compute(rates, metric):
         steps.append(AttributeDict(**{
             'quantity': quantity,
             'price': price,
-            'barrier': accumulated+1,
+            'barrier': barrier,
         }))
         accumulated += quantity
+        barrier += next_barrier
         value += quantity*price
         ix += 1
     return value, steps
@@ -32,9 +44,10 @@ def _compute(rates, metric):
 
 def step_price(rates, metric):
     # Step price
+    # TODO allow multiple plans
     group = []
     minimal = (sys.maxint, [])
-    for plan, rates in rates.group_by('plan'):
+    for plan, rates in rates.group_by('plan').iteritems():
         value, steps = _compute(rates, metric)
         if plan.is_combinable:
             group.append(steps)
@@ -90,7 +103,7 @@ def match_price(rates, metric):
     candidates = []
     selected = False
     prev = None
-    for rate in rates:
+    for rate in rates.distinct():
         if prev and prev.plan != rate.plan:
             if not selected and prev.quantity <= metric:
                 candidates.append(prev)

orchestra/apps/orders/tests/functional_tests/tests.py

@@ -1,4 +1,6 @@
 import datetime
+import decimal
+import sys
 
 from dateutil import relativedelta
 from django.contrib.contenttypes.models import ContentType
@@ -9,7 +11,7 @@ from orchestra.apps.users.models import User
 from orchestra.utils.tests import BaseTestCase, random_ascii
 
 from ... import settings, helpers
-from ...models import Service, Order
+from ...models import Plan, Service, Order
 
 
 class OrderTests(BaseTestCase):
@@ -26,122 +28,114 @@ class OrderTests(BaseTestCase):
         account.save()
         return account
     
-    def create_service(self):
+    def create_ftp_service(self):
         service = Service.objects.create(
             description="FTP Account",
             content_type=ContentType.objects.get_for_model(User),
             match='not user.is_main and user.has_posix()',
             billing_period=Service.ANUAL,
             billing_point=Service.FIXED_DATE,
-#            delayed_billing=Service.NEVER,
             is_fee=False,
             metric='',
             pricing_period=Service.BILLING_PERIOD,
             rate_algorithm=Service.STEP_PRICE,
-#            orders_effect=Service.CONCURRENT,
             on_cancel=Service.DISCOUNT,
             payment_style=Service.PREPAY,
-#            trial_period=Service.NEVER,
+            tax=0,
-#            refound_period=Service.NEVER,
-            tax=21,
             nominal_price=10,
         )
         return service
     
-#    def test_ftp_account_1_year_fiexed(self):
+    def create_ftp(self, account=None):
-#        service = self.create_service()
-#        bp = timezone.now().date() + relativedelta.relativedelta(years=1)
-#        bills = service.orders.bill(billing_point=bp, fixed_point=True)
-#        self.assertEqual(20, bills[0].get_total())
-    
-    def create_ftp(self):
         username = '%s_ftp' % random_ascii(10)
+        if not account:
             account = self.create_account()
         user = User.objects.create_user(username=username, account=account)
         POSIX = user._meta.get_field_by_name('posix')[0].model
         POSIX.objects.create(user=user)
         return user
     
-    def atest_get_chunks(self):
+    def test_get_chunks(self):
-        service = self.create_service()
+        service = self.create_ftp_service()
         handler = service.handler
         porders = []
         now = timezone.now().date()
         ct = ContentType.objects.get_for_model(User)
+        account = self.create_account()
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order = Order.objects.get(content_type=ct, object_id=ftp.pk)
         porders.append(order)
-        end = handler.get_billing_point(order).date()
+        end = handler.get_billing_point(order)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(1, len(chunks))
         self.assertIn([now, end, []], chunks)
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order1 = Order.objects.get(content_type=ct, object_id=ftp.pk)
         order1.billed_until = now+datetime.timedelta(days=2)
         porders.append(order1)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(2, len(chunks))
         self.assertIn([order1.registered_on, order1.billed_until, [order1]], chunks)
         self.assertIn([order1.billed_until, end, []], chunks)
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order2 = Order.objects.get(content_type=ct, object_id=ftp.pk)
         order2.billed_until = now+datetime.timedelta(days=700)
         porders.append(order2)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(2, len(chunks))
         self.assertIn([order.registered_on, order1.billed_until, [order1, order2]], chunks)
         self.assertIn([order1.billed_until, end, [order2]], chunks)
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order3 = Order.objects.get(content_type=ct, object_id=ftp.pk)
         order3.billed_until = now+datetime.timedelta(days=700)
         porders.append(order3)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(2, len(chunks))
         self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks)
         self.assertIn([order1.billed_until, end, [order2, order3]], chunks)
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order4 = Order.objects.get(content_type=ct, object_id=ftp.pk)
         order4.registered_on = now+datetime.timedelta(days=5)
         order4.billed_until = now+datetime.timedelta(days=10)
         porders.append(order4)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(4, len(chunks))
         self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks)
         self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks)
         self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks)
         self.assertIn([order4.billed_until, end, [order2, order3]], chunks)
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order5 = Order.objects.get(content_type=ct, object_id=ftp.pk)
         order5.registered_on = now+datetime.timedelta(days=700)
         order5.billed_until = now+datetime.timedelta(days=780)
         porders.append(order5)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(4, len(chunks))
         self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks)
         self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks)
         self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks)
         self.assertIn([order4.billed_until, end, [order2, order3]], chunks)
         
-        ftp = self.create_ftp()
+        ftp = self.create_ftp(account=account)
         order6 = Order.objects.get(content_type=ct, object_id=ftp.pk)
         order6.registered_on = now-datetime.timedelta(days=780)
         order6.billed_until = now-datetime.timedelta(days=700)
         porders.append(order6)
-        chunks = handler.get_chunks(porders, now, end)
+        chunks = helpers.get_chunks(porders, now, end)
         self.assertEqual(4, len(chunks))
         self.assertIn([order.registered_on, order1.billed_until, [order1, order2, order3]], chunks)
         self.assertIn([order1.billed_until, order4.registered_on, [order2, order3]], chunks)
         self.assertIn([order4.registered_on, order4.billed_until, [order2, order3, order4]], chunks)
         self.assertIn([order4.billed_until, end, [order2, order3]], chunks)
     
-    def atest_sort_billed_until_or_registered_on(self):
+    def test_sort_billed_until_or_registered_on(self):
-        service = self.create_service()
+        service = self.create_ftp_service()
         now = timezone.now()
         order = Order(
             service=service,
@@ -171,7 +165,7 @@ class OrderTests(BaseTestCase):
         orders = [order3, order, order1, order2, order4, order5, order6]
         self.assertEqual(orders, sorted(orders, cmp=helpers.cmp_billed_until_or_registered_on))
     
-    def atest_compensation(self):
+    def test_compensation(self):
         now = timezone.now()
         order = Order(
             description='0',
@@ -219,7 +213,7 @@ class OrderTests(BaseTestCase):
         ])
         porders = [order3, order, order1, order2, order4, order5, order6]
         porders = sorted(porders, cmp=helpers.cmp_billed_until_or_registered_on)
-        service = self.create_service()
+        service = self.create_ftp_service()
         compensations = []
         receivers = []
         for order in porders:
@@ -237,29 +231,22 @@ class OrderTests(BaseTestCase):
                 self.assertEqual(test_line[1], compensation.end)
                 self.assertEqual(test_line[2], compensation.order)
     
-    def get_rates(self, account):
-        rates = self.rates.filter(Q(plan__is_default=True) | Q(plan__contracts__account=account)).order_by('plan', 'quantity').select_related('plan').disctinct()
-    
     def test_rates(self):
-        from ...models import Plan
+        service = self.create_ftp_service()
-        import sys
+        account = self.create_account()
-        from decimal import Decimal
-        service = self.create_service()
-        
         superplan = Plan.objects.create(name='SUPER', allow_multiple=False, is_combinable=True)
         service.rates.create(plan=superplan, quantity=1, price=0)
         service.rates.create(plan=superplan, quantity=3, price=10)
         service.rates.create(plan=superplan, quantity=4, price=9)
         service.rates.create(plan=superplan, quantity=10, price=1)
-        account = self.create_account()
         account.plans.create(plan=superplan)
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         rates = [
-            {'price': Decimal('0.00'), 'quantity': 2},
+            {'price': decimal.Decimal('0.00'), 'quantity': 2},
-            {'price': Decimal('10.00'), 'quantity': 1},
+            {'price': decimal.Decimal('10.00'), 'quantity': 1},
-            {'price': Decimal('9.00'), 'quantity': 6},
+            {'price': decimal.Decimal('9.00'), 'quantity': 6},
-            {'price': Decimal('1.00'), 'quantity': 21}
+            {'price': decimal.Decimal('1.00'), 'quantity': 21}
         ]
         for rate, result in zip(rates, results):
             self.assertEqual(rate['price'], result.price)
@@ -268,44 +255,44 @@ class OrderTests(BaseTestCase):
         dupeplan = Plan.objects.create(name='DUPE', allow_multiple=True, is_combinable=True)
         service.rates.create(plan=dupeplan, quantity=1, price=0)
         service.rates.create(plan=dupeplan, quantity=3, price=9)
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         for rate, result in zip(rates, results):
             self.assertEqual(rate['price'], result.price)
             self.assertEqual(rate['quantity'], result.quantity)
         
         account.plans.create(plan=dupeplan)
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         rates = [
-            {'price': Decimal('0.00'), 'quantity': 4},
+            {'price': decimal.Decimal('0.00'), 'quantity': 4},
-            {'price': Decimal('9.00'), 'quantity': 5},
+            {'price': decimal.Decimal('9.00'), 'quantity': 5},
-            {'price': Decimal('1.00'), 'quantity': 21},
+            {'price': decimal.Decimal('1.00'), 'quantity': 21},
         ]
         for rate, result in zip(rates, results):
             self.assertEqual(rate['price'], result.price)
             self.assertEqual(rate['quantity'], result.quantity)
         
-        hyperplan = Plan.objects.create(name='HYPER', allow_multiple=True, is_combinable=False)
+        hyperplan = Plan.objects.create(name='HYPER', allow_multiple=False, is_combinable=False)
         service.rates.create(plan=hyperplan, quantity=1, price=0)
         service.rates.create(plan=hyperplan, quantity=20, price=5)
         account.plans.create(plan=hyperplan)
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         rates = [
-            {'price': Decimal('0.00'), 'quantity': 19},
+            {'price': decimal.Decimal('0.00'), 'quantity': 19},
-            {'price': Decimal('5.00'), 'quantity': 11}
+            {'price': decimal.Decimal('5.00'), 'quantity': 11}
         ]
         for rate, result in zip(rates, results):
             self.assertEqual(rate['price'], result.price)
             self.assertEqual(rate['quantity'], result.quantity)
         hyperplan.is_combinable = True
         hyperplan.save()
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         rates = [
-            {'price': Decimal('0.00'), 'quantity': 23},
+            {'price': decimal.Decimal('0.00'), 'quantity': 23},
-            {'price': Decimal('1.00'), 'quantity': 7}
+            {'price': decimal.Decimal('1.00'), 'quantity': 7}
         ]
         for rate, result in zip(rates, results):
             self.assertEqual(rate['price'], result.price)
@@ -313,67 +300,99 @@ class OrderTests(BaseTestCase):
         
         service.rate_algorithm = service.MATCH_PRICE
         service.save()
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         self.assertEqual(1, len(results))
-        self.assertEqual(Decimal('1.00'), results[0].price)
+        self.assertEqual(decimal.Decimal('1.00'), results[0].price)
         self.assertEqual(30, results[0].quantity)
         
         hyperplan.delete()
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 8)
         self.assertEqual(1, len(results))
-        self.assertEqual(Decimal('9.00'), results[0].price)
+        self.assertEqual(decimal.Decimal('9.00'), results[0].price)
         self.assertEqual(8, results[0].quantity)
         
         superplan.delete()
-        results = service.get_rates(account)
+        results = service.get_rates(account, cache=False)
         results = service.rate_method(results, 30)
         self.assertEqual(1, len(results))
-        self.assertEqual(Decimal('9.00'), results[0].price)
+        self.assertEqual(decimal.Decimal('9.00'), results[0].price)
         self.assertEqual(30, results[0].quantity)
     
-#    def test_ftp_account_1_year_fiexed(self):
+    def test_rates_allow_multiple(self):
-#        service = self.create_service()
+        service = self.create_ftp_service()
-#        now = timezone.now().date()etb
+        account = self.create_account()
-#        month = settings.ORDERS_SERVICE_ANUAL_BILLING_MONTH
+        dupeplan = Plan.objects.create(name='DUPE', allow_multiple=True, is_combinable=True)
-#        ini = datetime.datetime(year=now.year, month=month,
+        account.plans.create(plan=dupeplan)
-#                day=1, tzinfo=timezone.get_current_timezone())
+        service.rates.create(plan=dupeplan, quantity=1, price=0)
-#        order = service.orders.all()[0]
+        service.rates.create(plan=dupeplan, quantity=3, price=9)
-#        order.registered_on = ini
+        results = service.get_rates(account, cache=False)
-#        order.save()
+        results = service.rate_method(results, 30)
-#        bp = ini
+        rates = [
-#        bills = service.orders.bill(billing_point=bp, fixed_point=False, commit=False)
+            {'price': decimal.Decimal('0.00'), 'quantity': 2},
-#        print bills[0][1][0].subtotal
+            {'price': decimal.Decimal('9.00'), 'quantity': 28},
-#        print bills
+            ]
-#        bp = ini + relativedelta.relativedelta(months=12)
+        for rate, result in zip(rates, results):
-#        bills = service.orders.bill(billing_point=bp, fixed_point=False, commit=False)
+            self.assertEqual(rate['price'], result.price)
-#        print bills[0][1][0].subtotal
+            self.assertEqual(rate['quantity'], result.quantity)
-#        print bills
-#    def test_ftp_account_2_year_fiexed(self):
-#        service = self.create_service()
-#        bp = timezone.now().date() + relativedelta.relativedelta(years=2)
-#        bills = service.orders.bill(billing_point=bp, fixed_point=True)
-#        self.assertEqual(40, bills[0].get_total())
-#    
-#    def test_ftp_account_6_month_fixed(self):
-#        service = self.create_service()
-#        bp = timezone.now().date() + relativedelta.relativedelta(months=6)
-#        bills = service.orders.bill(billing_point=bp, fixed_point=True)
-#        self.assertEqual(6, bills[0].get_total())
-#    
-#    def test_ftp_account_next_billing_point(self):
-#        service = self.create_service()
-#        now = timezone.now().date()
-#        bp_month = settings.ORDERS_SERVICE_ANUAL_BILLING_MONTH
-#        if date.month > bp_month:
-#            bp = datetime.datetime(year=now.year+1, month=bp_month,
-#                day=1, tzinfo=timezone.get_current_timezone())
-#        else:
-#            bp = datetime.datetime(year=now.year, month=bp_month,
-#                day=1, tzinfo=timezone.get_current_timezone())
-#        
-#        days = (bp - now).days
-#        bills = service.orders.bill(billing_point=bp, fixed_point=False)
-#        self.assertEqual(40, bills[0].get_total())
         
+        account.plans.create(plan=dupeplan)
+        results = service.get_rates(account, cache=False)
+        results = service.rate_method(results, 30)
+        rates = [
+            {'price': decimal.Decimal('0.00'), 'quantity': 4},
+            {'price': decimal.Decimal('9.00'), 'quantity': 26},
+            ]
+        for rate, result in zip(rates, results):
+            self.assertEqual(rate['price'], result.price)
+            self.assertEqual(rate['quantity'], result.quantity)
+
+        account.plans.create(plan=dupeplan)
+        results = service.get_rates(account, cache=False)
+        results = service.rate_method(results, 30)
+        rates = [
+            {'price': decimal.Decimal('0.00'), 'quantity': 6},
+            {'price': decimal.Decimal('9.00'), 'quantity': 24},
+            ]
+        for rate, result in zip(rates, results):
+            self.assertEqual(rate['price'], result.price)
+            self.assertEqual(rate['quantity'], result.quantity)
+    
+    def test_ftp_account_1_year_fiexed(self):
+        service = self.create_ftp_service()
+        user = self.create_ftp()
+        bp = timezone.now().date() + relativedelta.relativedelta(years=1)
+        bills = service.orders.bill(billing_point=bp, fixed_point=True)
+        self.assertEqual(10, bills[0].get_total())
+    
+    def test_ftp_account_2_year_fiexed(self):
+        service = self.create_ftp_service()
+        user = self.create_ftp()
+        bp = timezone.now().date() + relativedelta.relativedelta(years=2)
+        bills = service.orders.bill(billing_point=bp, fixed_point=True)
+        self.assertEqual(20, bills[0].get_total())
+    
+    def test_ftp_account_6_month_fixed(self):
+        service = self.create_ftp_service()
+        self.create_ftp()
+        bp = timezone.now().date() + relativedelta.relativedelta(months=6)
+        bills = service.orders.bill(billing_point=bp, fixed_point=True)
+        self.assertEqual(5, bills[0].get_total())
+    
+    def test_ftp_account_next_billing_point(self):
+        service = self.create_ftp_service()
+        self.create_ftp()
+        now = timezone.now()
+        bp_month = settings.ORDERS_SERVICE_ANUAL_BILLING_MONTH
+        if now.month > bp_month:
+            bp = datetime.datetime(year=now.year+1, month=bp_month,
+                day=1, tzinfo=timezone.get_current_timezone())
+        else:
+            bp = datetime.datetime(year=now.year, month=bp_month,
+                day=1, tzinfo=timezone.get_current_timezone())
+        bills = service.orders.bill(billing_point=now, fixed_point=False)
+        size = decimal.Decimal((bp - now).days)/365
+        error = decimal.Decimal(0.05)
+        self.assertGreater(10*size+error*(10*size), bills[0].get_total())
+        self.assertLess(10*size-error*(10*size), bills[0].get_total())

orchestra/apps/payments/actions.py

@@ -11,7 +11,7 @@ def process_transactions(modeladmin, request, queryset):
         msg = _("Selected transactions must be on '{state}' state")
         messages.error(request, msg.format(state=Transaction.WAITTING_PROCESSING))
         return
-    for method, transactions in queryset.group_by('source__method'):
+    for method, transactions in queryset.group_by('source__method').iteritems():
         if method is not None:
             method = PaymentMethod.get_plugin(method)
             procs = method.process(transactions)

orchestra/apps/resources/admin.py

@@ -137,7 +137,7 @@ def resource_inline_factory(resources):
 
 if not running_syncdb():
     # not run during syncdb
-    for ct, resources in Resource.objects.group_by('content_type'):
+    for ct, resources in Resource.objects.group_by('content_type').iteritems():
         inline = resource_inline_factory(resources)
         model = ct.model_class()
         insertattr(model, 'inlines', inline)

orchestra/apps/resources/models.py

@@ -178,7 +178,7 @@ def create_resource_relation():
             return self
     
     relation = GenericRelation('resources.ResourceData')
-    for ct, resources in Resource.objects.group_by('content_type'):
+    for ct, resources in Resource.objects.group_by('content_type').iteritems():
         model = ct.model_class()
         model.add_to_class('resource_set', relation)
         model.resources = ResourceHandler()

orchestra/apps/resources/serializers.py

@@ -26,7 +26,7 @@ class ResourceSerializer(serializers.ModelSerializer):
 if not running_syncdb():
     # TODO why this is even loaded during syncdb?
     # Create nested serializers on target models
-    for ct, resources in Resource.objects.group_by('content_type'):
+    for ct, resources in Resource.objects.group_by('content_type').iteritems():
         model = ct.model_class()
         try:
             router.insert(model, 'resources', ResourceSerializer, required=False, many=True, source='resource_set')

orchestra/core/caches.py

@@ -1,5 +1,6 @@
 from threading import currentThread
 
+from django.core.cache.backends.dummy import DummyCache
 from django.core.cache.backends.locmem import LocMemCache
 
 
@@ -16,14 +17,12 @@ class RequestCache(LocMemCache):
 def get_request_cache():
     """
     Returns per-request cache when running RequestCacheMiddleware otherwise a
-    new LocMemCache instance (when running periodic tasks or shell)
+    DummyCache instance (when running periodic tasks, tests or shell)
     """
     try:
         return _request_cache[currentThread()]
     except KeyError:
-        cache = RequestCache()
+        return DummyCache('dummy', {})
-        _request_cache[currentThread()] = cache
-        return cache
 
 
 class RequestCacheMiddleware(object):
@@ -33,7 +32,6 @@ class RequestCacheMiddleware(object):
         cache.clear()
     
     def process_response(self, request, response):
-        # TODO not sure if this actually saves memory, remove otherwise
         if currentThread() in _request_cache:
             _request_cache[currentThread()].clear()
         return response

orchestra/models/queryset.py

@@ -1,28 +1,30 @@
+from collections import OrderedDict
+
 from .utils import get_field_value
 
 
-def group_by(qset, *fields, **kwargs):
+def group_by(qset, *fields):
-    """ group_by iterator with support for multiple nested fields """
+    """ 100% in python in order to preserve original order_by """
-    ix = kwargs.get('ix', 0)
+    first = OrderedDict()
-    if ix is 0:
+    num = len(fields)
-        qset = qset.order_by(*fields)
-    group = []
-    first = True
     for obj in qset:
+        ix = 0
+        group = first
+        while ix < num:
             try:
                 current = get_field_value(obj, fields[ix])
             except AttributeError:
                 # Intermediary relation does not exists
                 current = None
-        if first or current == previous:
+            if ix < num-1:
-            group.append(obj)
+                try:
+                    group = group[current]
+                except KeyError:
+                    group[current] = OrderedDict()
             else:
-            if ix < len(fields)-1:
+                try:
-                group = group_by(group, *fields, ix=ix+1)
+                    group[current].append(obj)
-            yield previous, group
+                except KeyError:
-            group = [obj]
+                    group[current] = [obj]
-        previous = current
+            ix += 1
-        first = False
+    return first
-    if ix < len(fields)-1:
-        group = group_by(group, *fields, ix=ix+1)
-    yield previous, group