Fix bugs on resource history and added order.billed_metric field

TODO.md

@@ -417,15 +417,19 @@ Greatest
 Colaesce('total', 'computed_total')
 Case
 
-# case on payment transaction state ? case when trans.amount > 
+# SQL case on payment transaction state ? case when trans.amount > 
 
 # Resource inline links point to custom changelist view that preserve state (breadcrumbs, title, etc) rather than generic changeview with queryarg filtering
 
 # ORDER diff Pending vs ALL
-# pre-bill confirmation: remove account if lines.count() == 0 ?
 
-# Discount prepaid metric should be more optimal https://orchestra.pangea.org/admin/orders/order/40/
+# DELETING RESOURCE RELATED OBJECT SHOULD NOT delete related monitor data for traffic accountancy
-#   -> order.billed_metric besides billed_until
 
+# round decimals on every billing operation
 
-# websites directives: redirect strip() and allow empty URL_path
+# Serie1
+
+# Pangea post-create: lorena no has afegit el webalizer
+# cleanup monitor data
+
+# Add SPF record type

orchestra/contrib/orders/filters.py

@@ -51,24 +51,12 @@ class BilledOrderListFilter(SimpleListFilter):
             queryset=MetricStorage.objects.filter(created_on__gt=F('order__billed_on'),
                 created_on__lte=(F('updated_on')-mindelta))
         )
-        prefetch_billed_metric = Prefetch('metrics', to_attr='billed_metric',
-            queryset=MetricStorage.objects.filter(order__billed_on__isnull=False,
-                created_on__lte=F('order__billed_on'), updated_on__gt=F('order__billed_on'))
-        )
         metric_queryset = queryset.exclude(service__metric='').exclude(billed_on__isnull=True)
-        for order in metric_queryset.prefetch_related(prefetch_valid_metrics, prefetch_billed_metric):
+        for order in metric_queryset.prefetch_related(prefetch_valid_metrics):
-            if len(order.billed_metric) != 1:
-                # corner case of prefetch_billed_metric: Does not always work with latests metrics
-                latest = order.metrics.latest()
-                if not latest:
-                    raise ValueError("Data inconsistency #metrics %i != 1." % len(order.billed_metric))
-                billed_metric = latest.value
-            else:
-                billed_metric = order.billed_metric[0].value
             for metric in order.valid_metrics:
                 if metric.created_on <= order.billed_on:
                     raise ValueError("This value should already be filtered on the prefetch query.")
-                if metric.value > billed_metric:
+                if metric.value > order.billed_metric:
                     metric_pks.append(order.pk)
                     break
         return metric_pks

orchestra/contrib/orders/migrations/0004_auto_20150729_0945.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orders', '0003_order_content_object_repr'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='order',
+            name='billed_metric',
+            field=models.DecimalField(verbose_name='billed metric', max_digits=16, decimal_places=2, null=True),
+        ),
+        migrations.AlterField(
+            model_name='order',
+            name='content_object_repr',
+            field=models.CharField(verbose_name='content object representation', max_length=256, editable=False),
+        ),
+    ]

orchestra/contrib/orders/models.py

@@ -114,13 +114,14 @@ class Order(models.Model):
         related_name='orders')
     registered_on = models.DateField(_("registered"), default=timezone.now, db_index=True)
     cancelled_on = models.DateField(_("cancelled"), null=True, blank=True)
-    # TODO billed metric
     billed_on = models.DateField(_("billed"), null=True, blank=True)
+    billed_metric = models.DecimalField(_("billed metric"), max_digits=16, decimal_places=2,
+        null=True, blank=True)
     billed_until = models.DateField(_("billed until"), null=True, blank=True)
     ignore = models.BooleanField(_("ignore"), default=False)
     description = models.TextField(_("description"), blank=True)
     content_object_repr = models.CharField(_("content object representation"), max_length=256,
-        editable=False)
+        editable=False, help_text=_("Used for searches."))
     
     content_object = GenericForeignKey()
     objects = OrderQuerySet.as_manager()
@@ -239,13 +240,15 @@ class Order(models.Model):
         if kwargs:
             raise AttributeError
         if len(args) == 2:
+            # Slot
             ini, end = args
-            metrics = self.metrics.filter(updated_on__lt=end, updated_on__gte=ini)
+            metrics = self.metrics.filter(created_on__lt=end, updated_on__gte=ini)
         elif len(args) == 1:
+            # On effect on date
             date = args[0]
             date = datetime.date(year=date.year, month=date.month, day=date.day)
             date += datetime.timedelta(days=1)
-            metrics = self.metrics.filter(updated_on__lt=date)
+            metrics = self.metrics.filter(created_on__lte=date)
         elif not args:
             return self.metrics.latest('updated_on').value
         else:
@@ -261,7 +264,6 @@ class MetricStorage(models.Model):
     order = models.ForeignKey(Order, verbose_name=_("order"), related_name='metrics')
     value = models.DecimalField(_("value"), max_digits=16, decimal_places=2)
     created_on = models.DateField(_("created"), auto_now_add=True)
-#    default=lambda: timezone.now())
     # TODO time field?
     updated_on = models.DateTimeField(_("updated"))
     

orchestra/contrib/orders/templates/admin/orders/order/bill_selected_options.html

@@ -47,27 +47,35 @@ $(document).ready( function () {
              <div class="tabular inline-related last-related">
             <fieldset class="module">
             <h2><a href="{% url 'admin:accounts_account_change' account.pk %}">{{ account }}</a><span style="float:right">{{ total | floatformat:"-2" }} &euro;</span></h2>
-            <table>
+            {% if not lines %}
-            <thead>
+                <table>
-                <tr><th style="width:30%;">Description</th> <th style="width:30%;">Period</th> <th style="width:10%;">Size&times;Quantity</th> <th style="width:10%;">Price</th></tr>
+                <thead>
-            </thead>
+                    <tr><th>{% trans 'Nothing to bill' %}</th></tr>
-            <tbody>
+                </thead>
-            {% for line in lines %}
+                </table>
-            <tr class="form-row {% if forloop.counter|divisibleby:2 %}row2{% else %}row1{% endif %}">
+            {% else %}
-                <td>
+                <table>
-                    <a href="{{ line.order | admin_url }}">{{ line.order.description }}</a>
+                <thead>
-                    {% for discount in line.discounts %}
+                    <tr><th style="width:30%;">Description</th> <th style="width:30%;">Period</th> <th style="width:10%;">Size&times;Quantity</th> <th style="width:10%;">Price</th></tr>
-                        <br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Discount per {{ discount.type }}
+                </thead>
-                    {% endfor %}
+                <tbody>
-                </td>
+                {% for line in lines %}
-                <td>{{ line.ini | date }} to {{ line.end | date }}</td>
+                <tr class="form-row {% if forloop.counter|divisibleby:2 %}row2{% else %}row1{% endif %}">
-                <td>{{ line.size | floatformat:"-2" }}&times;{{ line.metric | floatformat:"-2"}}</td>
+                    <td>
-                <td>
+                        <a href="{{ line.order | admin_url }}">{{ line.order.description }}</a>
-                    &nbsp;{{ line.subtotal | floatformat:"-2" }} &euro;
+                        {% for discount in line.discounts %}
-                    {% for discount in line.discounts %}<br>{{ discount.total | floatformat:"-2" }} &euro;{% endfor %}
+                            <br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Discount per {{ discount.type }}
-                </td>
+                        {% endfor %}
-            </tr>
+                    </td>
-            {% endfor %}
+                    <td>{{ line.ini | date }} to {{ line.end | date }}</td>
+                    <td>{{ line.size | floatformat:"-2" }}&times;{{ line.metric | floatformat:"-2"}}</td>
+                    <td>
+                        &nbsp;{{ line.subtotal | floatformat:"-2" }} &euro;
+                        {% for discount in line.discounts %}<br>{{ discount.total | floatformat:"-2" }} &euro;{% endfor %}
+                    </td>
+                </tr>
+                {% endfor %}
+            {% endif %}
             </tbody>
             </table>
             </fieldset>

orchestra/contrib/resources/aggregations.py

@@ -1,6 +1,7 @@
 import copy
 import datetime
 import decimal
+import itertools
 
 from dateutil.relativedelta import relativedelta
 from django.utils import timezone
@@ -74,37 +75,36 @@ class MonthlySum(Last):
         )
     
     def aggregate_history(self, dataset):
-        make_data = lambda mdata, current: AttrDict(
+        prev = None
-            date=datetime.date(
-                year=mdata.created_at.year,
-                month=mdata.created_at.month,
-                day=1
-            ),
-            value=current,
-            content_object_repr=mdata.content_object_repr
-        )
-        
-        prev_month = None
         prev_object_id = None
         datas = []
-        for mdata in dataset.order_by('object_id', 'created_at'):
+        sink = AttrDict(object_id=-1, value=-1, content_object_repr='',
+            created_at=AttrDict(year=-1, month=-1))
+        for mdata in itertools.chain(dataset.order_by('object_id', 'created_at'), [sink]):
             object_id = mdata.object_id
-            if object_id != prev_object_id:
+            ymonth = (mdata.created_at.year, mdata.created_at.month)
+            if object_id != prev_object_id or ymonth != prev.ymonth:
                 if prev_object_id is not None:
-                    yield (mdata.content_object_repr, datas)
+                    data = AttrDict(
-                datas = []
+                        date=datetime.date(
-            month = mdata.created_at.month
+                            year=prev.ymonth[0],
-            if object_id != prev_object_id or month != prev_month:
+                            month=prev.ymonth[1],
-                if prev_month is not None:
+                            day=1
-                    datas.append(make_data(mdata, current))
+                        ),
+                        value=current,
+                        content_object_repr=prev.content_object_repr
+                    )
+                    datas.append(data)
                 current = mdata.value
             else:
                 current += mdata.value
-            prev_month = month
+            if object_id != prev_object_id:
+                if prev_object_id is not None:
+                    yield(prev.content_object_repr, datas)
+                    datas = []
+            prev = mdata
+            prev.ymonth = ymonth
             prev_object_id = object_id
-        if prev_object_id is not None:
-            datas.append(make_data(mdata, current))
-            yield (mdata.content_object_repr, datas)
 
 
 class MonthlyAvg(MonthlySum):
@@ -122,7 +122,7 @@ class MonthlyAvg(MonthlySum):
             day=1,
         )
     
-    def compute_usage(self, dataset, historic=False):
+    def compute_usage(self, dataset):
         result = 0
         has_result = False
         aggregate = []
@@ -140,15 +140,9 @@ class MonthlyAvg(MonthlySum):
                 slot = (mdata.created_at-ini).total_seconds()
                 current += mdata.value * decimal.Decimal(str(slot/total))
                 ini = mdata.created_at
-            if historic:
-                aggregate.append(
-                    (mdata, current)
-                )
             else:
                 result += current
         if has_result:
-            if historic:
-                return aggregate
             return result
         return None
     

orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html

@@ -186,12 +186,12 @@
                 };
                 
                 divs = (
-                    '<div style="background: '+(i % 2 ? "#EDF3FE" : "#FFFFFF")+'; margin: 10px; margin-bottom: -1px; border: 1px solid grey; padding: 20px;">' +
+                    '<div class="chart-box" style="background: '+(i % 2 ? "#EDF3FE" : "#FFFFFF")+';">' +
                     '<h1>'+resource['content_type'].capitalize() + ' ' + resource['verbose_name'].toLowerCase() + '</h1>' +
-                    '<div id="resource-'+i+'" style="height: 400px; min-width: 310px"></div>'
+                    '<div id="resource-'+i+'" class="chart"></div>'
                 );
                 if (a_index > 1 && aggregated && resource['aggregated_history'])
-                    divs += '<br><div id="resource-'+i+'-aggregate" style="height: 400px; min-width: 310px"></div>';
+                    divs += '<br><div class="chart" id="resource-'+i+'-aggregate"></div>';
                 divs += '</div>';
                 
                 $("#charts").append(divs);
@@ -211,11 +211,36 @@
        font-family: sans;
        font-size: 21px;
     }
+    #notice {
+       font-family: sans;
+       font-size: 12px;
+       text-align: right;
+       padding-right: 10px;
+    }
+    #message {
+        width:300px;
+        margin:0 auto;
+        font-family: monospace;
+        font-weight: bold;
+        font-size: 18px;
+        margin-top: 5%;
+    }
+    .chart-box {
+        margin: 10px;
+        margin-bottom: -1px;
+        border: 1px solid grey;
+        padding: 20px;
+    }
+    .chart {
+        height: 400px;
+        min-width: 310px;
+    }
     </style>
 </head>
 <body>
+    <div id="notice">&#9830;Notice that resources used by deleted services will not appear.</div>
     <div id="charts">
-        <div id="message" style="width:300px; margin:0 auto; font-family: monospace; font-weight: bold; font-size: 18px; margin-top: 5%">
+        <div id="message">
             > crunching data <span id="dancing-dots-text"> <span><span>.</span><span>.</span><span>.</span></span></span>
         </div>
     </div>

orchestra/contrib/services/handlers.py

@@ -203,6 +203,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
             size = 1
         else:
             raise NotImplementedError
+        size = round(size, 2)
         return decimal.Decimal(str(size))
     
     def get_pricing_slots(self, ini, end):
@@ -492,52 +493,69 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
         lines = []
         bp = None
         for order in orders:
-            recharges = []
+            prepay_discount = 0
             bp = self.get_billing_point(order, bp=bp, **options)
             if (self.billing_period != self.NEVER and
                 self.get_pricing_period() == self.NEVER and
                 self.payment_style == self.PREPAY and order.billed_on):
                     # Recharge
                     if self.payment_style == self.PREPAY and order.billed_on:
+                        recharges = []
                         rini = order.billed_on
                         rend = min(bp, order.billed_until)
-                        cmetric = None
+                        bmetric = order.billed_metric
+                        bsize = self.get_price_size(rini, order.billed_until)
+                        prepay_discount = self.get_price(account, bmetric) * bsize
+                        prepay_discount = round(prepay_discount, 2)
                         for cini, cend, metric in order.get_metric(rini, rend, changes=True):
-                            if cmetric is None:
-                                cmetric = metric
-                                csize = self.get_price_size(rini, order.billed_until)
-                                cprice = self.get_price(account, cmetric) * csize
                             size = self.get_price_size(cini, cend)
                             price = self.get_price(account, metric) * size
                             discounts = ()
-                            discount = min(price, max(cprice, 0))
+                            discount = min(price, max(prepay_discount, 0))
-                            if discount:
+                            prepay_discount -= price
-                                cprice -= price
+                            if discount > 0:
                                 price -= discount
                                 discounts = (
                                     ('prepay', -discount),
                                 )
-                            # if price-discount:
+                            # Don't overdload bills with lots of lines
-                            recharges.append((order, price, cini, cend, metric, discounts))
+                            if price > 0:
-                        # only recharge when appropiate in order to preserve bigger prepays.
+                                recharges.append((order, price, cini, cend, metric, discounts))
-                        if cmetric < metric or bp > order.billed_until:
+                        if prepay_discount < 0:
+                            # User has prepaid less than the actual consumption
                             for order, price, cini, cend, metric, discounts in recharges:
                                 line = self.generate_line(order, price, cini, cend, metric=metric,
                                     computed=True, discounts=discounts)
                                 lines.append(line)
             if order.billed_until and order.cancelled_on and order.cancelled_on >= order.billed_until:
+                # Cancelled order
                 continue
             if self.billing_period != self.NEVER:
                 ini = order.billed_until or order.registered_on
                 # Periodic billing
                 if bp <= ini:
+                    # Already billed
                     continue
                 order.new_billed_until = bp
                 if self.get_pricing_period() == self.NEVER:
                     # Changes (Mailbox disk-like)
                     for cini, cend, metric in order.get_metric(ini, bp, changes=True):
                         price = self.get_price(account, metric)
-                        lines.append(self.generate_line(order, price, cini, cend, metric=metric))
+                        discounts = ()
+                        # Since the current datamodel can't guarantee to retrieve the exact
+                        # state for calculating prepay_discount (service price could have change)
+                        # maybe is it better not to discount anything.
+#                        discount = min(price, max(prepay_discount, 0))
+#                        if discount > 0:
+#                            price -= discount
+#                            prepay_discount -= discount
+#                            discounts = (
+#                                ('prepay', -discount),
+#                            )
+                        if metric > 0:
+                            line = self.generate_line(order, price, cini, cend, metric=metric,
+                                discounts=discounts)
+                            lines.append(line)
                 elif self.get_pricing_period() == self.billing_period:
                     # pricing_slots (Traffic-like)
                     if self.payment_style == self.PREPAY:
@@ -545,7 +563,18 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
                     for cini, cend in self.get_pricing_slots(ini, bp):
                         metric = order.get_metric(cini, cend)
                         price = self.get_price(account, metric)
-                        lines.append(self.generate_line(order, price, cini, cend, metric=metric))
+                        discounts = ()
+#                        discount = min(price, max(prepay_discount, 0))
+#                        if discount > 0:
+#                            price -= discount
+#                            prepay_discount -= discount
+#                            discounts = (
+#                                ('prepay', -discount),
+#                            )
+                        if metric > 0:
+                            line = self.generate_line(order, price, cini, cend, metric=metric,
+                                discounts=discounts)
+                            lines.append(line)
                 else:
                     raise NotImplementedError
             else:
@@ -558,9 +587,12 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
                     # get metric (Job-like)
                     metric = order.get_metric(date)
                     price = self.get_price(account, metric)
-                    lines.append(self.generate_line(order, price, date, metric=metric))
+                    line = self.generate_line(order, price, date, metric=metric)
+                    lines.append(line)
                 else:
                     raise NotImplementedError
+            # Last processed metric for futrue recharges
+            order.new_billed_metric = metric
         return lines
     
     def generate_bill_lines(self, orders, account, **options):
@@ -575,6 +607,7 @@ class ServiceHandler(plugins.Plugin, metaclass=plugins.PluginMount):
             for line in lines:
                 order = line.order
                 order.billed_on = now
+                order.billed_metric = getattr(order, 'new_billed_metric', order.billed_metric)
                 order.billed_until = getattr(order, 'new_billed_until', order.billed_until)
-                order.save(update_fields=['billed_on', 'billed_until'])
+                order.save(update_fields=('billed_on', 'billed_until', 'billed_metric'))
         return lines

orchestra/contrib/services/models.py

@@ -211,6 +211,7 @@ class Service(models.Model):
                 if counter >= metric:
                     counter = metric
                     accumulated += (counter - ant_counter) * rate['price']
+                    accumulated = round(accumulated, 2)
                     return decimal.Decimal(str(accumulated))
                 ant_counter = counter
                 accumulated += rate['price'] * rate['quantity']
@@ -221,6 +222,7 @@ class Service(models.Model):
             for rate in rates:
                 counter += rate['quantity']
                 if counter >= position:
+                    price = round(rate['price'], 2)
                     return decimal.Decimal(str(rate['price']))
             raise RuntimeError("Rating algorithm bad result")
     

orchestra/utils/humanize.py

@@ -40,7 +40,7 @@ def _un(singular__plural, n=None):
     return ungettext(singular, plural, n)
 
 
-def naturaldatetime(date, include_seconds=True):
+def naturaldatetime(date, show_seconds=False):
     """Convert datetime into a human natural date string."""
     if not date:
         return ''
@@ -63,19 +63,17 @@ def naturaldatetime(date, include_seconds=True):
     
     if days == 0:
         if hours == 0:
-            if minutes >= 1:
+            if minutes >= 1 or not show_seconds:
                 minutes = float(seconds)/60
                 return ungettext(
                     _("{minutes:.1f} minute{ago}"),
                     _("{minutes:.1f} minutes{ago}"), minutes
                 ).format(minutes=minutes, ago=ago)
             else:
-                if include_seconds:
+                return ungettext(
-                    return ungettext(
+                    _("{seconds} second{ago}"),
-                        _("{seconds} second{ago}"),
+                    _("{seconds} seconds{ago}"), seconds
-                        _("{seconds} seconds{ago}"), seconds
+                ).format(seconds=seconds, ago=ago)
-                    ).format(seconds=seconds, ago=ago)
-                return _("just now")
         else:
             hours = float(minutes)/60
             return ungettext(