Improved resource data history

TODO.md

@@ -365,7 +365,6 @@ method(
     arg, arg, arg)
 
 
-
 Bash/Python/PHPBackend
 
 # services.handler as generator in order to save memory? not swell like a balloon
@@ -385,8 +384,6 @@ uwsgi --reload /tmp/project-master.pid
 # or if uwsgi was started with touch-reload=/tmp/somefile
 touch /tmp/somefile
 
-# batch zone edditing
-
 # datetime metric storage granularity: otherwise innacurate detection of billed metric on order.billed_on
 
 # Serializers.validation migration to DRF3: grep -r 'attrs, source' *|grep -v '~'
@@ -422,18 +419,10 @@ Case
 
 # case on payment transaction state ? case when trans.amount > 
 
-# Resource data inline show info: link to monitor data, and history chart: link to monitor data of each item
+# Resource inline links point to custom changelist view that preserve state (breadcrumbs, title, etc) rather than generic changeview with queryarg filtering
-
 
 # ORDER diff Pending vs ALL
 # pre-bill confirmation: remove account if lines.count() == 0 ?
 
 # Discount prepaid metric should be more optimal https://orchestra.pangea.org/admin/orders/order/40/
 #   -> order.billed_metric besides billed_until
-
-# USE CONTROLLED MONITOR GRAPHS
-# graph metric storage
-# CLEANUP MONITOR DATA 0.00 (from SUM aggregators, AVG are needed for maintaining state), maybe also aggregate older values
-# MULTIPLE GRAPH TYPE: datapoint line chart for AVG, stacked for SUM())
-
-

orchestra/contrib/resources/actions.py

@@ -42,9 +42,8 @@ def run_monitor(modeladmin, request, queryset):
 run_monitor.url_name = 'monitor'
 
 
-def history(modeladmin, request, queryset):
+def show_history(modeladmin, request, queryset):
     context = {
         'ids': ','.join(map(str, queryset.values_list('id', flat=True))),
     }
     return render(request, 'admin/resources/resourcedata/history.html', context)
-history.url_name = 'history'

orchestra/contrib/resources/admin.py

@@ -1,12 +1,15 @@
 from urllib.parse import parse_qs
 
+from django.apps import apps
 from django.conf.urls import url
 from django.contrib import admin, messages
 from django.contrib.contenttypes.admin import GenericTabularInline
 from django.contrib.contenttypes.forms import BaseGenericInlineFormSet
 from django.contrib.admin.utils import unquote
 from django.core.urlresolvers import reverse
+from django.db.models import Q
 from django.shortcuts import redirect
+from django.templatetags.static import static
 from django.utils.functional import cached_property
 from django.utils.safestring import mark_safe
 from django.utils.translation import ungettext, ugettext, ugettext_lazy as _
@@ -18,7 +21,7 @@ from orchestra.core import services
 from orchestra.utils import db, sys
 from orchestra.utils.functional import cached
 
-from .actions import run_monitor, history
+from .actions import run_monitor, show_history
 from .api import history_data
 from .filters import ResourceDataListFilter
 from .forms import ResourceForm
@@ -120,7 +123,7 @@ class ResourceDataAdmin(ExtendedModelAdmin):
     )
     search_fields = ('content_object_repr',)
     readonly_fields = fields
-    actions = (run_monitor, history)
+    actions = (run_monitor, show_history)
     change_view_actions = actions
     ordering = ('-updated_at',)
     list_select_related = ('resource__content_type', 'content_type')
@@ -166,15 +169,13 @@ class ResourceDataAdmin(ExtendedModelAdmin):
         return redirect(url)
     
     def list_related_view(self, request, app_name, model_name, object_id):
-        from django.apps import apps
-        from django.db.models import Q
         resources = Resource.objects.select_related('content_type')
         resource_models = {r.content_type.model_class(): r.content_type_id for r in resources}
         # Self
         model = apps.get_model(app_name, model_name)
         obj = model.objects.get(id=int(object_id))
         ct_id = resource_models[model]
-        qset = Q(content_type_id=ct_id, object_id=obj.id)
+        qset = Q(content_type_id=ct_id, object_id=obj.id, resource__is_active=True)
         # Related
         for field, rel in obj._meta.fields_map.items():
             try:
@@ -184,7 +185,7 @@ class ResourceDataAdmin(ExtendedModelAdmin):
             else:
                 manager = getattr(obj, field)
                 ids = manager.values_list('id', flat=True)
-                qset = Q(qset) | Q(content_type_id=ct_id, object_id__in=ids)
+                qset = Q(qset) | Q(content_type_id=ct_id, object_id__in=ids, resource__is_active=True)
         related = ResourceData.objects.filter(qset)
         related_ids = related.values_list('id', flat=True)
         related_ids = ','.join(map(str, related_ids))
@@ -211,7 +212,7 @@ class MonitorDataAdmin(ExtendedModelAdmin):
             mdata = ResourceData.objects.get(pk=int(resource_data[0]))
             resource = mdata.resource
             ids = []
-            for dataset in mdata.get_monitor_datasets():
+            for monitor, dataset in mdata.get_monitor_datasets():
                 dataset = resource.aggregation_instance.filter(dataset)
                 if isinstance(dataset, MonitorData):
                     ids.append(dataset.id)
@@ -302,9 +303,8 @@ def resource_inline_factory(resources):
             return super(ResourceInline, self).get_fieldsets(request, obj)
         
         def display_used(self, rdata):
-            from django.templatetags.static import static
+            update = ''
-            update_link = ''
+            history = ''
-            history_link = ''
             if rdata.pk:
                 context = {
                     'title': _("Update"),
@@ -315,13 +315,15 @@ def resource_inline_factory(resources):
                 context.update({
                     'title': _("Show history"),
                     'image': '<img src="%s"></img>' % static('orchestra/images/history.png'),
-                    'url': reverse('admin:resources_resourcedata_history', args=(rdata.pk,)),
+                    'url': reverse('admin:resources_resourcedata_show_history', args=(rdata.pk,)),
                     'popup': 'onclick="return showAddAnotherPopup(this);"',
                 })
                 history = '<a href="%(url)s" title="%(title)s" %(popup)s>%(image)s</a>' % context
             if rdata.used is not None:
-                return ' '.join(map(str, (rdata.used, rdata.resource.unit, update, history)))
+                used_url = reverse('admin:resources_resourcedata_used_monitordata', args=(rdata.pk,))
-            return _("Unknonw %s") % update_link
+                used =  '<a href="%s">%s %s</a>' % (used_url, rdata.used, rdata.unit)
+                return ' '.join(map(str, (used, update, history)))
+            return _("Unknonw %s %s") % (update, history)
         display_used.short_description = _("Used")
         display_used.allow_tags = True
         

orchestra/contrib/resources/api.py

@@ -11,9 +11,5 @@ def history_data(request):
     ids = map(int, parse_qs(request.META['QUERY_STRING'])['ids'][0].split(','))
     queryset = ResourceData.objects.filter(id__in=ids)
     history = get_history_data(queryset)
-    def default(obj):
+    response = json.dumps(history, indent=4)
-        if isinstance(obj, set):
-            return list(obj)
-        return obj
-    response = json.dumps(history, default=default, indent=4)
     return HttpResponse(response, content_type="application/json")

orchestra/contrib/resources/helpers.py

@@ -1,4 +1,4 @@
-from django.template.defaultfilters import date as date_filter
+from django.template.defaultfilters import date as date_format
 
 
 def get_history_data(queryset):
@@ -18,7 +18,7 @@ def get_history_data(queryset):
                 'unit': resource.unit,
                 'scale': resource.get_scale(),
                 'verbose_name': str(resource.verbose_name),
-                'dates': set(),
+                'dates': set() if aggregation.aggregated_history else None,
                 'objects': [],
             }
             resources[resource] = (options, aggregation)
@@ -33,10 +33,9 @@ def get_history_data(queryset):
                     needs_aggregation = True
                     serie = {}
                     for data in datas:
-                        date = date_filter(data.date)
                         value = round(float(data.value)/scale, 3) if data.value is not None else None
-                        all_dates.add(date)
+                        all_dates.add(data.date)
-                        serie[date] = value
+                        serie[data.date] = value
                 else:
                     serie = []
                     for data in datas:
@@ -62,7 +61,8 @@ def get_history_data(queryset):
         result = []
         for options, aggregation in resources.values():
             if aggregation.aggregated_history:
-                all_dates = options['dates']
+                all_dates = sorted(options['dates'])
+                options['dates'] = [date_format(date) for date in all_dates]
                 for obj in options['objects']:
                     for monitor in obj['monitors']:
                         series = []

orchestra/contrib/resources/templates/admin/resources/resourcedata/history.html

@@ -4,6 +4,7 @@
 <head>
     <title>Resource history</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+    <link rel="stylesheet" type="text/css" href="{% static "orchestra/css/dancing-dots.css" %}"/>
     <script src="{% static "admin/js/jquery.js" %}" type="text/javascript"></script>
     <script src="{% static "orchestra/js/highcharts/stock/highstock.js" %}" type="text/javascript"></script>
     <script src="{% static "orchestra/js/highcharts/modules/exporting.js" %}" type="text/javascript"></script>
@@ -23,7 +24,10 @@
                         selected: 4
                     },
                     title: {
-                        text: resource['verbose_name'] + ' ' + resource['content_type'] + ' ' + resource['aggregation'] + (div.indexOf('aggregate') > 0 ? ' (aggregated)': '')
+                        text: resource['content_type'].capitalize() + ' ' +
+                              resource['verbose_name'].toLowerCase() + ' ' +
+                              resource['aggregation'] +
+                              (div.indexOf('aggregate') > 0 ? ' (aggregated)': '')
                     },
                     yAxis: {
                         labels: {
@@ -38,9 +42,25 @@
                         }],
                         min: 0,
                     },
+                    /*legend: {
+                        align: 'right',
+                        x: -30,
+                        verticalAlign: 'top',
+                        y: 25,
+                        floating: true,
+                        backgroundColor: (Highcharts.theme && Highcharts.theme.background2) || 'white',
+                        borderColor: '#CCC',
+                        borderWidth: 1,
+                        shadow: false,
+                        enabled: true
+                    },
+                    rangeSelector: {
+                        enabled: false
+                    },*/
                     tooltip: {
-                        pointFormat: '<span style="color:{series.color}">{series.name}</span>: <b>{point.y} '+resource['unit']+ '</b><br/>',
+                        pointFormat: '<span style="color:{series.color}">{series.name}</span>: <b>{point.y:.3f} ' +
-                        valueDecimals: 2
+                                     resource['unit']+ '</b><br/>',
+                        valueDecimals: 3
                     },
                     series: seriesOptions
                 });
@@ -52,7 +72,10 @@
                         backgroundColor: (i % 2 ? "#EDF3FE" : "#FFFFFF")
                     },
                     title: {
-                        text: resource['verbose_name'] + ' ' + resource['content_type'] + ' ' + resource['aggregation'] + (div.indexOf('aggregate') > 0 ? ' (aggregated)': '')
+                        text: resource['content_type'].capitalize() + ' ' +
+                              resource['verbose_name'].toLowerCase() + ' ' +
+                              resource['aggregation'] +
+                              (div.indexOf('aggregate') > 0 ? ' (aggregated)': '')
                     },
                     xAxis: {
                         categories: resource['dates']
@@ -86,10 +109,15 @@
                     },
                     tooltip: {
                         formatter: function () {
-                            return '<b>' + this.x + '</b><br/>' +
+                            var s = ['<b>' + this.x + '</b>'];
-                                this.series.name + ': ' + this.y.toFixed(3) + ' ' + resource['unit'] + '<br/>' +
+                            $.each(this.points, function(i, point) {
-                                'Total: ' + this.point.stackTotal.toFixed(3) + ' ' + resource['unit'];
+                                s.push('<span style="color:' + this.series.color + '">' + this.series.name + ': ' + this.y + ' ' + resource['unit']);
-                        }
+                            });
+                            s.push('<b>Total: ' + this.points[0].total + ' ' + resource['unit'] + '</b>');
+                            return s.join('<br>');
+                        },
+                        valueDecimals: 3,
+                        shared: true
                     },
                     plotOptions: {
                         column: {
@@ -158,7 +186,7 @@
                 };
                 
                 divs = (
-                    '<div style="background: '+(i % 2 ? "#EDF3FE" : "#FFFFFF")+'; margin-bottom: -1px; border: 1px solid grey; padding: 10px;">' +
+                    '<div style="background: '+(i % 2 ? "#EDF3FE" : "#FFFFFF")+'; margin: 10px; margin-bottom: -1px; border: 1px solid grey; padding: 20px;">' +
                     '<h1>'+resource['content_type'].capitalize() + ' ' + resource['verbose_name'].toLowerCase() + '</h1>' +
                     '<div id="resource-'+i+'" style="height: 400px; min-width: 310px"></div>'
                 );
@@ -179,40 +207,17 @@
     @page {
        size: 11.69in 8.27in;
     }
-    h1{
+    h1 {
        font-family: sans;
        font-size: 21px;
     }
-    table {
-       max-width: 10in;
-       font-family: sans;
-       font-size: 10px;
-    }
-    table tr:nth-child(even) {
-        background-color: #eee;
-    }
-    table tr:nth-child(odd) {
-        background-color: #fff;
-    }
-    table th {
-        color: white;
-        background-color: grey;
-    }
-    .item.column-created, .item.column-updated {
-        text-align: center;
-    }
-    .item.column-amount {
-        text-align: right;
-    }
-    .footnote {
-       font-family: sans;
-       font-size: 10px;
-    }
     </style>
 </head>
 <body>
     <div id="charts">
-    Crunching data ...
+        <div id="message" style="width:300px; margin:0 auto; font-family: monospace; font-weight: bold; font-size: 18px; margin-top: 5%">
+            > crunching data <span id="dancing-dots-text"> <span><span>.</span><span>.</span><span>.</span></span></span>
+        </div>
     </div>
 </body>
 </html>

orchestra/static/orchestra/css/dancing-dots.css

@@ -0,0 +1,40 @@
+@-webkit-keyframes dancing-dots-jump {
+  0% { top: 0; }
+  55% { top: 0; }
+  60% { top: -10px; }
+  80% { top: 3px; }
+  90% { top: -2px; }
+  95% { top: 1px; }
+  100% { top: 0; }
+}
+
+@-moz-keyframes dancing-dots-jump {
+  0% { top: 0; }
+  55% { top: 0; }
+  60% { top: -10px; }
+  80% { top: 3px; }
+  90% { top: -2px; }
+  95% { top: 1px; }
+  100% { top: 0; }
+}
+
+#dancing-dots-text span span {
+  -webkit-animation-duration: 1800ms;
+  -webkit-animation-iteration-count: infinite;
+  -webkit-animation-name: dancing-dots-jump;
+  -moz-animation-duration: 1800ms;
+  -moz-animation-iteration-count: infinite;
+  -moz-animation-name: dancing-dots-jump;
+  padding: 1px;
+  position: relative;
+}
+
+#dancing-dots-text span span:nth-child(2) {
+  -webkit-animation-delay: 100ms;
+  -moz-animation-delay: 100ms;
+}
+
+#dancing-dots-text span span:nth-child(3) {
+  -webkit-animation-delay: 300ms;
+  -moz-animation-delay: 300ms;
+}