From 810a9b360c08e13305e1d5fc01367dab11c31e63 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Fri, 8 Jan 2021 17:37:52 +0100 Subject: [PATCH 1/4] adding new endpoit for download csv of metrics --- CHANGELOG.md | 1 + .../resources/documents/device_row.py | 10 ++++++ .../resources/documents/documents.py | 34 ++++++++++++++++++- tests/test_basic.py | 1 + tests/test_documents.py | 26 +++++++++++++- 5 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf7df40..93602dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ml). - [addend] #95 adding endpoint for check the hash of one report - [addend] #98 adding endpoint for insert a new live - [addend] #98 adding endpoint for get all licences in one query +- [addend] #102 adding endpoint for download metrics - [bugfix] #100 fixing bug of scheme live - [bugfix] #101 fixing bug when 2 users have one device and launch one live diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py index 70e7bfda..f53ea13d 100644 --- a/ereuse_devicehub/resources/documents/device_row.py +++ b/ereuse_devicehub/resources/documents/device_row.py @@ -5,6 +5,7 @@ from flask import url_for from ereuse_devicehub.resources.enums import Severity from ereuse_devicehub.resources.device import models as d, states +from ereuse_devicehub.resources.action import models as da from ereuse_devicehub.resources.action.models import (BenchmarkDataStorage, RateComputer, TestDataStorage) @@ -360,3 +361,12 @@ def get_action(component, action): """ Filter one action from a component or return None """ result = [a for a in component.actions if a.type == action] return result[-1] if result else None + + +class ActionRow(OrderedDict): + + def __init__(self, action: da.Action) -> None: + super().__init__() + self.action = action + # General information about action + self['type'] = action.type diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index a7db64c6..b9b214da 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -20,7 +20,7 @@ from ereuse_devicehub.db import db from ereuse_devicehub.resources.action import models as evs from ereuse_devicehub.resources.device import models as devs from ereuse_devicehub.resources.device.views import DeviceView -from ereuse_devicehub.resources.documents.device_row import DeviceRow, StockRow +from ereuse_devicehub.resources.documents.device_row import DeviceRow, StockRow, ActionRow from ereuse_devicehub.resources.lot import LotView from ereuse_devicehub.resources.lot.models import Lot from ereuse_devicehub.resources.hash_reports import insert_hash, ReportHash @@ -133,6 +133,32 @@ class DevicesDocumentView(DeviceView): return output +class ActionsDocumentView(DeviceView): + @cache(datetime.timedelta(minutes=1)) + def find(self, args: dict): + query = (x for x in self.query(args) if x.owner_id == g.user.id) + return self.generate_post_csv(query) + + def generate_post_csv(self, query): + """Get device query and put information in csv format.""" + data = StringIO() + cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') + first = True + for device in query: + for action in device.actions: + d = ActionRow(action) + if first: + cw.writerow(d.keys()) + first = False + cw.writerow(d.values()) + bfile = data.getvalue().encode('utf-8') + output = make_response(bfile) + insert_hash(bfile) + output.headers['Content-Disposition'] = 'attachment; filename=actions_export.csv' + output.headers['Content-type'] = 'text/csv' + return output + + class LotsDocumentView(LotView): def find(self, args: dict): query = (x for x in self.query(args) if x.owner_id == g.user.id) @@ -256,3 +282,9 @@ class DocumentDef(Resource): check_view = CheckView.as_view('CheckView', definition=self, auth=app.auth) self.add_url_rule('/check/', defaults={}, view_func=check_view, methods=get) + + actions_view = ActionsDocumentView.as_view('ActionsDocumentView', + definition=self, + auth=app.auth) + actions_view = app.auth.requires_auth(actions_view) + self.add_url_rule('/actions/', defaults=d, view_func=actions_view, methods=get) diff --git a/tests/test_basic.py b/tests/test_basic.py index 978948af..31208608 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -49,6 +49,7 @@ def test_api_docs(client: Client): '/displays/{dev1_id}/merge/{dev2_id}', '/diy-and-gardenings/{dev1_id}/merge/{dev2_id}', '/documents/devices/', + '/documents/actions/', '/documents/erasures/', '/documents/lots/', '/documents/static/{filename}', diff --git a/tests/test_documents.py b/tests/test_documents.py index eecee5ad..bb7d802b 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -85,7 +85,7 @@ def test_erasure_certificate_wrong_id(client: Client): @pytest.mark.mvp def test_export_csv_permitions(user: UserClient, user2: UserClient, client: Client): - """Test export device information in a csv file with others users.""" + """test export device information in a csv file with others users.""" snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot) csv_user, _ = user.get(res=documents.DocumentDef.t, item='devices/', @@ -107,6 +107,30 @@ def test_export_csv_permitions(user: UserClient, user2: UserClient, client: Clie assert len(csv_user2) == 0 +@pytest.mark.mvp +def test_export_csv_actions(user: UserClient, user2: UserClient, client: Client): + """Test export device information in a csv file with others users.""" + snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot) + csv_user, _ = user.get(res=documents.DocumentDef.t, + item='actions/', + accept='text/csv', + query=[('filter', {'type': ['Computer']})]) + + csv_user2, _ = user2.get(res=documents.DocumentDef.t, + item='actions/', + accept='text/csv', + query=[('filter', {'type': ['Computer']})]) + + _, res = client.get(res=documents.DocumentDef.t, + item='actions/', + accept='text/csv', + query=[('filter', {'type': ['Computer']})], status=401) + assert res.status_code == 401 + + assert len(csv_user) > 0 + assert len(csv_user2) == 0 + + @pytest.mark.mvp def test_export_basic_snapshot(user: UserClient): """Test export device information in a csv file.""" From 12f28ba76fa0ec584db3e00af15f01bbf5480ffd Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Wed, 13 Jan 2021 18:11:41 +0100 Subject: [PATCH 2/4] metrics in csv --- ereuse_devicehub/resources/action/models.py | 24 ++++++++- ereuse_devicehub/resources/action/views.py | 4 +- ereuse_devicehub/resources/device/models.py | 49 +++++++++++++++++++ .../resources/documents/device_row.py | 14 ++++-- .../resources/documents/documents.py | 4 +- 5 files changed, 86 insertions(+), 9 deletions(-) diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py index af5f1047..7dce355b 100644 --- a/ereuse_devicehub/resources/action/models.py +++ b/ereuse_devicehub/resources/action/models.py @@ -32,8 +32,8 @@ from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import backref, relationship, validates from sqlalchemy.orm.events import AttributeEvents as Events from sqlalchemy.util import OrderedSet -from teal.db import CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \ - POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range +from teal.db import (CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, + POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range, ResourceNotFound) from teal.enums import Country, Currency, Subdivision from teal.marshmallow import ValidationError from teal.resource import url_for_resource @@ -543,6 +543,26 @@ class Snapshot(JoinedWithOneDeviceMixin, ActionWithOneDevice): of time it took to complete. """ + def get_last_lifetimes(self): + """We get the lifetime and serial_number of the first disk""" + hdds = [] + components = copy.copy(self.components) + components.sort(key=lambda x: x.created) + for hd in components: + data = {'serial_number': None, 'lifetime': 0} + if not isinstance(hd, DataStorage): + continue + + data['serial_number'] = hd.serial_number + for act in hd.actions: + if not act.type == "TestDataStorage": + continue + data['lifetime'] = act.lifetime + break + hdds.append(data) + + return hdds + def __str__(self) -> str: return '{}. {} version {}.'.format(self.severity, self.software, self.version) diff --git a/ereuse_devicehub/resources/action/views.py b/ereuse_devicehub/resources/action/views.py index f1e496dd..01295028 100644 --- a/ereuse_devicehub/resources/action/views.py +++ b/ereuse_devicehub/resources/action/views.py @@ -124,7 +124,9 @@ class LiveView(View): """We get the liftime and serial_number of the disk""" usage_time_hdd = None serial_number = None - for hd in snapshot['components']: + components = [c for c in snapshot['components']] + components.sort(key=lambda x: x.created) + for hd in components: if not isinstance(hd, DataStorage): continue diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py index 87f673ce..a3f825e4 100644 --- a/ereuse_devicehub/resources/device/models.py +++ b/ereuse_devicehub/resources/device/models.py @@ -1,4 +1,5 @@ import pathlib +import copy from contextlib import suppress from fractions import Fraction from itertools import chain @@ -317,6 +318,54 @@ class Device(Thing): def _warning_actions(self, actions): return sorted(ev for ev in actions if ev.severity >= Severity.Warning) + def get_metrics(self): + """ + This method get a list of values for calculate a metrics from a spreadsheet + """ + actions = copy.copy(self.actions) + actions.sort(key=lambda x: x.created) + allocates = [] + from ereuse_devicehub.resources.action.models import Snapshot, Allocate, Deallocate, Live + for act in actions: + if isinstance(act, Snapshot): + snapshot = act + lifestimes = snapshot.get_last_lifetimes() + lifetime = 0 + if lifestimes: + lifetime = lifestimes[0]['lifetime'] + + if isinstance(act, Allocate): + allo = {'type': 'Allocate', + 'userCode': act.final_user_code, + 'numUsers': act.end_users, + 'hid': self.hid, + 'liveCreate': 0, + 'liveLifetime': 0, + 'start': act.start_time, + 'allocateLifetime': lifetime} + allocates.append(allo) + + if isinstance(act, Live): + allocate = copy.copy(allo) + lifetime = act.usage_time_hdd + allocate['type'] = 'Live' + allocate['liveCreate'] = act.created + allocate['liveLifetime'] = lifetime + allocates.append(allocate) + + if isinstance(act, Deallocate): + deallo = {'type': 'Deallocate', + 'userCode': '', + 'numUsers': '', + 'hid': self.hid, + 'liveCreate': 0, + 'liveLifetime': lifetime, + 'start': act.start_time, + 'allocateLifetime': 0} + allocates.append(deallo) + + return allocates + def __lt__(self, other): return self.id < other.id diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py index f53ea13d..82f0c628 100644 --- a/ereuse_devicehub/resources/documents/device_row.py +++ b/ereuse_devicehub/resources/documents/device_row.py @@ -365,8 +365,14 @@ def get_action(component, action): class ActionRow(OrderedDict): - def __init__(self, action: da.Action) -> None: + def __init__(self, allocate): super().__init__() - self.action = action - # General information about action - self['type'] = action.type + # General information about allocates, deallocate and lives + self['Hid'] = allocate['hid'] + self['Start'] = allocate['start'] + self['UserCode'] = allocate['userCode'] + self['NumUsers'] = allocate['numUsers'] + self['AllocateLifetime'] = allocate['allocateLifetime'] + self['Type'] = allocate['type'] + self['LiveCreate'] = allocate['liveCreate'] + self['LiveLifetime'] = allocate['liveLifetime'] diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py index b9b214da..f1da06a6 100644 --- a/ereuse_devicehub/resources/documents/documents.py +++ b/ereuse_devicehub/resources/documents/documents.py @@ -145,8 +145,8 @@ class ActionsDocumentView(DeviceView): cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"') first = True for device in query: - for action in device.actions: - d = ActionRow(action) + for allocate in device.get_metrics(): + d = ActionRow(allocate) if first: cw.writerow(d.keys()) first = False From d974d3d78fb64298d0fa1e6f979058ce9f6f01b1 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 14 Jan 2021 17:32:03 +0100 Subject: [PATCH 3/4] ereuse_devicehub/resources/action/models.py --- ereuse_devicehub/resources/device/models.py | 17 ++++++++------- tests/test_documents.py | 23 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py index a3f825e4..7081ce49 100644 --- a/ereuse_devicehub/resources/device/models.py +++ b/ereuse_devicehub/resources/device/models.py @@ -325,16 +325,15 @@ class Device(Thing): actions = copy.copy(self.actions) actions.sort(key=lambda x: x.created) allocates = [] - from ereuse_devicehub.resources.action.models import Snapshot, Allocate, Deallocate, Live for act in actions: - if isinstance(act, Snapshot): + if act.type == 'Snapshot': snapshot = act lifestimes = snapshot.get_last_lifetimes() lifetime = 0 if lifestimes: lifetime = lifestimes[0]['lifetime'] - if isinstance(act, Allocate): + if act.type == 'Allocate': allo = {'type': 'Allocate', 'userCode': act.final_user_code, 'numUsers': act.end_users, @@ -345,15 +344,19 @@ class Device(Thing): 'allocateLifetime': lifetime} allocates.append(allo) - if isinstance(act, Live): + if act.type == 'Live': allocate = copy.copy(allo) - lifetime = act.usage_time_hdd + lifetime = act.usage_time_hdd.total_seconds() allocate['type'] = 'Live' allocate['liveCreate'] = act.created + for hd in lifestimes: + if hd['serial_number'] == act.serial_number: + lifetime = hd['lifetime'] + break allocate['liveLifetime'] = lifetime allocates.append(allocate) - if isinstance(act, Deallocate): + if act.type == 'Deallocate': deallo = {'type': 'Deallocate', 'userCode': '', 'numUsers': '', @@ -364,7 +367,7 @@ class Device(Thing): 'allocateLifetime': 0} allocates.append(deallo) - return allocates + return allocates def __lt__(self, other): return self.id < other.id diff --git a/tests/test_documents.py b/tests/test_documents.py index bb7d802b..f76ca0ed 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -11,7 +11,7 @@ from ereuse_utils.test import ANY from ereuse_devicehub.client import Client, UserClient from ereuse_devicehub.devicehub import Devicehub -from ereuse_devicehub.resources.action.models import Snapshot +from ereuse_devicehub.resources.action.models import Snapshot, Allocate, Live from ereuse_devicehub.resources.documents import documents from ereuse_devicehub.resources.device import models as d from ereuse_devicehub.resources.lot.models import Lot @@ -110,7 +110,24 @@ def test_export_csv_permitions(user: UserClient, user2: UserClient, client: Clie @pytest.mark.mvp def test_export_csv_actions(user: UserClient, user2: UserClient, client: Client): """Test export device information in a csv file with others users.""" - snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot) + acer = file('acer.happy.battery.snapshot') + snapshot, _ = user.post(acer, res=Snapshot) + device_id = snapshot['device']['id'] + post_request = {"transaction": "ccc", "name": "John", "endUsers": 1, + "devices": [device_id], "description": "aaa", + "finalUserCode": "abcdefjhi", + "startTime": "2020-11-01T02:00:00+00:00", + "endTime": "2020-12-01T02:00:00+00:00" + } + + user.post(res=Allocate, data=post_request) + hdd = [c for c in acer['components'] if c['type'] == 'HardDrive'][0] + hdd_action = [a for a in hdd['actions'] if a['type'] == 'TestDataStorage'][0] + hdd_action['lifetime'] += 1000 + acer.pop('elapsed') + acer['licence_version'] = '1.0.0' + snapshot, _ = client.post(acer, res=Live) + csv_user, _ = user.get(res=documents.DocumentDef.t, item='actions/', accept='text/csv', @@ -131,7 +148,7 @@ def test_export_csv_actions(user: UserClient, user2: UserClient, client: Client) assert len(csv_user2) == 0 -@pytest.mark.mvp +@pytest.mark.mvp def test_export_basic_snapshot(user: UserClient): """Test export device information in a csv file.""" snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot) From 652273db90572918b6bb216b7e285e7554443e00 Mon Sep 17 00:00:00 2001 From: Cayo Puigdefabregas Date: Thu, 14 Jan 2021 17:32:46 +0100 Subject: [PATCH 4/4] fixinfg bufgs and tests --- ereuse_devicehub/resources/action/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py index 7dce355b..9c93ef29 100644 --- a/ereuse_devicehub/resources/action/models.py +++ b/ereuse_devicehub/resources/action/models.py @@ -546,7 +546,7 @@ class Snapshot(JoinedWithOneDeviceMixin, ActionWithOneDevice): def get_last_lifetimes(self): """We get the lifetime and serial_number of the first disk""" hdds = [] - components = copy.copy(self.components) + components = [c for c in self.components] components.sort(key=lambda x: x.created) for hd in components: data = {'serial_number': None, 'lifetime': 0} @@ -557,7 +557,7 @@ class Snapshot(JoinedWithOneDeviceMixin, ActionWithOneDevice): for act in hd.actions: if not act.type == "TestDataStorage": continue - data['lifetime'] = act.lifetime + data['lifetime'] = act.lifetime.total_seconds() break hdds.append(data)