trustchain-oc1-orchestral
/
devicehub-teal
Archived
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #396 from eReuse/changes/3853-report 

Changes/3853 report
This commit is contained in:
cayop 2022-11-03 23:56:57 +01:00 committed by GitHub
parent 3a7a8d6e1d 6ee62eb415
commit c784fb7499
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 114 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -127,3 +127,8 @@ yarn.lock
# ESLint Report
eslint_report.json
modules/
tmp/
.env*
bin/
env*

96
ereuse_devicehub/commands/reports.py
View File

@ -1,96 +0,0 @@
import csv
# import click_spinner
# import ereuse_utils.cli
from io import StringIO
from ereuse_devicehub.resources.action import models as evs
from ereuse_devicehub.resources.device.models import Placeholder
from ereuse_devicehub.resources.documents.device_row import InternalStatsRow
# import click
class Report:
def __init__(self, app) -> None:
super().__init__()
self.app = app
short_help = 'Creates reports devices and users.'
self.app.cli.command('report', short_help=short_help)(self.run)
def run(self):
stats = InternalStatsView()
stats.print()
class InternalStatsView:
def print(self):
query = evs.Action.query.filter(
evs.Action.type.in_(
(
'Snapshot',
'Live',
'Allocate',
'Deallocate',
'EraseBasic',
'EraseSectors',
)
)
)
return self.generate_post_csv(query)
def generate_post_csv(self, query):
data = StringIO()
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"')
cw.writerow(InternalStatsRow('', "2000-1", [], []).keys())
for row in self.get_rows(query):
cw.writerow(row)
return print(data.getvalue())
def get_rows(self, query):
d = {}
dd = {}
disks = []
for ac in query:
create = '{}-{}'.format(ac.created.year, ac.created.month)
user = ac.author.email
if user not in d:
d[user] = {}
dd[user] = {}
if create not in d[user]:
d[user][create] = []
dd[user][create] = None
d[user][create].append(ac)
for user, createds in d.items():
for create, actions in createds.items():
r = InternalStatsRow(user, create, actions, disks)
dd[user][create] = r
return self.get_placeholders(dd)
def get_placeholders(self, dd):
for p in Placeholder.query.all():
create = '{}-{}'.format(p.created.year, p.created.month)
user = p.owner.email
if user not in dd:
dd[user] = {}
if create not in dd[user]:
dd[user][create] = None
if not dd[user][create]:
dd[user][create] = InternalStatsRow(user, create, [], [])
dd[user][create]['Placeholders'] += 1
rows = []
for user, createds in dd.items():
for create, row in createds.items():
rows.append(row.values())
return rows

7
ereuse_devicehub/devicehub.py
View File

@ -15,7 +15,8 @@ from teal.teal import Teal
from ereuse_devicehub.auth import Auth
from ereuse_devicehub.client import Client, UserClient
from ereuse_devicehub.commands.reports import Report
# from ereuse_devicehub.commands.reports import Report
from ereuse_devicehub.commands.users import GetToken
from ereuse_devicehub.config import DevicehubConfig
from ereuse_devicehub.db import db
@ -29,7 +30,7 @@ from ereuse_devicehub.templating import Environment
class Devicehub(Teal):
test_client_class = Client
Dummy = Dummy
Report = Report
# Report = Report
jinja_environment = Environment
def __init__(
@ -70,7 +71,7 @@ class Devicehub(Teal):
self.id = inventory
"""The Inventory ID of this instance. In Teal is the app.schema."""
self.dummy = Dummy(self)
self.report = Report(self)
# self.report = Report(self)
self.get_token = GetToken(self)
@self.cli.group(

101
ereuse_devicehub/resources/documents/device_row.py
View File

@ -614,104 +614,3 @@ class ActionRow(OrderedDict):
self['Type'] = allocate['type']
self['LiveCreate'] = allocate['liveCreate']
self['UsageTimeHdd'] = allocate['usageTimeHdd']
class InternalStatsRow(OrderedDict):
def __init__(self, user, create, actions, disks):
super().__init__()
# General information about all internal stats
# user, quart, month, year:
# Snapshot (Registers)
# Snapshots (Update)
# Snapshots (All)
# Drives Erasure
# Drives Erasure Uniques
# Placeholders
# Allocate
# Deallocate
# Live
self.actions = actions
year, month = create.split('-')
self.disks = disks
self['User'] = user
self['Year'] = year
self['Quarter'] = self.quarter(month)
self['Month'] = month
self['Snapshot (Registers)'] = 0
self['Snapshot (Update)'] = 0
self['Snapshot (All)'] = 0
self['Drives Erasure'] = 0
self['Drives Erasure Uniques'] = 0
self['Placeholders'] = 0
self['Allocates'] = 0
self['Deallocates'] = 0
self['Lives'] = 0
self.count_actions()
def count_actions(self):
for ac in self.actions:
self.is_snapshot(
self.is_deallocate(self.is_live(self.is_allocate(self.is_erase(ac))))
)
def is_allocate(self, ac):
if ac.type == 'Allocate':
self['Allocates'] += 1
return ac
def is_erase(self, ac):
if ac.type in ['EraseBasic', 'EraseSectors']:
self['Drives Erasure'] += 1
if ac.device in self.disks:
return ac
self['Drives Erasure Uniques'] += 1
self.disks.append(ac.device)
return ac
def is_live(self, ac):
if ac.type == 'Live':
self['Lives'] += 1
return ac
def is_deallocate(self, ac):
if ac.type == 'Deallocate':
self['Deallocates'] += 1
return ac
def is_snapshot(self, ac):
if not ac.type == 'Snapshot':
return
self['Snapshot (All)'] += 1
canary = False
for _ac in ac.device.actions:
if not _ac.type == 'Snapshot':
continue
if _ac.created < ac.created:
canary = True
break
if canary:
self['Snapshot (Update)'] += 1
else:
self['Snapshot (Registers)'] += 1
def quarter(self, month):
q = {
1: 'Q1',
2: 'Q1',
3: 'Q1',
4: 'Q2',
5: 'Q2',
6: 'Q2',
7: 'Q3',
8: 'Q3',
9: 'Q3',
10: 'Q4',
11: 'Q4',
12: 'Q4',
}
return q[int(month)]

195
ereuse_devicehub/resources/documents/documents.py
View File

@ -30,7 +30,6 @@ from ereuse_devicehub.resources.device.views import DeviceView
from ereuse_devicehub.resources.documents.device_row import (
ActionRow,
DeviceRow,
InternalStatsRow,
StockRow,
)
from ereuse_devicehub.resources.enums import SessionType
@ -62,9 +61,9 @@ class DocumentView(DeviceView):
200:
description: Return the collection or the specific one.
"""
args = self.QUERY_PARSER.parse(self.find_args,
flask.request,
locations=('querystring',))
args = self.QUERY_PARSER.parse(
self.find_args, flask.request, locations=('querystring',)
)
ids = []
if 'filter' in request.args:
filters = json.loads(request.args.get('filter', {}))
@ -113,7 +112,8 @@ class DocumentView(DeviceView):
template = self.erasure(query)
if args.get('format') == Format.PDF:
res = flask_weasyprint.render_pdf(
flask_weasyprint.HTML(string=template), download_filename='{}.pdf'.format(type)
flask_weasyprint.HTML(string=template),
download_filename='{}.pdf'.format(type),
)
insert_hash(res.data)
else:
@ -140,7 +140,7 @@ class DocumentView(DeviceView):
params = {
'title': 'Erasure Certificate',
'erasures': tuple(erasures()),
'url_pdf': url_pdf.to_text()
'url_pdf': url_pdf.to_text(),
}
return flask.render_template('documents/erasure.html', **params)
@ -159,7 +159,13 @@ class DevicesDocumentView(DeviceView):
def generate_post_csv(self, query):
"""Get device query and put information in csv format."""
data = StringIO()
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL)
cw = csv.writer(
data,
delimiter=';',
lineterminator="\n",
quotechar='"',
quoting=csv.QUOTE_ALL,
)
first = True
document_ids = self.get_documents_id()
for device in query:
@ -193,7 +199,13 @@ class ActionsDocumentView(DeviceView):
def generate_post_csv(self, query):
"""Get device query and put information in csv format."""
data = StringIO()
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL)
cw = csv.writer(
data,
delimiter=';',
lineterminator="\n",
quotechar='"',
quoting=csv.QUOTE_ALL,
)
first = True
devs_id = []
for device in query:
@ -204,7 +216,9 @@ class ActionsDocumentView(DeviceView):
cw.writerow(d.keys())
first = False
cw.writerow(d.values())
query_trade = Trade.query.filter(Trade.devices.any(Device.id.in_(devs_id))).all()
query_trade = Trade.query.filter(
Trade.devices.any(Device.id.in_(devs_id))
).all()
lot_id = request.args.get('lot')
if lot_id and not query_trade:
@ -225,7 +239,9 @@ class ActionsDocumentView(DeviceView):
bfile = data.getvalue().encode('utf-8')
output = make_response(bfile)
insert_hash(bfile)
output.headers['Content-Disposition'] = 'attachment; filename=actions_export.csv'
output.headers[
'Content-Disposition'
] = 'attachment; filename=actions_export.csv'
output.headers['Content-type'] = 'text/csv'
return output
@ -277,7 +293,13 @@ class StockDocumentView(DeviceView):
def generate_post_csv(self, query):
"""Get device query and put information in csv format."""
data = StringIO()
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL)
cw = csv.writer(
data,
delimiter=';',
lineterminator="\n",
quotechar='"',
quoting=csv.QUOTE_ALL,
)
first = True
for device in query:
d = StockRow(device)
@ -311,6 +333,7 @@ class StampsView(View):
This view render one public ans static page for see the links for to do the check
of one csv file
"""
def get_url_path(self):
url = urlutils.URL(request.url)
url.normalize()
@ -319,8 +342,9 @@ class StampsView(View):
def get(self):
result = ('', '')
return flask.render_template('documents/stamp.html', rq_url=self.get_url_path(),
result=result)
return flask.render_template(
'documents/stamp.html', rq_url=self.get_url_path(), result=result
)
def post(self):
result = ('', '')
@ -331,54 +355,26 @@ class StampsView(View):
ok = '100% coincidence. The attached file contains data 100% existing in \
to our backend'
result = ('Bad', bad)
mime = ['text/csv', 'application/pdf', 'text/plain', 'text/markdown',
'image/jpeg', 'image/png', 'text/html',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.oasis.opendocument.spreadsheet',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/msword']
mime = [
'text/csv',
'application/pdf',
'text/plain',
'text/markdown',
'image/jpeg',
'image/png',
'text/html',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.oasis.opendocument.spreadsheet',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/msword',
]
if file_check.mimetype in mime:
if verify_hash(file_check):
result = ('Ok', ok)
return flask.render_template('documents/stamp.html', rq_url=self.get_url_path(),
result=result)
class InternalStatsView(DeviceView):
@cache(datetime.timedelta(minutes=1))
def find(self, args: dict):
if not g.user.email == app.config['EMAIL_ADMIN']:
return jsonify('')
query = evs.Action.query.filter(
evs.Action.type.in_(('Snapshot', 'Live', 'Allocate', 'Deallocate')))
return self.generate_post_csv(query)
def generate_post_csv(self, query):
d = {}
for ac in query:
create = '{}-{}'.format(ac.created.year, ac.created.month)
user = ac.author.email
if user not in d:
d[user] = {}
if create not in d[user]:
d[user][create] = []
d[user][create].append(ac)
data = StringIO()
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"')
cw.writerow(InternalStatsRow('', "2000-1", []).keys())
for user, createds in d.items():
for create, actions in createds.items():
cw.writerow(InternalStatsRow(user, create, actions).values())
bfile = data.getvalue().encode('utf-8')
output = make_response(bfile)
insert_hash(bfile)
output.headers['Content-Disposition'] = 'attachment; filename=internal-stats.csv'
output.headers['Content-type'] = 'text/csv'
return output
return flask.render_template(
'documents/stamp.html', rq_url=self.get_url_path(), result=result
)
class WbConfDocumentView(DeviceView):
@ -386,10 +382,11 @@ class WbConfDocumentView(DeviceView):
if not wbtype.lower() in ['usodyrate', 'usodywipe']:
return jsonify('')
data = {'token': self.get_token(),
'host': app.config['HOST'],
'inventory': app.config['SCHEMA']
}
data = {
'token': self.get_token(),
'host': app.config['HOST'],
'inventory': app.config['SCHEMA'],
}
data['erase'] = False
# data['erase'] = True if wbtype == 'usodywipe' else False
@ -424,18 +421,31 @@ class DocumentDef(Resource):
VIEW = None # We do not want to create default / documents endpoint
AUTH = False
def __init__(self, app,
import_name=__name__,
static_folder='static',
static_url_path=None,
template_folder='templates',
url_prefix=None,
subdomain=None,
url_defaults=None,
root_path=None,
cli_commands: Iterable[Tuple[Callable, str or None]] = tuple()):
super().__init__(app, import_name, static_folder, static_url_path, template_folder,
url_prefix, subdomain, url_defaults, root_path, cli_commands)
def __init__(
self,
app,
import_name=__name__,
static_folder='static',
static_url_path=None,
template_folder='templates',
url_prefix=None,
subdomain=None,
url_defaults=None,
root_path=None,
cli_commands: Iterable[Tuple[Callable, str or None]] = tuple(),
):
super().__init__(
app,
import_name,
static_folder,
static_url_path,
template_folder,
url_prefix,
subdomain,
url_defaults,
root_path,
cli_commands,
)
d = {'id': None}
get = {'GET'}
@ -446,12 +456,15 @@ class DocumentDef(Resource):
view = app.auth.requires_auth(view)
self.add_url_rule('/erasures/', defaults=d, view_func=view, methods=get)
self.add_url_rule('/erasures/<{}:{}>'.format(self.ID_CONVERTER.value, self.ID_NAME),
view_func=view, methods=get)
self.add_url_rule(
'/erasures/<{}:{}>'.format(self.ID_CONVERTER.value, self.ID_NAME),
view_func=view,
methods=get,
)
devices_view = DevicesDocumentView.as_view('devicesDocumentView',
definition=self,
auth=app.auth)
devices_view = DevicesDocumentView.as_view(
'devicesDocumentView', definition=self, auth=app.auth
)
devices_view = app.auth.requires_auth(devices_view)
stock_view = StockDocumentView.as_view('stockDocumentView', definition=self)
@ -463,7 +476,9 @@ class DocumentDef(Resource):
lots_view = app.auth.requires_auth(lots_view)
self.add_url_rule('/lots/', defaults=d, view_func=lots_view, methods=get)
stock_view = StockDocumentView.as_view('stockDocumentView', definition=self, auth=app.auth)
stock_view = StockDocumentView.as_view(
'stockDocumentView', definition=self, auth=app.auth
)
stock_view = app.auth.requires_auth(stock_view)
self.add_url_rule('/stock/', defaults=d, view_func=stock_view, methods=get)
@ -471,22 +486,18 @@ class DocumentDef(Resource):
self.add_url_rule('/check/', defaults={}, view_func=check_view, methods=get)
stamps_view = StampsView.as_view('StampsView', definition=self, auth=app.auth)
self.add_url_rule('/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'})
self.add_url_rule(
'/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'}
)
# internalstats_view = InternalStatsView.as_view(
# 'InternalStatsView', definition=self, auth=app.auth)
# internalstats_view = app.auth.requires_auth(internalstats_view)
# self.add_url_rule('/internalstats/', defaults=d, view_func=internalstats_view,
# methods=get)
actions_view = ActionsDocumentView.as_view('ActionsDocumentView',
definition=self,
auth=app.auth)
actions_view = ActionsDocumentView.as_view(
'ActionsDocumentView', definition=self, auth=app.auth
)
actions_view = app.auth.requires_auth(actions_view)
self.add_url_rule('/actions/', defaults=d, view_func=actions_view, methods=get)
wbconf_view = WbConfDocumentView.as_view('WbConfDocumentView',
definition=self,
auth=app.auth)
wbconf_view = WbConfDocumentView.as_view(
'WbConfDocumentView', definition=self, auth=app.auth
)
wbconf_view = app.auth.requires_auth(wbconf_view)
self.add_url_rule('/wbconf/<string:wbtype>', view_func=wbconf_view, methods=get)

3
tests/test_render_2_0.py
View File

@ -1,10 +1,11 @@
import datetime
import pytest
import json
from io import BytesIO
from pathlib import Path
from uuid import UUID
import pytest
from flask import g
from flask.testing import FlaskClient
from flask_wtf.csrf import generate_csrf