Merge pull request #396 from eReuse/changes/3853-report
Changes/3853 report
This commit is contained in:
cayop
GitHub
commit
c784fb7499
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -127,3 +127,8 @@ yarn.lock
|
|||
# ESLint Report
|
||||
eslint_report.json
|
||||
|
||||
modules/
|
||||
tmp/
|
||||
.env*
|
||||
bin/
|
||||
env*
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
import csv
|
||||
|
||||
# import click_spinner
|
||||
# import ereuse_utils.cli
|
||||
from io import StringIO
|
||||
|
||||
from ereuse_devicehub.resources.action import models as evs
|
||||
from ereuse_devicehub.resources.device.models import Placeholder
|
||||
from ereuse_devicehub.resources.documents.device_row import InternalStatsRow
|
||||
|
||||
# import click
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self, app) -> None:
|
||||
super().__init__()
|
||||
self.app = app
|
||||
short_help = 'Creates reports devices and users.'
|
||||
self.app.cli.command('report', short_help=short_help)(self.run)
|
||||
|
||||
def run(self):
|
||||
stats = InternalStatsView()
|
||||
stats.print()
|
||||
|
||||
|
||||
class InternalStatsView:
|
||||
def print(self):
|
||||
query = evs.Action.query.filter(
|
||||
evs.Action.type.in_(
|
||||
(
|
||||
'Snapshot',
|
||||
'Live',
|
||||
'Allocate',
|
||||
'Deallocate',
|
||||
'EraseBasic',
|
||||
'EraseSectors',
|
||||
)
|
||||
)
|
||||
)
|
||||
return self.generate_post_csv(query)
|
||||
|
||||
def generate_post_csv(self, query):
|
||||
data = StringIO()
|
||||
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"')
|
||||
cw.writerow(InternalStatsRow('', "2000-1", [], []).keys())
|
||||
|
||||
for row in self.get_rows(query):
|
||||
cw.writerow(row)
|
||||
|
||||
return print(data.getvalue())
|
||||
|
||||
def get_rows(self, query):
|
||||
d = {}
|
||||
dd = {}
|
||||
disks = []
|
||||
for ac in query:
|
||||
create = '{}-{}'.format(ac.created.year, ac.created.month)
|
||||
user = ac.author.email
|
||||
|
||||
if user not in d:
|
||||
d[user] = {}
|
||||
dd[user] = {}
|
||||
if create not in d[user]:
|
||||
d[user][create] = []
|
||||
dd[user][create] = None
|
||||
d[user][create].append(ac)
|
||||
|
||||
for user, createds in d.items():
|
||||
for create, actions in createds.items():
|
||||
r = InternalStatsRow(user, create, actions, disks)
|
||||
dd[user][create] = r
|
||||
|
||||
return self.get_placeholders(dd)
|
||||
|
||||
def get_placeholders(self, dd):
|
||||
|
||||
for p in Placeholder.query.all():
|
||||
create = '{}-{}'.format(p.created.year, p.created.month)
|
||||
user = p.owner.email
|
||||
|
||||
if user not in dd:
|
||||
dd[user] = {}
|
||||
|
||||
if create not in dd[user]:
|
||||
dd[user][create] = None
|
||||
|
||||
if not dd[user][create]:
|
||||
dd[user][create] = InternalStatsRow(user, create, [], [])
|
||||
|
||||
dd[user][create]['Placeholders'] += 1
|
||||
|
||||
rows = []
|
||||
for user, createds in dd.items():
|
||||
for create, row in createds.items():
|
||||
rows.append(row.values())
|
||||
return rows
|
||||
|
|
@ -15,7 +15,8 @@ from teal.teal import Teal
|
|||
|
||||
from ereuse_devicehub.auth import Auth
|
||||
from ereuse_devicehub.client import Client, UserClient
|
||||
from ereuse_devicehub.commands.reports import Report
|
||||
|
||||
# from ereuse_devicehub.commands.reports import Report
|
||||
from ereuse_devicehub.commands.users import GetToken
|
||||
from ereuse_devicehub.config import DevicehubConfig
|
||||
from ereuse_devicehub.db import db
|
||||
|
|
@ -29,7 +30,7 @@ from ereuse_devicehub.templating import Environment
|
|||
class Devicehub(Teal):
|
||||
test_client_class = Client
|
||||
Dummy = Dummy
|
||||
Report = Report
|
||||
# Report = Report
|
||||
jinja_environment = Environment
|
||||
|
||||
def __init__(
|
||||
|
|
@ -70,7 +71,7 @@ class Devicehub(Teal):
|
|||
self.id = inventory
|
||||
"""The Inventory ID of this instance. In Teal is the app.schema."""
|
||||
self.dummy = Dummy(self)
|
||||
self.report = Report(self)
|
||||
# self.report = Report(self)
|
||||
self.get_token = GetToken(self)
|
||||
|
||||
@self.cli.group(
|
||||
|
|
|
|||
|
|
@ -614,104 +614,3 @@ class ActionRow(OrderedDict):
|
|||
self['Type'] = allocate['type']
|
||||
self['LiveCreate'] = allocate['liveCreate']
|
||||
self['UsageTimeHdd'] = allocate['usageTimeHdd']
|
||||
|
||||
|
||||
class InternalStatsRow(OrderedDict):
|
||||
def __init__(self, user, create, actions, disks):
|
||||
super().__init__()
|
||||
# General information about all internal stats
|
||||
# user, quart, month, year:
|
||||
# Snapshot (Registers)
|
||||
# Snapshots (Update)
|
||||
# Snapshots (All)
|
||||
# Drives Erasure
|
||||
# Drives Erasure Uniques
|
||||
# Placeholders
|
||||
# Allocate
|
||||
# Deallocate
|
||||
# Live
|
||||
self.actions = actions
|
||||
year, month = create.split('-')
|
||||
self.disks = disks
|
||||
|
||||
self['User'] = user
|
||||
self['Year'] = year
|
||||
self['Quarter'] = self.quarter(month)
|
||||
self['Month'] = month
|
||||
self['Snapshot (Registers)'] = 0
|
||||
self['Snapshot (Update)'] = 0
|
||||
self['Snapshot (All)'] = 0
|
||||
self['Drives Erasure'] = 0
|
||||
self['Drives Erasure Uniques'] = 0
|
||||
self['Placeholders'] = 0
|
||||
self['Allocates'] = 0
|
||||
self['Deallocates'] = 0
|
||||
self['Lives'] = 0
|
||||
|
||||
self.count_actions()
|
||||
|
||||
def count_actions(self):
|
||||
for ac in self.actions:
|
||||
self.is_snapshot(
|
||||
self.is_deallocate(self.is_live(self.is_allocate(self.is_erase(ac))))
|
||||
)
|
||||
|
||||
def is_allocate(self, ac):
|
||||
if ac.type == 'Allocate':
|
||||
self['Allocates'] += 1
|
||||
return ac
|
||||
|
||||
def is_erase(self, ac):
|
||||
if ac.type in ['EraseBasic', 'EraseSectors']:
|
||||
self['Drives Erasure'] += 1
|
||||
if ac.device in self.disks:
|
||||
return ac
|
||||
self['Drives Erasure Uniques'] += 1
|
||||
self.disks.append(ac.device)
|
||||
return ac
|
||||
|
||||
def is_live(self, ac):
|
||||
if ac.type == 'Live':
|
||||
self['Lives'] += 1
|
||||
return ac
|
||||
|
||||
def is_deallocate(self, ac):
|
||||
if ac.type == 'Deallocate':
|
||||
self['Deallocates'] += 1
|
||||
return ac
|
||||
|
||||
def is_snapshot(self, ac):
|
||||
if not ac.type == 'Snapshot':
|
||||
return
|
||||
self['Snapshot (All)'] += 1
|
||||
|
||||
canary = False
|
||||
for _ac in ac.device.actions:
|
||||
if not _ac.type == 'Snapshot':
|
||||
continue
|
||||
|
||||
if _ac.created < ac.created:
|
||||
canary = True
|
||||
break
|
||||
|
||||
if canary:
|
||||
self['Snapshot (Update)'] += 1
|
||||
else:
|
||||
self['Snapshot (Registers)'] += 1
|
||||
|
||||
def quarter(self, month):
|
||||
q = {
|
||||
1: 'Q1',
|
||||
2: 'Q1',
|
||||
3: 'Q1',
|
||||
4: 'Q2',
|
||||
5: 'Q2',
|
||||
6: 'Q2',
|
||||
7: 'Q3',
|
||||
8: 'Q3',
|
||||
9: 'Q3',
|
||||
10: 'Q4',
|
||||
11: 'Q4',
|
||||
12: 'Q4',
|
||||
}
|
||||
return q[int(month)]
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ from ereuse_devicehub.resources.device.views import DeviceView
|
|||
from ereuse_devicehub.resources.documents.device_row import (
|
||||
ActionRow,
|
||||
DeviceRow,
|
||||
InternalStatsRow,
|
||||
StockRow,
|
||||
)
|
||||
from ereuse_devicehub.resources.enums import SessionType
|
||||
|
|
@ -62,9 +61,9 @@ class DocumentView(DeviceView):
|
|||
200:
|
||||
description: Return the collection or the specific one.
|
||||
"""
|
||||
args = self.QUERY_PARSER.parse(self.find_args,
|
||||
flask.request,
|
||||
locations=('querystring',))
|
||||
args = self.QUERY_PARSER.parse(
|
||||
self.find_args, flask.request, locations=('querystring',)
|
||||
)
|
||||
ids = []
|
||||
if 'filter' in request.args:
|
||||
filters = json.loads(request.args.get('filter', {}))
|
||||
|
|
@ -113,7 +112,8 @@ class DocumentView(DeviceView):
|
|||
template = self.erasure(query)
|
||||
if args.get('format') == Format.PDF:
|
||||
res = flask_weasyprint.render_pdf(
|
||||
flask_weasyprint.HTML(string=template), download_filename='{}.pdf'.format(type)
|
||||
flask_weasyprint.HTML(string=template),
|
||||
download_filename='{}.pdf'.format(type),
|
||||
)
|
||||
insert_hash(res.data)
|
||||
else:
|
||||
|
|
@ -140,7 +140,7 @@ class DocumentView(DeviceView):
|
|||
params = {
|
||||
'title': 'Erasure Certificate',
|
||||
'erasures': tuple(erasures()),
|
||||
'url_pdf': url_pdf.to_text()
|
||||
'url_pdf': url_pdf.to_text(),
|
||||
}
|
||||
return flask.render_template('documents/erasure.html', **params)
|
||||
|
||||
|
|
@ -159,7 +159,13 @@ class DevicesDocumentView(DeviceView):
|
|||
def generate_post_csv(self, query):
|
||||
"""Get device query and put information in csv format."""
|
||||
data = StringIO()
|
||||
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
cw = csv.writer(
|
||||
data,
|
||||
delimiter=';',
|
||||
lineterminator="\n",
|
||||
quotechar='"',
|
||||
quoting=csv.QUOTE_ALL,
|
||||
)
|
||||
first = True
|
||||
document_ids = self.get_documents_id()
|
||||
for device in query:
|
||||
|
|
@ -193,7 +199,13 @@ class ActionsDocumentView(DeviceView):
|
|||
def generate_post_csv(self, query):
|
||||
"""Get device query and put information in csv format."""
|
||||
data = StringIO()
|
||||
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
cw = csv.writer(
|
||||
data,
|
||||
delimiter=';',
|
||||
lineterminator="\n",
|
||||
quotechar='"',
|
||||
quoting=csv.QUOTE_ALL,
|
||||
)
|
||||
first = True
|
||||
devs_id = []
|
||||
for device in query:
|
||||
|
|
@ -204,7 +216,9 @@ class ActionsDocumentView(DeviceView):
|
|||
cw.writerow(d.keys())
|
||||
first = False
|
||||
cw.writerow(d.values())
|
||||
query_trade = Trade.query.filter(Trade.devices.any(Device.id.in_(devs_id))).all()
|
||||
query_trade = Trade.query.filter(
|
||||
Trade.devices.any(Device.id.in_(devs_id))
|
||||
).all()
|
||||
|
||||
lot_id = request.args.get('lot')
|
||||
if lot_id and not query_trade:
|
||||
|
|
@ -225,7 +239,9 @@ class ActionsDocumentView(DeviceView):
|
|||
bfile = data.getvalue().encode('utf-8')
|
||||
output = make_response(bfile)
|
||||
insert_hash(bfile)
|
||||
output.headers['Content-Disposition'] = 'attachment; filename=actions_export.csv'
|
||||
output.headers[
|
||||
'Content-Disposition'
|
||||
] = 'attachment; filename=actions_export.csv'
|
||||
output.headers['Content-type'] = 'text/csv'
|
||||
return output
|
||||
|
||||
|
|
@ -277,7 +293,13 @@ class StockDocumentView(DeviceView):
|
|||
def generate_post_csv(self, query):
|
||||
"""Get device query and put information in csv format."""
|
||||
data = StringIO()
|
||||
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"', quoting=csv.QUOTE_ALL)
|
||||
cw = csv.writer(
|
||||
data,
|
||||
delimiter=';',
|
||||
lineterminator="\n",
|
||||
quotechar='"',
|
||||
quoting=csv.QUOTE_ALL,
|
||||
)
|
||||
first = True
|
||||
for device in query:
|
||||
d = StockRow(device)
|
||||
|
|
@ -311,6 +333,7 @@ class StampsView(View):
|
|||
This view render one public ans static page for see the links for to do the check
|
||||
of one csv file
|
||||
"""
|
||||
|
||||
def get_url_path(self):
|
||||
url = urlutils.URL(request.url)
|
||||
url.normalize()
|
||||
|
|
@ -319,8 +342,9 @@ class StampsView(View):
|
|||
|
||||
def get(self):
|
||||
result = ('', '')
|
||||
return flask.render_template('documents/stamp.html', rq_url=self.get_url_path(),
|
||||
result=result)
|
||||
return flask.render_template(
|
||||
'documents/stamp.html', rq_url=self.get_url_path(), result=result
|
||||
)
|
||||
|
||||
def post(self):
|
||||
result = ('', '')
|
||||
|
|
@ -331,54 +355,26 @@ class StampsView(View):
|
|||
ok = '100% coincidence. The attached file contains data 100% existing in \
|
||||
to our backend'
|
||||
result = ('Bad', bad)
|
||||
mime = ['text/csv', 'application/pdf', 'text/plain', 'text/markdown',
|
||||
'image/jpeg', 'image/png', 'text/html',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'application/vnd.oasis.opendocument.spreadsheet',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/msword']
|
||||
mime = [
|
||||
'text/csv',
|
||||
'application/pdf',
|
||||
'text/plain',
|
||||
'text/markdown',
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'text/html',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||
'application/vnd.oasis.opendocument.spreadsheet',
|
||||
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||
'application/msword',
|
||||
]
|
||||
if file_check.mimetype in mime:
|
||||
if verify_hash(file_check):
|
||||
result = ('Ok', ok)
|
||||
|
||||
return flask.render_template('documents/stamp.html', rq_url=self.get_url_path(),
|
||||
result=result)
|
||||
|
||||
|
||||
class InternalStatsView(DeviceView):
|
||||
@cache(datetime.timedelta(minutes=1))
|
||||
def find(self, args: dict):
|
||||
if not g.user.email == app.config['EMAIL_ADMIN']:
|
||||
return jsonify('')
|
||||
query = evs.Action.query.filter(
|
||||
evs.Action.type.in_(('Snapshot', 'Live', 'Allocate', 'Deallocate')))
|
||||
return self.generate_post_csv(query)
|
||||
|
||||
def generate_post_csv(self, query):
|
||||
d = {}
|
||||
for ac in query:
|
||||
create = '{}-{}'.format(ac.created.year, ac.created.month)
|
||||
user = ac.author.email
|
||||
|
||||
if user not in d:
|
||||
d[user] = {}
|
||||
if create not in d[user]:
|
||||
d[user][create] = []
|
||||
d[user][create].append(ac)
|
||||
|
||||
data = StringIO()
|
||||
cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"')
|
||||
cw.writerow(InternalStatsRow('', "2000-1", []).keys())
|
||||
for user, createds in d.items():
|
||||
for create, actions in createds.items():
|
||||
cw.writerow(InternalStatsRow(user, create, actions).values())
|
||||
|
||||
bfile = data.getvalue().encode('utf-8')
|
||||
output = make_response(bfile)
|
||||
insert_hash(bfile)
|
||||
output.headers['Content-Disposition'] = 'attachment; filename=internal-stats.csv'
|
||||
output.headers['Content-type'] = 'text/csv'
|
||||
return output
|
||||
return flask.render_template(
|
||||
'documents/stamp.html', rq_url=self.get_url_path(), result=result
|
||||
)
|
||||
|
||||
|
||||
class WbConfDocumentView(DeviceView):
|
||||
|
|
@ -386,10 +382,11 @@ class WbConfDocumentView(DeviceView):
|
|||
if not wbtype.lower() in ['usodyrate', 'usodywipe']:
|
||||
return jsonify('')
|
||||
|
||||
data = {'token': self.get_token(),
|
||||
'host': app.config['HOST'],
|
||||
'inventory': app.config['SCHEMA']
|
||||
}
|
||||
data = {
|
||||
'token': self.get_token(),
|
||||
'host': app.config['HOST'],
|
||||
'inventory': app.config['SCHEMA'],
|
||||
}
|
||||
data['erase'] = False
|
||||
# data['erase'] = True if wbtype == 'usodywipe' else False
|
||||
|
||||
|
|
@ -424,18 +421,31 @@ class DocumentDef(Resource):
|
|||
VIEW = None # We do not want to create default / documents endpoint
|
||||
AUTH = False
|
||||
|
||||
def __init__(self, app,
|
||||
import_name=__name__,
|
||||
static_folder='static',
|
||||
static_url_path=None,
|
||||
template_folder='templates',
|
||||
url_prefix=None,
|
||||
subdomain=None,
|
||||
url_defaults=None,
|
||||
root_path=None,
|
||||
cli_commands: Iterable[Tuple[Callable, str or None]] = tuple()):
|
||||
super().__init__(app, import_name, static_folder, static_url_path, template_folder,
|
||||
url_prefix, subdomain, url_defaults, root_path, cli_commands)
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
import_name=__name__,
|
||||
static_folder='static',
|
||||
static_url_path=None,
|
||||
template_folder='templates',
|
||||
url_prefix=None,
|
||||
subdomain=None,
|
||||
url_defaults=None,
|
||||
root_path=None,
|
||||
cli_commands: Iterable[Tuple[Callable, str or None]] = tuple(),
|
||||
):
|
||||
super().__init__(
|
||||
app,
|
||||
import_name,
|
||||
static_folder,
|
||||
static_url_path,
|
||||
template_folder,
|
||||
url_prefix,
|
||||
subdomain,
|
||||
url_defaults,
|
||||
root_path,
|
||||
cli_commands,
|
||||
)
|
||||
d = {'id': None}
|
||||
get = {'GET'}
|
||||
|
||||
|
|
@ -446,12 +456,15 @@ class DocumentDef(Resource):
|
|||
view = app.auth.requires_auth(view)
|
||||
|
||||
self.add_url_rule('/erasures/', defaults=d, view_func=view, methods=get)
|
||||
self.add_url_rule('/erasures/<{}:{}>'.format(self.ID_CONVERTER.value, self.ID_NAME),
|
||||
view_func=view, methods=get)
|
||||
self.add_url_rule(
|
||||
'/erasures/<{}:{}>'.format(self.ID_CONVERTER.value, self.ID_NAME),
|
||||
view_func=view,
|
||||
methods=get,
|
||||
)
|
||||
|
||||
devices_view = DevicesDocumentView.as_view('devicesDocumentView',
|
||||
definition=self,
|
||||
auth=app.auth)
|
||||
devices_view = DevicesDocumentView.as_view(
|
||||
'devicesDocumentView', definition=self, auth=app.auth
|
||||
)
|
||||
devices_view = app.auth.requires_auth(devices_view)
|
||||
|
||||
stock_view = StockDocumentView.as_view('stockDocumentView', definition=self)
|
||||
|
|
@ -463,7 +476,9 @@ class DocumentDef(Resource):
|
|||
lots_view = app.auth.requires_auth(lots_view)
|
||||
self.add_url_rule('/lots/', defaults=d, view_func=lots_view, methods=get)
|
||||
|
||||
stock_view = StockDocumentView.as_view('stockDocumentView', definition=self, auth=app.auth)
|
||||
stock_view = StockDocumentView.as_view(
|
||||
'stockDocumentView', definition=self, auth=app.auth
|
||||
)
|
||||
stock_view = app.auth.requires_auth(stock_view)
|
||||
self.add_url_rule('/stock/', defaults=d, view_func=stock_view, methods=get)
|
||||
|
||||
|
|
@ -471,22 +486,18 @@ class DocumentDef(Resource):
|
|||
self.add_url_rule('/check/', defaults={}, view_func=check_view, methods=get)
|
||||
|
||||
stamps_view = StampsView.as_view('StampsView', definition=self, auth=app.auth)
|
||||
self.add_url_rule('/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'})
|
||||
self.add_url_rule(
|
||||
'/stamps/', defaults={}, view_func=stamps_view, methods={'GET', 'POST'}
|
||||
)
|
||||
|
||||
# internalstats_view = InternalStatsView.as_view(
|
||||
# 'InternalStatsView', definition=self, auth=app.auth)
|
||||
# internalstats_view = app.auth.requires_auth(internalstats_view)
|
||||
# self.add_url_rule('/internalstats/', defaults=d, view_func=internalstats_view,
|
||||
# methods=get)
|
||||
|
||||
actions_view = ActionsDocumentView.as_view('ActionsDocumentView',
|
||||
definition=self,
|
||||
auth=app.auth)
|
||||
actions_view = ActionsDocumentView.as_view(
|
||||
'ActionsDocumentView', definition=self, auth=app.auth
|
||||
)
|
||||
actions_view = app.auth.requires_auth(actions_view)
|
||||
self.add_url_rule('/actions/', defaults=d, view_func=actions_view, methods=get)
|
||||
|
||||
wbconf_view = WbConfDocumentView.as_view('WbConfDocumentView',
|
||||
definition=self,
|
||||
auth=app.auth)
|
||||
wbconf_view = WbConfDocumentView.as_view(
|
||||
'WbConfDocumentView', definition=self, auth=app.auth
|
||||
)
|
||||
wbconf_view = app.auth.requires_auth(wbconf_view)
|
||||
self.add_url_rule('/wbconf/<string:wbtype>', view_func=wbconf_view, methods=get)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import datetime
|
||||
import pytest
|
||||
import json
|
||||
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from flask import g
|
||||
from flask.testing import FlaskClient
|
||||
from flask_wtf.csrf import generate_csrf
|
||||
|
|
|
|||
Reference in a new issue