From 64ef1dcf5701c88283fb785a0b065f7f41202c12 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Tue, 6 Jul 2021 16:03:03 +0200
Subject: [PATCH 01/26] test erase from web

---
 tests/test_action.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test_action.py b/tests/test_action.py
index 5c0af44f..95690800 100644
--- a/tests/test_action.py
+++ b/tests/test_action.py
@@ -2415,3 +2415,9 @@ def test_trade_case14(user: UserClient, user2: UserClient):
     assert device.actions[-4].user == trade.user_from
     assert device.actions[-5].t == 'Trade'
     assert device.actions[-5].author == trade.user_to
+
+
+@pytest.mark.mvp
+@pytest.mark.usefixtures(conftest.app_context.__name__)
+def test_action_web_erase(user: UserClient, user2: UserClient):
+    {'type': 'ToErased', 'devices': [174], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': 'fedbcbd057d25df9915ca9758b7537794148b896b66b3bbc972fe966dcced34b'}

From 740783b9fd36e1b8fe0a81276bfa4cee1ab137fe Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Wed, 21 Jul 2021 08:35:18 +0200
Subject: [PATCH 02/26] adding generic documents model

---
 .../versions/7ecb8ff7abad_documents.py        | 73 +++++++++++++++++++
 ereuse_devicehub/resources/action/models.py   | 17 +++++
 ereuse_devicehub/resources/action/schemas.py  |  4 +
 .../resources/action/views/documents.py       | 63 ++++++++++++++++
 .../resources/action/views/views.py           |  5 ++
 .../resources/documents/models.py             | 47 ++++++++++++
 .../resources/documents/schemas.py            | 26 +++++++
 7 files changed, 235 insertions(+)
 create mode 100644 ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
 create mode 100644 ereuse_devicehub/resources/action/views/documents.py
 create mode 100644 ereuse_devicehub/resources/documents/models.py
 create mode 100644 ereuse_devicehub/resources/documents/schemas.py

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
new file mode 100644
index 00000000..163da39b
--- /dev/null
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -0,0 +1,73 @@
+"""documents
+
+Revision ID: 7ecb8ff7abad
+Revises: 3a3601ac8224
+Create Date: 2021-07-19 14:46:42.375331
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+import citext
+import teal
+
+from alembic import op
+from alembic import context
+from sqlalchemy.dialects import postgresql
+
+
+# revision identifiers, used by Alembic.
+revision = '7ecb8ff7abad'
+down_revision = '3a3601ac8224'
+branch_labels = None
+depends_on = None
+
+
+def get_inv():
+    INV = context.get_x_argument(as_dictionary=True).get('inventory')
+    if not INV:
+        raise ValueError("Inventory value is not specified")
+    return INV
+
+def upgrade():
+    # Document table
+    op.create_table('document',
+                    sa.Column('id', sa.BigInteger(), nullable=False),
+                    sa.Column('updated', sa.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'),
+                              nullable=False,
+                              comment='The last time Document recorded a change for \n    this thing.\n    '),
+                    sa.Column('created', sa.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'),
+                              nullable=False, comment='When Document created this.'),
+                    sa.Column('type', sa.Unicode(), nullable=False),
+                    sa.Column('date', sa.TIMESTAMP(timezone=True), nullable=True),
+                    sa.Column('id_document', sa.Unicode(), nullable=True),
+                    sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=False),
+                    sa.Column('file_name', sa.Unicode(), nullable=False),
+                    sa.Column('file_hash', sa.Unicode(), nullable=False),
+                    sa.Column('url', sa.Unicode(), nullable=False),
+
+                    sa.ForeignKeyConstraint(['owner_id'], ['common.user.id'], ),
+                    sa.PrimaryKeyConstraint('id'),
+                    schema=f'{get_inv()}'
+                    )
+    op.create_index('generic_document_id', 'document', ['id'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
+    op.create_index(op.f('ix_document_created'), 'document', ['created'], unique=False, schema=f'{get_inv()}')
+    op.create_index(op.f('ix_document_updated'), 'document', ['updated'], unique=False, schema=f'{get_inv()}')
+    op.create_index('document_type_index', 'document', ['type'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
+
+
+    # ToErased table
+    op.create_table('to_erased',
+                    # sa.Column('document_id', sa.BigInteger(), nullable=True),
+                    sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
+                    # sa.ForeignKeyConstraint(['document_id'], [f'{get_inv()}.document.id'], ),
+                    sa.ForeignKeyConstraint(['id'], [f'{get_inv()}.action.id'], ),
+                    sa.PrimaryKeyConstraint('id'),
+                    schema=f'{get_inv()}'
+                    )
+
+
+
+def downgrade():
+    op.drop_table('to_erased', schema=f'{get_inv()}')
+    op.drop_table('document', schema=f'{get_inv()}')
diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py
index d8bcb3ce..f9dfd45d 100644
--- a/ereuse_devicehub/resources/action/models.py
+++ b/ereuse_devicehub/resources/action/models.py
@@ -1327,6 +1327,23 @@ class ToPrepare(ActionWithMultipleDevices):
     pass
 
 
+class ToErased(ActionWithMultipleDevices):
+    """The device has been selected for insert one proof of erease disk.
+    """
+    document_comment = """The user that gets the device due this deal."""
+    # document_id = db.Column(BigInteger,
+                       # db.ForeignKey('document.id',
+                                     # use_alter=True,
+                                     # name='document'),
+                       # nullable=False)
+    # document = relationship('EraseDocument',
+                       # backref=backref('actions',
+                                       # lazy=True,
+                                       # uselist=False,
+                                       # cascade=CASCADE_OWN),
+                       # primaryjoin='ToErased.document_id == EraseDocument.id')
+
+
 class Prepare(ActionWithMultipleDevices):
     """Work has been performed to the device to a defined point of
     acceptance.
diff --git a/ereuse_devicehub/resources/action/schemas.py b/ereuse_devicehub/resources/action/schemas.py
index 087af53d..f667eebc 100644
--- a/ereuse_devicehub/resources/action/schemas.py
+++ b/ereuse_devicehub/resources/action/schemas.py
@@ -430,6 +430,10 @@ class Prepare(ActionWithMultipleDevices):
     __doc__ = m.Prepare.__doc__
 
 
+class ToErased(ActionWithMultipleDevices):
+    __doc__ = m.ToErased.__doc__
+
+
 class Live(ActionWithOneDevice):
     __doc__ = m.Live.__doc__
     """
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
new file mode 100644
index 00000000..58d7b579
--- /dev/null
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -0,0 +1,63 @@
+import copy
+
+from flask import g
+from sqlalchemy.util import OrderedSet
+from teal.marshmallow import ValidationError
+
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.action.models import (Trade, Confirm, ConfirmRevoke, 
+                                                      Revoke, RevokeDocument, ConfirmDocument,
+                                                      ConfirmRevokeDocument)
+from ereuse_devicehub.resources.user.models import User
+from ereuse_devicehub.resources.action.models import ToErased
+from ereuse_devicehub.resources.documents.models import EraseDocument
+from ereuse_devicehub.resources.documents.schemas import EraseDocument as sh_document
+
+
+class ErasedView():
+    """Handler for manager the trade action register from post
+
+       request_post = {
+           'type': 'Trade',
+           'devices': [device_id],
+           'documents': [document_id],
+           'userFrom': user2.email,
+           'userTo': user.email,
+           'price': 10,
+           'date': "2020-12-01T02:00:00+00:00",
+           'lot': lot['id'],
+           'confirm': True,
+       }
+
+    """
+
+    def __init__(self, data, schema):
+        self.schema = schema
+        self.insert_document(copy.copy(data))
+        self.insert_action(copy.copy(data))
+
+    def post(self):
+        # import pdb; pdb.set_trace()
+        db.session().final_flush()
+        ret = self.schema.jsonify(self.erased)
+        ret.status_code = 201
+        db.session.commit()
+        return ret
+
+    def insert_document(self, data):
+        # import pdb; pdb.set_trace()
+        schema = sh_document()
+        [data.pop(x) for x in ['severity', 'devices', 'name', 'description']]
+        doc_data = schema.load(data)
+        doc_data['type'] = 'ToErased'
+        self.document = EraseDocument(**doc_data)
+        db.session.add(self.document)
+        db.session.commit()
+
+    def insert_action(self, data):
+        import pdb; pdb.set_trace()
+        [data.pop(x, None) for x in ['url', 'documentId', 'filename', 'hash']]
+        self.data = self.schema.load(data)
+        # self.data['document'] = self.document
+        self.erased = ToErased(**self.data)
+        db.session.add(self.erased)
diff --git a/ereuse_devicehub/resources/action/views/views.py b/ereuse_devicehub/resources/action/views/views.py
index e51a8e7a..c42089a2 100644
--- a/ereuse_devicehub/resources/action/views/views.py
+++ b/ereuse_devicehub/resources/action/views/views.py
@@ -18,6 +18,7 @@ from ereuse_devicehub.resources.action.models import (Action, Snapshot, VisualTe
                                                       Trade, Confirm, ConfirmRevoke, Revoke)
 from ereuse_devicehub.resources.action.views import trade as trade_view
 from ereuse_devicehub.resources.action.views.snapshot import SnapshotView, save_json, move_json
+from ereuse_devicehub.resources.action.views.documents import ErasedView
 from ereuse_devicehub.resources.device.models import Device, Computer, DataStorage
 from ereuse_devicehub.resources.enums import Severity
 
@@ -250,6 +251,10 @@ class ActionView(View):
             confirm_revoke = trade_view.ConfirmRevokeDocumentView(json, resource_def, self.schema)
             return confirm_revoke.post()
 
+        if json['type'] == 'ToErased':
+            erased = ErasedView(json, resource_def.schema)
+            return erased.post()
+
         a = resource_def.schema.load(json)
         Model = db.Model._decl_class_registry.data[json['type']]()
         action = Model(**a)
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
new file mode 100644
index 00000000..700cf59d
--- /dev/null
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -0,0 +1,47 @@
+from citext import CIText
+from flask import g
+
+from sqlalchemy.dialects.postgresql import UUID
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.user.models import User
+from ereuse_devicehub.resources.models import Thing, STR_SM_SIZE
+
+from sqlalchemy import BigInteger, Column, Sequence, Unicode
+from teal.db import URL
+
+
+class Document(Thing):
+    """This represent a generic document."""
+
+    id = Column(BigInteger, Sequence('device_seq'), primary_key=True)
+    id.comment = """The identifier of the device for this database. Used only
+    internally for software; users should not use this.
+    """
+    type = Column(Unicode(STR_SM_SIZE), nullable=False)
+    date = Column(db.DateTime, nullable=True)
+    date.comment = """The date of document, some documents need to have one date
+    """
+    id_document = Column(CIText(), nullable=True)
+    id_document.comment = """The id of one document like invoice so they can be linked."""
+    owner_id = db.Column(UUID(as_uuid=True),
+                         db.ForeignKey(User.id),
+                         nullable=False,
+                         default=lambda: g.user.id)
+    owner = db.relationship(User, primaryjoin=owner_id == User.id)
+    file_name = Column(db.CIText(), nullable=False)
+    file_name.comment = """This is the name of the file when user up the document."""
+    file_hash = Column(db.CIText(), nullable=False)
+    file_hash.comment = """This is the hash of the file produced from frontend."""
+    url = db.Column(URL(), nullable=False)
+    url.comment = """This is the url where resides the document."""
+
+    def __str__(self) -> str:
+        return '{0.file_name}'.format(self)
+
+
+class EraseDocument(Document):
+    """This represent a document involved in a erase manual action.
+    This represent the proof.
+
+    """
+    pass
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
new file mode 100644
index 00000000..636caf56
--- /dev/null
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -0,0 +1,26 @@
+from marshmallow.fields import DateTime, Integer, validate
+from teal.marshmallow import SanitizedStr, URL
+from ereuse_devicehub.resources.schemas import Thing
+from ereuse_devicehub.resources.documents import models as m
+# from marshmallow import ValidationError, validates_schema
+
+
+class EraseDocument(Thing):
+    __doc__ = m.EraseDocument.__doc__
+    id = Integer(description=m.EraseDocument.id.comment, dump_only=True)
+    type = SanitizedStr(default='EraseDocument')
+    date = DateTime(data_key='endTime', 
+                    required=False, 
+                    description=m.EraseDocument.date.comment)
+    id_document = SanitizedStr(data_key='documentId', 
+                               default='', 
+                               description=m.EraseDocument.id_document.comment)
+    file_name = SanitizedStr(data_key='filename',
+                             default='',
+                             description=m.EraseDocument.file_name.comment, 
+                             validate=validate.Length(max=100))
+    file_hash = SanitizedStr(data_key='hash',
+                             default='',
+                             description=m.EraseDocument.file_hash.comment,
+                             validate=validate.Length(max=64))
+    url = URL(description=m.EraseDocument.url.comment)

From 2ec03d962e175212ee08883f5050940066552df0 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Wed, 21 Jul 2021 08:46:09 +0200
Subject: [PATCH 03/26] change changelog

---
 CHANGELOG.md                                  | 7 +++----
 ereuse_devicehub/resources/action/__init__.py | 5 +++++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01024d2d..fe55f52d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,11 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.ht
 ml).
 
-## master
-  [1.0.7-beta]
+## master [1.0.7-beta]
 
-## testing
-  [1.0.8-beta]
+## testing [1.0.8-beta]
+- [addend] #159 external document as proof of erase of disk
 
 ## [1.0.7-beta]
 - [addend] #158 support for encrypted snapshots data
diff --git a/ereuse_devicehub/resources/action/__init__.py b/ereuse_devicehub/resources/action/__init__.py
index 5e581cd5..2ee1e0da 100644
--- a/ereuse_devicehub/resources/action/__init__.py
+++ b/ereuse_devicehub/resources/action/__init__.py
@@ -199,6 +199,11 @@ class ToPrepareDef(ActionDef):
     SCHEMA = schemas.ToPrepare
 
 
+class ToErasedDef(ActionDef):
+    VIEW = None
+    SCHEMA = schemas.ToErased
+
+
 class AllocateDef(ActionDef):
     VIEW = AllocateView
     SCHEMA = schemas.Allocate

From 811a3c526215f54b61ef426283623e4bfcf34fa8 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Wed, 21 Jul 2021 09:03:26 +0200
Subject: [PATCH 04/26] fixing with bigsIntegers

---
 ...103a9c96b2d_testdatastorage_bigintegers.py | 38 +++++++++++++++++++
 ereuse_devicehub/resources/action/models.py   | 10 ++---
 2 files changed, 43 insertions(+), 5 deletions(-)
 create mode 100644 ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py

diff --git a/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py b/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py
new file mode 100644
index 00000000..87d0626f
--- /dev/null
+++ b/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py
@@ -0,0 +1,38 @@
+"""TestDataStorage_bigIntegers
+
+Revision ID: 0103a9c96b2d
+Revises: 3a3601ac8224
+Create Date: 2021-07-21 08:56:48.342503
+
+"""
+from alembic import op, context
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0103a9c96b2d'
+down_revision = '3a3601ac8224'
+branch_labels = None
+depends_on = None
+
+
+def get_inv():
+    INV = context.get_x_argument(as_dictionary=True).get('inventory')
+    if not INV:
+        raise ValueError("Inventory value is not specified")
+    return INV
+
+def upgrade():
+    op.alter_column('test_data_storage', 'reallocated_sector_count', type_=sa.BigInteger(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'power_cycle_count', type_=sa.Integer(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', '_reported_uncorrectable_errors', type_=sa.BigInteger(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'current_pending_sector_count', type_=sa.BigInteger(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'offline_uncorrectable', type_=sa.BigInteger(), schema=f'{get_inv()}')
+
+
+def downgrade():
+    op.alter_column('test_data_storage', 'reallocated_sector_count', type_=sa.SmallInteger(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'power_cycle_count', type_=sa.SmallInteger(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', '_reported_uncorrectable_errors', type_=sa.Integer(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'current_pending_sector_count', type_=sa.Integer(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'offline_uncorrectable', type_=sa.Integer(), schema=f'{get_inv()}')
diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py
index d8bcb3ce..29b32c00 100644
--- a/ereuse_devicehub/resources/action/models.py
+++ b/ereuse_devicehub/resources/action/models.py
@@ -754,12 +754,12 @@ class TestDataStorage(TestMixin, Test):
     status = Column(Unicode(), check_lower('status'), nullable=False)
     lifetime = Column(Interval)
     assessment = Column(Boolean)
-    reallocated_sector_count = Column(SmallInteger)
-    power_cycle_count = Column(SmallInteger)
-    _reported_uncorrectable_errors = Column('reported_uncorrectable_errors', Integer)
+    reallocated_sector_count = Column(BigInteger)
+    power_cycle_count = Column(Integer)
+    _reported_uncorrectable_errors = Column('reported_uncorrectable_errors', BigInteger)
     command_timeout = Column(BigInteger)
-    current_pending_sector_count = Column(Integer)
-    offline_uncorrectable = Column(Integer)
+    current_pending_sector_count = Column(BigInteger)
+    offline_uncorrectable = Column(BigInteger)
     remaining_lifetime_percentage = Column(SmallInteger)
     elapsed = Column(Interval, nullable=False)
 

From c5e62914b698bad3121c436ce7ebc804af2f7429 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Wed, 21 Jul 2021 15:02:23 +0200
Subject: [PATCH 05/26] try solve problems with foreignkey

---
 .../versions/7ecb8ff7abad_documents.py        |  6 ++--
 ereuse_devicehub/resources/action/models.py   | 28 ++++++++++++-------
 ereuse_devicehub/resources/action/schemas.py  |  2 ++
 .../resources/action/views/documents.py       |  9 ++++--
 .../resources/documents/models.py             |  2 +-
 5 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index 163da39b..9b018f04 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -29,6 +29,7 @@ def get_inv():
         raise ValueError("Inventory value is not specified")
     return INV
 
+
 def upgrade():
     # Document table
     op.create_table('document',
@@ -58,16 +59,15 @@ def upgrade():
 
     # ToErased table
     op.create_table('to_erased',
-                    # sa.Column('document_id', sa.BigInteger(), nullable=True),
+                    sa.Column('document_id', sa.BigInteger(), nullable=False),
                     sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
-                    # sa.ForeignKeyConstraint(['document_id'], [f'{get_inv()}.document.id'], ),
+                    sa.ForeignKeyConstraint(['document_id'], [f'{get_inv()}.document.id'], ),
                     sa.ForeignKeyConstraint(['id'], [f'{get_inv()}.action.id'], ),
                     sa.PrimaryKeyConstraint('id'),
                     schema=f'{get_inv()}'
                     )
 
 
-
 def downgrade():
     op.drop_table('to_erased', schema=f'{get_inv()}')
     op.drop_table('document', schema=f'{get_inv()}')
diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py
index f9dfd45d..81e08074 100644
--- a/ereuse_devicehub/resources/action/models.py
+++ b/ereuse_devicehub/resources/action/models.py
@@ -1332,16 +1332,24 @@ class ToErased(ActionWithMultipleDevices):
     """
     document_comment = """The user that gets the device due this deal."""
     # document_id = db.Column(BigInteger,
-                       # db.ForeignKey('document.id',
-                                     # use_alter=True,
-                                     # name='document'),
-                       # nullable=False)
-    # document = relationship('EraseDocument',
-                       # backref=backref('actions',
-                                       # lazy=True,
-                                       # uselist=False,
-                                       # cascade=CASCADE_OWN),
-                       # primaryjoin='ToErased.document_id == EraseDocument.id')
+                            # db.ForeignKey('document.id'),
+                            # nullable=False)
+    # document = db.relationship('EraseDocument',
+                               # backref=backref('actions',
+                                               # # lazy=True,
+                                               # # uselist=False,
+                                               # # cascade=CASCADE_OWN),
+                                               # uselist=True,
+                                               # lazy=True,
+                                               # order_by=lambda: Action.end_time,
+                                               # collection_class=list),
+                               # primaryjoin='ToErased.document_id == EraseDocument.id')
+    document_id = Column(BigInteger, ForeignKey('document.id'), nullable=False)
+    document = relationship('Document',
+                          backref=backref('document_one',
+                                          lazy=True,
+                                          cascade=CASCADE_OWN),
+                          primaryjoin='ToErased.document_id == Document.id')
 
 
 class Prepare(ActionWithMultipleDevices):
diff --git a/ereuse_devicehub/resources/action/schemas.py b/ereuse_devicehub/resources/action/schemas.py
index f667eebc..1c2eff1b 100644
--- a/ereuse_devicehub/resources/action/schemas.py
+++ b/ereuse_devicehub/resources/action/schemas.py
@@ -432,6 +432,8 @@ class Prepare(ActionWithMultipleDevices):
 
 class ToErased(ActionWithMultipleDevices):
     __doc__ = m.ToErased.__doc__
+    document = NestedOn('Document', only_query='id')
+    # device = NestedOn(s_device.Device, only_query='id')
 
 
 class Live(ActionWithOneDevice):
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index 58d7b579..bb5c03ec 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -52,12 +52,15 @@ class ErasedView():
         doc_data['type'] = 'ToErased'
         self.document = EraseDocument(**doc_data)
         db.session.add(self.document)
-        db.session.commit()
+        # db.session.commit()
 
     def insert_action(self, data):
         import pdb; pdb.set_trace()
         [data.pop(x, None) for x in ['url', 'documentId', 'filename', 'hash']]
-        self.data = self.schema.load(data)
+        # self.data = self.schema.load(data)
+        # self.data['document_id'] = self.document.id
         # self.data['document'] = self.document
-        self.erased = ToErased(**self.data)
+        # data['document_id'] = self.document.id
+        data['document'] = self.document
+        self.erased = ToErased(**data)
         db.session.add(self.erased)
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index 700cf59d..1dbee994 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -21,7 +21,7 @@ class Document(Thing):
     date = Column(db.DateTime, nullable=True)
     date.comment = """The date of document, some documents need to have one date
     """
-    id_document = Column(CIText(), nullable=True)
+    id_document = Column(CIText(), nullable=False)
     id_document.comment = """The id of one document like invoice so they can be linked."""
     owner_id = db.Column(UUID(as_uuid=True),
                          db.ForeignKey(User.id),

From 833b55aefcfe0b942fa062c60453b508a873b287 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Mon, 26 Jul 2021 11:33:11 +0200
Subject: [PATCH 06/26] external document erase

---
 .../versions/7ecb8ff7abad_documents.py        |  2 +-
 ereuse_devicehub/resources/action/models.py   | 23 +++------
 ereuse_devicehub/resources/action/schemas.py  |  4 +-
 .../resources/action/views/documents.py       | 49 ++++++++-----------
 .../resources/documents/models.py             | 14 +-----
 .../resources/documents/schemas.py            | 18 +++----
 6 files changed, 41 insertions(+), 69 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index 9b018f04..deaed24e 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -59,7 +59,7 @@ def upgrade():
 
     # ToErased table
     op.create_table('to_erased',
-                    sa.Column('document_id', sa.BigInteger(), nullable=False),
+                    sa.Column('document_id', sa.BigInteger(), nullable=True),
                     sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
                     sa.ForeignKeyConstraint(['document_id'], [f'{get_inv()}.document.id'], ),
                     sa.ForeignKeyConstraint(['id'], [f'{get_inv()}.action.id'], ),
diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py
index 81e08074..30a91890 100644
--- a/ereuse_devicehub/resources/action/models.py
+++ b/ereuse_devicehub/resources/action/models.py
@@ -1327,26 +1327,15 @@ class ToPrepare(ActionWithMultipleDevices):
     pass
 
 
-class ToErased(ActionWithMultipleDevices):
+class ToErased(JoinedTableMixin, ActionWithMultipleDevices):
     """The device has been selected for insert one proof of erease disk.
     """
     document_comment = """The user that gets the device due this deal."""
-    # document_id = db.Column(BigInteger,
-                            # db.ForeignKey('document.id'),
-                            # nullable=False)
-    # document = db.relationship('EraseDocument',
-                               # backref=backref('actions',
-                                               # # lazy=True,
-                                               # # uselist=False,
-                                               # # cascade=CASCADE_OWN),
-                                               # uselist=True,
-                                               # lazy=True,
-                                               # order_by=lambda: Action.end_time,
-                                               # collection_class=list),
-                               # primaryjoin='ToErased.document_id == EraseDocument.id')
-    document_id = Column(BigInteger, ForeignKey('document.id'), nullable=False)
-    document = relationship('Document',
-                          backref=backref('document_one',
+    document_id = db.Column(BigInteger,
+                            db.ForeignKey('document.id'),
+                            nullable=False)
+    document = db.relationship('Document',
+                          backref=backref('actions',
                                           lazy=True,
                                           cascade=CASCADE_OWN),
                           primaryjoin='ToErased.document_id == Document.id')
diff --git a/ereuse_devicehub/resources/action/schemas.py b/ereuse_devicehub/resources/action/schemas.py
index 1c2eff1b..1a498a36 100644
--- a/ereuse_devicehub/resources/action/schemas.py
+++ b/ereuse_devicehub/resources/action/schemas.py
@@ -17,6 +17,7 @@ from ereuse_devicehub.resources.action import models as m
 from ereuse_devicehub.resources.agent import schemas as s_agent
 from ereuse_devicehub.resources.device import schemas as s_device
 from ereuse_devicehub.resources.tradedocument import schemas as s_document
+from ereuse_devicehub.resources.documents import schemas as s_generic_document
 from ereuse_devicehub.resources.enums import AppearanceRange, BiosAccessRange, FunctionalityRange, \
     PhysicalErasureMethod, R_POSITIVE, RatingRange, \
     Severity, SnapshotSoftware, TestDataStorageLength
@@ -432,7 +433,8 @@ class Prepare(ActionWithMultipleDevices):
 
 class ToErased(ActionWithMultipleDevices):
     __doc__ = m.ToErased.__doc__
-    document = NestedOn('Document', only_query='id')
+    document = NestedOn(s_generic_document.Document, only_query='id')
+    # document = NestedOn('EraseDocument')
     # device = NestedOn(s_device.Device, only_query='id')
 
 
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index bb5c03ec..65c20857 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -10,25 +10,13 @@ from ereuse_devicehub.resources.action.models import (Trade, Confirm, ConfirmRev
                                                       ConfirmRevokeDocument)
 from ereuse_devicehub.resources.user.models import User
 from ereuse_devicehub.resources.action.models import ToErased
-from ereuse_devicehub.resources.documents.models import EraseDocument
-from ereuse_devicehub.resources.documents.schemas import EraseDocument as sh_document
+from ereuse_devicehub.resources.documents.models import Document
+from ereuse_devicehub.resources.device.models import DataStorage 
+from ereuse_devicehub.resources.documents.schemas import Document as sh_document
 
 
 class ErasedView():
-    """Handler for manager the trade action register from post
-
-       request_post = {
-           'type': 'Trade',
-           'devices': [device_id],
-           'documents': [document_id],
-           'userFrom': user2.email,
-           'userTo': user.email,
-           'price': 10,
-           'date': "2020-12-01T02:00:00+00:00",
-           'lot': lot['id'],
-           'confirm': True,
-       }
-
+    """Handler for manager the action register for add to a device one proof of erase
     """
 
     def __init__(self, data, schema):
@@ -37,30 +25,33 @@ class ErasedView():
         self.insert_action(copy.copy(data))
 
     def post(self):
-        # import pdb; pdb.set_trace()
         db.session().final_flush()
-        ret = self.schema.jsonify(self.erased)
+        from flask import jsonify
+        ret = jsonify(self.erased)
         ret.status_code = 201
         db.session.commit()
         return ret
 
     def insert_document(self, data):
-        # import pdb; pdb.set_trace()
         schema = sh_document()
-        [data.pop(x) for x in ['severity', 'devices', 'name', 'description']]
+        [data.pop(x, None) for x in ['severity', 'devices', 'name', 'description']]
         doc_data = schema.load(data)
         doc_data['type'] = 'ToErased'
-        self.document = EraseDocument(**doc_data)
+        self.document = Document(**doc_data)
         db.session.add(self.document)
-        # db.session.commit()
 
     def insert_action(self, data):
-        import pdb; pdb.set_trace()
         [data.pop(x, None) for x in ['url', 'documentId', 'filename', 'hash']]
-        # self.data = self.schema.load(data)
-        # self.data['document_id'] = self.document.id
-        # self.data['document'] = self.document
-        # data['document_id'] = self.document.id
-        data['document'] = self.document
-        self.erased = ToErased(**data)
+        self.data = self.schema.load(data)
+       
+        for dev in self.data['devices']:
+            if not hasattr(dev, 'components'):
+                continue
+
+            for component in dev.components:
+                if isinstance(component, DataStorage):
+                    self.data['devices'].add(component)
+
+        self.data['document'] = self.document
+        self.erased = ToErased(**self.data)
         db.session.add(self.erased)
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index 1dbee994..7ee5eb99 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -1,14 +1,12 @@
 from citext import CIText
 from flask import g
-
+from sqlalchemy import BigInteger, Column, Sequence, Unicode
 from sqlalchemy.dialects.postgresql import UUID
+from teal.db import URL
 from ereuse_devicehub.db import db
 from ereuse_devicehub.resources.user.models import User
 from ereuse_devicehub.resources.models import Thing, STR_SM_SIZE
 
-from sqlalchemy import BigInteger, Column, Sequence, Unicode
-from teal.db import URL
-
 
 class Document(Thing):
     """This represent a generic document."""
@@ -37,11 +35,3 @@ class Document(Thing):
 
     def __str__(self) -> str:
         return '{0.file_name}'.format(self)
-
-
-class EraseDocument(Document):
-    """This represent a document involved in a erase manual action.
-    This represent the proof.
-
-    """
-    pass
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
index 636caf56..21b7017e 100644
--- a/ereuse_devicehub/resources/documents/schemas.py
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -5,22 +5,22 @@ from ereuse_devicehub.resources.documents import models as m
 # from marshmallow import ValidationError, validates_schema
 
 
-class EraseDocument(Thing):
-    __doc__ = m.EraseDocument.__doc__
-    id = Integer(description=m.EraseDocument.id.comment, dump_only=True)
-    type = SanitizedStr(default='EraseDocument')
+class Document(Thing):
+    __doc__ = m.Document.__doc__
+    id = Integer(description=m.Document.id.comment, dump_only=True)
+    type = SanitizedStr(default='Document')
     date = DateTime(data_key='endTime', 
                     required=False, 
-                    description=m.EraseDocument.date.comment)
+                    description=m.Document.date.comment)
     id_document = SanitizedStr(data_key='documentId', 
                                default='', 
-                               description=m.EraseDocument.id_document.comment)
+                               description=m.Document.id_document.comment)
     file_name = SanitizedStr(data_key='filename',
                              default='',
-                             description=m.EraseDocument.file_name.comment, 
+                             description=m.Document.file_name.comment, 
                              validate=validate.Length(max=100))
     file_hash = SanitizedStr(data_key='hash',
                              default='',
-                             description=m.EraseDocument.file_hash.comment,
+                             description=m.Document.file_hash.comment,
                              validate=validate.Length(max=64))
-    url = URL(description=m.EraseDocument.url.comment)
+    url = URL(description=m.Document.url.comment)

From fb4f6f93864aa41ffa806c08aefe983557078863 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Mon, 26 Jul 2021 11:54:05 +0200
Subject: [PATCH 07/26] adding test for erase documents

---
 tests/test_action.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tests/test_action.py b/tests/test_action.py
index 99ef16fe..99d266ab 100644
--- a/tests/test_action.py
+++ b/tests/test_action.py
@@ -2412,5 +2412,12 @@ def test_trade_case14(user: UserClient, user2: UserClient):
 
 @pytest.mark.mvp
 @pytest.mark.usefixtures(conftest.app_context.__name__)
-def test_action_web_erase(user: UserClient, user2: UserClient):
-    {'type': 'ToErased', 'devices': [174], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': 'fedbcbd057d25df9915ca9758b7537794148b896b66b3bbc972fe966dcced34b'}
+def test_action_web_erase(user: UserClient):
+    # import pdb; pdb.set_trace()
+    snap, _ = user.post(file('acer.happy.battery.snapshot'), res=models.Snapshot)
+    request = {'type': 'ToErased', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': 'fedbcbd057d25df9915ca9758b7537794148b896b66b3bbc972fe966dcced34b'}
+   
+    user.post(res=models.Action, data=request)
+    action = models.ToErased.query.one()
+    for dev in action.devices:
+        assert action in dev.actions

From 5e6ecaae49c7808de0822d095a7cb7390b789420 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Mon, 26 Jul 2021 11:55:12 +0200
Subject: [PATCH 08/26] adding test for erase documents

---
 tests/test_action.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/test_action.py b/tests/test_action.py
index 99d266ab..951e91ab 100644
--- a/tests/test_action.py
+++ b/tests/test_action.py
@@ -2421,3 +2421,5 @@ def test_action_web_erase(user: UserClient):
     action = models.ToErased.query.one()
     for dev in action.devices:
         assert action in dev.actions
+
+    assert action.document.file_hash == request['hash']

From 1e32b74b8ff21228079e6a3fb9478d91efa0ad52 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Mon, 26 Jul 2021 14:00:28 +0200
Subject: [PATCH 09/26] fixing test basics

---
 tests/test_basic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_basic.py b/tests/test_basic.py
index 4cdce061..c04e8fb6 100644
--- a/tests/test_basic.py
+++ b/tests/test_basic.py
@@ -122,4 +122,4 @@ def test_api_docs(client: Client):
         'scheme': 'basic',
         'name': 'Authorization'
     }
-    assert len(docs['definitions']) == 125
+    assert len(docs['definitions']) == 126

From bb3145f3d8f8e4fe59b70b4bff92f9baed8bdfdd Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Wed, 28 Jul 2021 15:59:11 +0200
Subject: [PATCH 10/26] fixing test_basic

---
 .../resources/action/views/documents.py       |  4 +++
 ereuse_devicehub/resources/device/models.py   | 30 ++++++++++++++++
 .../resources/documents/documents.py          | 36 ++++++++++++++++++-
 tests/test_action.py                          | 21 +++++++++--
 tests/test_basic.py                           |  1 +
 5 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index 65c20857..c21f9d42 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -13,6 +13,7 @@ from ereuse_devicehub.resources.action.models import ToErased
 from ereuse_devicehub.resources.documents.models import Document
 from ereuse_devicehub.resources.device.models import DataStorage 
 from ereuse_devicehub.resources.documents.schemas import Document as sh_document
+from ereuse_devicehub.resources.hash_reports import ReportHash
 
 
 class ErasedView():
@@ -39,6 +40,9 @@ class ErasedView():
         doc_data['type'] = 'ToErased'
         self.document = Document(**doc_data)
         db.session.add(self.document)
+        
+        db_hash = ReportHash(hash3=self.document.file_hash)
+        db.session.add(db_hash)
 
     def insert_action(self, data):
         [data.pop(x, None) for x in ['url', 'documentId', 'filename', 'hash']]
diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py
index ea585785..35937101 100644
--- a/ereuse_devicehub/resources/device/models.py
+++ b/ereuse_devicehub/resources/device/models.py
@@ -696,6 +696,25 @@ class Computer(Device):
             if privacy
         )
 
+    @property
+    def external_document_erasure(self):
+        """Returns the external ``DataStorage`` proof of erasure.
+        """
+        from ereuse_devicehub.resources.action.models import ToErased
+        urls = set()
+        try:
+            ev = self.last_action_of(ToErased)
+            urls.add(ev.document.url.to_text())
+        except LookupError:
+            pass
+
+        for comp in self.components:
+            if isinstance(comp, DataStorage):
+                doc = comp.external_document_erasure
+                if doc:
+                    urls.add(doc)
+        return urls
+
     def add_mac_to_hid(self, components_snap=None):
         """Returns the Naming.hid with the first mac of network adapter, 
         following an alphabetical order.
@@ -879,6 +898,17 @@ class DataStorage(JoinedComponentTableMixin, Component):
             v += ' – {} GB'.format(self.size // 1000 if self.size else '?')
         return v
 
+    @property
+    def external_document_erasure(self):
+        """Returns the external ``DataStorage`` proof of erasure.
+        """
+        from ereuse_devicehub.resources.action.models import ToErased
+        try:
+            ev = self.last_action_of(ToErased)
+            return ev.document.url.to_text()
+        except LookupError:
+            return None
+
 
 class HardDrive(DataStorage):
     pass
diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py
index 26e94292..8e8db6ff 100644
--- a/ereuse_devicehub/resources/documents/documents.py
+++ b/ereuse_devicehub/resources/documents/documents.py
@@ -90,6 +90,7 @@ class DocumentView(DeviceView):
             res = flask.make_response(template)
         return res
 
+
     @staticmethod
     def erasure(query: db.Query):
         def erasures():
@@ -114,6 +115,34 @@ class DocumentView(DeviceView):
         }
         return flask.render_template('documents/erasure.html', **params)
 
+class ExternalErasureDocumentView(DeviceView):
+    @cache(datetime.timedelta(minutes=1))
+    def find(self, args: dict):
+        query = (x for x in self.query(args) if x.owner_id == g.user.id)
+        return self.generate_post_csv(query)
+
+    def generate_post_csv(self, query):
+        """Get device query and put information in csv format."""
+        data = StringIO()
+        cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"')
+        cw.writerow(['Urls'])
+        for device in query:
+            if isinstance(device, devs.Computer):
+                urls = device.external_document_erasure
+                if urls:
+                    cw.writerow(urls)
+            elif isinstance(device, devs.DataStorage):
+                url = device.external_document_erasure
+                if url:
+                    cw.writerow(set(url))
+                    
+        bfile = data.getvalue().encode('utf-8')
+        output = make_response(bfile)
+        insert_hash(bfile)
+        output.headers['Content-Disposition'] = 'attachment; filename=export_urls_external_proof.csv'
+        output.headers['Content-type'] = 'text/csv'
+        return output
+
 
 class DevicesDocumentView(DeviceView):
     @cache(datetime.timedelta(minutes=1))
@@ -291,7 +320,6 @@ class InternalStatsView(DeviceView):
             evs.Action.type.in_(('Snapshot', 'Live', 'Allocate', 'Deallocate')))
         return self.generate_post_csv(query)
 
-
     def generate_post_csv(self, query):
         d = {}
         for ac in query:
@@ -417,6 +445,12 @@ class DocumentDef(Resource):
         self.add_url_rule('/internalstats/', defaults=d, view_func=internalstats_view,
                           methods=get)
 
+        externalErasureDocument_view = ExternalErasureDocumentView.as_view(
+            'ExternalErasureDocumentView', definition=self, auth=app.auth)
+        externalErasureDocument_view = app.auth.requires_auth(externalErasureDocument_view)
+        self.add_url_rule('/externalErasureDocuments/', defaults=d,
+                          view_func=externalErasureDocument_view, methods=get)
+
         actions_view = ActionsDocumentView.as_view('ActionsDocumentView',
                                                    definition=self,
                                                    auth=app.auth)
diff --git a/tests/test_action.py b/tests/test_action.py
index 951e91ab..bfdcebd2 100644
--- a/tests/test_action.py
+++ b/tests/test_action.py
@@ -6,6 +6,7 @@ import copy
 import pytest
 
 from datetime import datetime, timedelta
+from io import BytesIO
 from dateutil.tz import tzutc
 from decimal import Decimal
 from typing import Tuple, Type
@@ -2412,10 +2413,13 @@ def test_trade_case14(user: UserClient, user2: UserClient):
 
 @pytest.mark.mvp
 @pytest.mark.usefixtures(conftest.app_context.__name__)
-def test_action_web_erase(user: UserClient):
-    # import pdb; pdb.set_trace()
+def test_action_web_erase(user: UserClient, client: Client):
+    import hashlib
+    from ereuse_devicehub.resources.documents import documents
+    bfile = BytesIO(b'abc')
+    hash3 = hashlib.sha3_256(bfile.read()).hexdigest()
     snap, _ = user.post(file('acer.happy.battery.snapshot'), res=models.Snapshot)
-    request = {'type': 'ToErased', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': 'fedbcbd057d25df9915ca9758b7537794148b896b66b3bbc972fe966dcced34b'}
+    request = {'type': 'ToErased', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': hash3}
    
     user.post(res=models.Action, data=request)
     action = models.ToErased.query.one()
@@ -2423,3 +2427,14 @@ def test_action_web_erase(user: UserClient):
         assert action in dev.actions
 
     assert action.document.file_hash == request['hash']
+
+    bfile = BytesIO(b'abc')
+    response, _ = client.post(res=documents.DocumentDef.t,
+            item='stamps/',
+            content_type='multipart/form-data',
+            accept='text/html',
+            data={'docUpload': [(bfile, 'example.csv')]},
+            status=200)
+    assert "alert alert-info" in response
+    assert "100% coincidence." in response
+    assert not "alert alert-danger" in response
diff --git a/tests/test_basic.py b/tests/test_basic.py
index c04e8fb6..18973f6a 100644
--- a/tests/test_basic.py
+++ b/tests/test_basic.py
@@ -40,6 +40,7 @@ def test_api_docs(client: Client):
         '/documents/erasures/',
         '/documents/devices/',
         '/documents/stamps/',
+        '/documents/externalErasureDocuments/',
         '/documents/wbconf/{wbtype}',
         '/documents/internalstats/',
         '/documents/stock/',

From f8ce81cc19f86a492219438f001aabda3a8c259f Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Thu, 29 Jul 2021 11:19:45 +0200
Subject: [PATCH 11/26] drop endpoint

---
 .../resources/documents/documents.py          | 34 -------------------
 tests/test_basic.py                           |  1 -
 2 files changed, 35 deletions(-)

diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py
index 8e8db6ff..f45cf808 100644
--- a/ereuse_devicehub/resources/documents/documents.py
+++ b/ereuse_devicehub/resources/documents/documents.py
@@ -115,34 +115,6 @@ class DocumentView(DeviceView):
         }
         return flask.render_template('documents/erasure.html', **params)
 
-class ExternalErasureDocumentView(DeviceView):
-    @cache(datetime.timedelta(minutes=1))
-    def find(self, args: dict):
-        query = (x for x in self.query(args) if x.owner_id == g.user.id)
-        return self.generate_post_csv(query)
-
-    def generate_post_csv(self, query):
-        """Get device query and put information in csv format."""
-        data = StringIO()
-        cw = csv.writer(data, delimiter=';', lineterminator="\n", quotechar='"')
-        cw.writerow(['Urls'])
-        for device in query:
-            if isinstance(device, devs.Computer):
-                urls = device.external_document_erasure
-                if urls:
-                    cw.writerow(urls)
-            elif isinstance(device, devs.DataStorage):
-                url = device.external_document_erasure
-                if url:
-                    cw.writerow(set(url))
-                    
-        bfile = data.getvalue().encode('utf-8')
-        output = make_response(bfile)
-        insert_hash(bfile)
-        output.headers['Content-Disposition'] = 'attachment; filename=export_urls_external_proof.csv'
-        output.headers['Content-type'] = 'text/csv'
-        return output
-
 
 class DevicesDocumentView(DeviceView):
     @cache(datetime.timedelta(minutes=1))
@@ -445,12 +417,6 @@ class DocumentDef(Resource):
         self.add_url_rule('/internalstats/', defaults=d, view_func=internalstats_view,
                           methods=get)
 
-        externalErasureDocument_view = ExternalErasureDocumentView.as_view(
-            'ExternalErasureDocumentView', definition=self, auth=app.auth)
-        externalErasureDocument_view = app.auth.requires_auth(externalErasureDocument_view)
-        self.add_url_rule('/externalErasureDocuments/', defaults=d,
-                          view_func=externalErasureDocument_view, methods=get)
-
         actions_view = ActionsDocumentView.as_view('ActionsDocumentView',
                                                    definition=self,
                                                    auth=app.auth)
diff --git a/tests/test_basic.py b/tests/test_basic.py
index 18973f6a..c04e8fb6 100644
--- a/tests/test_basic.py
+++ b/tests/test_basic.py
@@ -40,7 +40,6 @@ def test_api_docs(client: Client):
         '/documents/erasures/',
         '/documents/devices/',
         '/documents/stamps/',
-        '/documents/externalErasureDocuments/',
         '/documents/wbconf/{wbtype}',
         '/documents/internalstats/',
         '/documents/stock/',

From bfe56a8626907e6f22e0e115782e369b527d6dc7 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Thu, 29 Jul 2021 12:45:43 +0200
Subject: [PATCH 12/26] DataWipe instead of ToErased

---
 .../migrations/versions/7ecb8ff7abad_documents.py         | 6 +++---
 ereuse_devicehub/resources/action/__init__.py             | 4 ++--
 ereuse_devicehub/resources/action/models.py               | 4 ++--
 ereuse_devicehub/resources/action/schemas.py              | 6 ++----
 ereuse_devicehub/resources/action/views/documents.py      | 6 +++---
 ereuse_devicehub/resources/action/views/views.py          | 2 +-
 ereuse_devicehub/resources/device/models.py               | 8 ++++----
 tests/test_action.py                                      | 4 ++--
 8 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index deaed24e..730e8f8a 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -57,8 +57,8 @@ def upgrade():
     op.create_index('document_type_index', 'document', ['type'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
 
 
-    # ToErased table
-    op.create_table('to_erased',
+    # DataWipe table
+    op.create_table('data_wipe',
                     sa.Column('document_id', sa.BigInteger(), nullable=True),
                     sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
                     sa.ForeignKeyConstraint(['document_id'], [f'{get_inv()}.document.id'], ),
@@ -69,5 +69,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_table('to_erased', schema=f'{get_inv()}')
+    op.drop_table('data_wipe', schema=f'{get_inv()}')
     op.drop_table('document', schema=f'{get_inv()}')
diff --git a/ereuse_devicehub/resources/action/__init__.py b/ereuse_devicehub/resources/action/__init__.py
index 2ee1e0da..bdc55014 100644
--- a/ereuse_devicehub/resources/action/__init__.py
+++ b/ereuse_devicehub/resources/action/__init__.py
@@ -199,9 +199,9 @@ class ToPrepareDef(ActionDef):
     SCHEMA = schemas.ToPrepare
 
 
-class ToErasedDef(ActionDef):
+class DataWipeDef(ActionDef):
     VIEW = None
-    SCHEMA = schemas.ToErased
+    SCHEMA = schemas.DataWipe
 
 
 class AllocateDef(ActionDef):
diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py
index 30a91890..e29c37a2 100644
--- a/ereuse_devicehub/resources/action/models.py
+++ b/ereuse_devicehub/resources/action/models.py
@@ -1327,7 +1327,7 @@ class ToPrepare(ActionWithMultipleDevices):
     pass
 
 
-class ToErased(JoinedTableMixin, ActionWithMultipleDevices):
+class DataWipe(JoinedTableMixin, ActionWithMultipleDevices):
     """The device has been selected for insert one proof of erease disk.
     """
     document_comment = """The user that gets the device due this deal."""
@@ -1338,7 +1338,7 @@ class ToErased(JoinedTableMixin, ActionWithMultipleDevices):
                           backref=backref('actions',
                                           lazy=True,
                                           cascade=CASCADE_OWN),
-                          primaryjoin='ToErased.document_id == Document.id')
+                          primaryjoin='DataWipe.document_id == Document.id')
 
 
 class Prepare(ActionWithMultipleDevices):
diff --git a/ereuse_devicehub/resources/action/schemas.py b/ereuse_devicehub/resources/action/schemas.py
index 1a498a36..f0a2e895 100644
--- a/ereuse_devicehub/resources/action/schemas.py
+++ b/ereuse_devicehub/resources/action/schemas.py
@@ -431,11 +431,9 @@ class Prepare(ActionWithMultipleDevices):
     __doc__ = m.Prepare.__doc__
 
 
-class ToErased(ActionWithMultipleDevices):
-    __doc__ = m.ToErased.__doc__
+class DataWipe(ActionWithMultipleDevices):
+    __doc__ = m.DataWipe.__doc__
     document = NestedOn(s_generic_document.Document, only_query='id')
-    # document = NestedOn('EraseDocument')
-    # device = NestedOn(s_device.Device, only_query='id')
 
 
 class Live(ActionWithOneDevice):
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index c21f9d42..18030fad 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -9,7 +9,7 @@ from ereuse_devicehub.resources.action.models import (Trade, Confirm, ConfirmRev
                                                       Revoke, RevokeDocument, ConfirmDocument,
                                                       ConfirmRevokeDocument)
 from ereuse_devicehub.resources.user.models import User
-from ereuse_devicehub.resources.action.models import ToErased
+from ereuse_devicehub.resources.action.models import DataWipe
 from ereuse_devicehub.resources.documents.models import Document
 from ereuse_devicehub.resources.device.models import DataStorage 
 from ereuse_devicehub.resources.documents.schemas import Document as sh_document
@@ -37,7 +37,7 @@ class ErasedView():
         schema = sh_document()
         [data.pop(x, None) for x in ['severity', 'devices', 'name', 'description']]
         doc_data = schema.load(data)
-        doc_data['type'] = 'ToErased'
+        doc_data['type'] = 'DataWipe'
         self.document = Document(**doc_data)
         db.session.add(self.document)
         
@@ -57,5 +57,5 @@ class ErasedView():
                     self.data['devices'].add(component)
 
         self.data['document'] = self.document
-        self.erased = ToErased(**self.data)
+        self.erased = DataWipe(**self.data)
         db.session.add(self.erased)
diff --git a/ereuse_devicehub/resources/action/views/views.py b/ereuse_devicehub/resources/action/views/views.py
index c42089a2..415effb8 100644
--- a/ereuse_devicehub/resources/action/views/views.py
+++ b/ereuse_devicehub/resources/action/views/views.py
@@ -251,7 +251,7 @@ class ActionView(View):
             confirm_revoke = trade_view.ConfirmRevokeDocumentView(json, resource_def, self.schema)
             return confirm_revoke.post()
 
-        if json['type'] == 'ToErased':
+        if json['type'] == 'DataWipe':
             erased = ErasedView(json, resource_def.schema)
             return erased.post()
 
diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py
index 35937101..a0917bef 100644
--- a/ereuse_devicehub/resources/device/models.py
+++ b/ereuse_devicehub/resources/device/models.py
@@ -700,10 +700,10 @@ class Computer(Device):
     def external_document_erasure(self):
         """Returns the external ``DataStorage`` proof of erasure.
         """
-        from ereuse_devicehub.resources.action.models import ToErased
+        from ereuse_devicehub.resources.action.models import DataWipe
         urls = set()
         try:
-            ev = self.last_action_of(ToErased)
+            ev = self.last_action_of(DataWipe)
             urls.add(ev.document.url.to_text())
         except LookupError:
             pass
@@ -902,9 +902,9 @@ class DataStorage(JoinedComponentTableMixin, Component):
     def external_document_erasure(self):
         """Returns the external ``DataStorage`` proof of erasure.
         """
-        from ereuse_devicehub.resources.action.models import ToErased
+        from ereuse_devicehub.resources.action.models import DataWipe
         try:
-            ev = self.last_action_of(ToErased)
+            ev = self.last_action_of(DataWipe)
             return ev.document.url.to_text()
         except LookupError:
             return None
diff --git a/tests/test_action.py b/tests/test_action.py
index bfdcebd2..5b5dc877 100644
--- a/tests/test_action.py
+++ b/tests/test_action.py
@@ -2419,10 +2419,10 @@ def test_action_web_erase(user: UserClient, client: Client):
     bfile = BytesIO(b'abc')
     hash3 = hashlib.sha3_256(bfile.read()).hexdigest()
     snap, _ = user.post(file('acer.happy.battery.snapshot'), res=models.Snapshot)
-    request = {'type': 'ToErased', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': hash3}
+    request = {'type': 'DataWipe', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': hash3}
    
     user.post(res=models.Action, data=request)
-    action = models.ToErased.query.one()
+    action = models.DataWipe.query.one()
     for dev in action.devices:
         assert action in dev.actions
 

From 1811b81e93b489d31b4bf2a0a11db5b170abdb99 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Thu, 29 Jul 2021 14:54:58 +0200
Subject: [PATCH 13/26] adding success and software field in document

---
 .../migrations/versions/7ecb8ff7abad_documents.py         | 2 ++
 ereuse_devicehub/resources/action/views/documents.py      | 2 +-
 ereuse_devicehub/resources/documents/models.py            | 8 ++++++--
 ereuse_devicehub/resources/documents/schemas.py           | 6 ++++--
 tests/test_action.py                                      | 2 +-
 5 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index 730e8f8a..88d9ebe8 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -42,6 +42,8 @@ def upgrade():
                     sa.Column('type', sa.Unicode(), nullable=False),
                     sa.Column('date', sa.TIMESTAMP(timezone=True), nullable=True),
                     sa.Column('id_document', sa.Unicode(), nullable=True),
+                    sa.Column('software', sa.Unicode(), nullable=True),
+                    sa.Column('success', sa.Boolean(), nullable=False),
                     sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=False),
                     sa.Column('file_name', sa.Unicode(), nullable=False),
                     sa.Column('file_hash', sa.Unicode(), nullable=False),
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index 18030fad..19d40bb8 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -45,7 +45,7 @@ class ErasedView():
         db.session.add(db_hash)
 
     def insert_action(self, data):
-        [data.pop(x, None) for x in ['url', 'documentId', 'filename', 'hash']]
+        [data.pop(x, None) for x in ['url', 'documentId', 'filename', 'hash', 'software', 'success']]
         self.data = self.schema.load(data)
        
         for dev in self.data['devices']:
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index 7ee5eb99..46be23e8 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -1,6 +1,6 @@
-from citext import CIText
+from citext import CIText 
 from flask import g
-from sqlalchemy import BigInteger, Column, Sequence, Unicode
+from sqlalchemy import BigInteger, Column, Sequence, Unicode, Boolean
 from sqlalchemy.dialects.postgresql import UUID
 from teal.db import URL
 from ereuse_devicehub.db import db
@@ -19,6 +19,10 @@ class Document(Thing):
     date = Column(db.DateTime, nullable=True)
     date.comment = """The date of document, some documents need to have one date
     """
+    software = Column(CIText(), nullable=False)
+    software.comment = """Which software is used"""
+    success = Column(Boolean)
+    success.comment = """If the erase was success"""
     id_document = Column(CIText(), nullable=False)
     id_document.comment = """The id of one document like invoice so they can be linked."""
     owner_id = db.Column(UUID(as_uuid=True),
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
index 21b7017e..7b15103a 100644
--- a/ereuse_devicehub/resources/documents/schemas.py
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -1,4 +1,4 @@
-from marshmallow.fields import DateTime, Integer, validate
+from marshmallow.fields import DateTime, Integer, validate, Boolean
 from teal.marshmallow import SanitizedStr, URL
 from ereuse_devicehub.resources.schemas import Thing
 from ereuse_devicehub.resources.documents import models as m
@@ -9,6 +9,9 @@ class Document(Thing):
     __doc__ = m.Document.__doc__
     id = Integer(description=m.Document.id.comment, dump_only=True)
     type = SanitizedStr(default='Document')
+    url = URL(description=m.Document.url.comment)
+    success = Boolean(description=m.Document.success.comment)
+    software = SanitizedStr(description=m.Document.software.comment)
     date = DateTime(data_key='endTime', 
                     required=False, 
                     description=m.Document.date.comment)
@@ -23,4 +26,3 @@ class Document(Thing):
                              default='',
                              description=m.Document.file_hash.comment,
                              validate=validate.Length(max=64))
-    url = URL(description=m.Document.url.comment)
diff --git a/tests/test_action.py b/tests/test_action.py
index 5b5dc877..03868c3c 100644
--- a/tests/test_action.py
+++ b/tests/test_action.py
@@ -2419,7 +2419,7 @@ def test_action_web_erase(user: UserClient, client: Client):
     bfile = BytesIO(b'abc')
     hash3 = hashlib.sha3_256(bfile.read()).hexdigest()
     snap, _ = user.post(file('acer.happy.battery.snapshot'), res=models.Snapshot)
-    request = {'type': 'DataWipe', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': hash3}
+    request = {'type': 'DataWipe', 'devices': [snap['device']['id']], 'name': 'borrado universal', 'severity': 'Info', 'description': 'nada que describir', 'url': 'http://www.google.com/', 'documentId': '33', 'endTime': '2021-07-07T22:00:00.000Z', 'filename': 'Certificado de borrado1.pdf', 'hash': hash3, 'success': 1, 'software': "Blanco"}
    
     user.post(res=models.Action, data=request)
     action = models.DataWipe.query.one()

From 3119178c9c2354c022b33ed0ea385f7b0a4a03eb Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Thu, 29 Jul 2021 15:46:49 +0200
Subject: [PATCH 14/26] heredit datawipe document form general document model

---
 .../versions/7ecb8ff7abad_documents.py        | 13 ++++++++--
 ereuse_devicehub/resources/action/models.py   |  6 ++---
 ereuse_devicehub/resources/action/schemas.py  |  2 +-
 .../resources/action/views/documents.py       | 14 +++-------
 .../resources/documents/models.py             | 26 +++++++++++++++----
 .../resources/documents/schemas.py            | 23 ++++++++--------
 6 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index 88d9ebe8..7f99a5b3 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -42,8 +42,6 @@ def upgrade():
                     sa.Column('type', sa.Unicode(), nullable=False),
                     sa.Column('date', sa.TIMESTAMP(timezone=True), nullable=True),
                     sa.Column('id_document', sa.Unicode(), nullable=True),
-                    sa.Column('software', sa.Unicode(), nullable=True),
-                    sa.Column('success', sa.Boolean(), nullable=False),
                     sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=False),
                     sa.Column('file_name', sa.Unicode(), nullable=False),
                     sa.Column('file_hash', sa.Unicode(), nullable=False),
@@ -59,6 +57,17 @@ def upgrade():
     op.create_index('document_type_index', 'document', ['type'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
 
 
+    # DataWipeDocument table
+    op.create_table('data_wipe_document',
+                    sa.Column('id', sa.BigInteger(), nullable=False),
+                    sa.Column('software', sa.Unicode(), nullable=True),
+                    sa.Column('success', sa.Boolean(), nullable=False),
+                    sa.ForeignKeyConstraint(['id'], [f'{get_inv()}.document.id'], ),
+                    sa.PrimaryKeyConstraint('id'),
+                    schema=f'{get_inv()}'
+                    )
+
+
     # DataWipe table
     op.create_table('data_wipe',
                     sa.Column('document_id', sa.BigInteger(), nullable=True),
diff --git a/ereuse_devicehub/resources/action/models.py b/ereuse_devicehub/resources/action/models.py
index e29c37a2..20bd5819 100644
--- a/ereuse_devicehub/resources/action/models.py
+++ b/ereuse_devicehub/resources/action/models.py
@@ -1332,13 +1332,13 @@ class DataWipe(JoinedTableMixin, ActionWithMultipleDevices):
     """
     document_comment = """The user that gets the device due this deal."""
     document_id = db.Column(BigInteger,
-                            db.ForeignKey('document.id'),
+                            db.ForeignKey('data_wipe_document.id'),
                             nullable=False)
-    document = db.relationship('Document',
+    document = db.relationship('DataWipeDocument',
                           backref=backref('actions',
                                           lazy=True,
                                           cascade=CASCADE_OWN),
-                          primaryjoin='DataWipe.document_id == Document.id')
+                          primaryjoin='DataWipe.document_id == DataWipeDocument.id')
 
 
 class Prepare(ActionWithMultipleDevices):
diff --git a/ereuse_devicehub/resources/action/schemas.py b/ereuse_devicehub/resources/action/schemas.py
index f0a2e895..3232f11b 100644
--- a/ereuse_devicehub/resources/action/schemas.py
+++ b/ereuse_devicehub/resources/action/schemas.py
@@ -433,7 +433,7 @@ class Prepare(ActionWithMultipleDevices):
 
 class DataWipe(ActionWithMultipleDevices):
     __doc__ = m.DataWipe.__doc__
-    document = NestedOn(s_generic_document.Document, only_query='id')
+    document = NestedOn(s_generic_document.DataWipeDocument, only_query='id')
 
 
 class Live(ActionWithOneDevice):
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index 19d40bb8..3f3aaa84 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -1,18 +1,10 @@
 import copy
 
-from flask import g
-from sqlalchemy.util import OrderedSet
-from teal.marshmallow import ValidationError
-
 from ereuse_devicehub.db import db
-from ereuse_devicehub.resources.action.models import (Trade, Confirm, ConfirmRevoke, 
-                                                      Revoke, RevokeDocument, ConfirmDocument,
-                                                      ConfirmRevokeDocument)
-from ereuse_devicehub.resources.user.models import User
 from ereuse_devicehub.resources.action.models import DataWipe
-from ereuse_devicehub.resources.documents.models import Document
+from ereuse_devicehub.resources.documents.models import DataWipeDocument
 from ereuse_devicehub.resources.device.models import DataStorage 
-from ereuse_devicehub.resources.documents.schemas import Document as sh_document
+from ereuse_devicehub.resources.documents.schemas import DataWipeDocument as sh_document
 from ereuse_devicehub.resources.hash_reports import ReportHash
 
 
@@ -38,7 +30,7 @@ class ErasedView():
         [data.pop(x, None) for x in ['severity', 'devices', 'name', 'description']]
         doc_data = schema.load(data)
         doc_data['type'] = 'DataWipe'
-        self.document = Document(**doc_data)
+        self.document = DataWipeDocument(**doc_data)
         db.session.add(self.document)
         
         db_hash = ReportHash(hash3=self.document.file_hash)
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index 46be23e8..85aeb03f 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -1,6 +1,7 @@
 from citext import CIText 
 from flask import g
-from sqlalchemy import BigInteger, Column, Sequence, Unicode, Boolean
+from sqlalchemy import BigInteger, Column, Sequence, Unicode, Boolean, ForeignKey
+from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.dialects.postgresql import UUID
 from teal.db import URL
 from ereuse_devicehub.db import db
@@ -19,10 +20,6 @@ class Document(Thing):
     date = Column(db.DateTime, nullable=True)
     date.comment = """The date of document, some documents need to have one date
     """
-    software = Column(CIText(), nullable=False)
-    software.comment = """Which software is used"""
-    success = Column(Boolean)
-    success.comment = """If the erase was success"""
     id_document = Column(CIText(), nullable=False)
     id_document.comment = """The id of one document like invoice so they can be linked."""
     owner_id = db.Column(UUID(as_uuid=True),
@@ -39,3 +36,22 @@ class Document(Thing):
 
     def __str__(self) -> str:
         return '{0.file_name}'.format(self)
+
+
+class JoinedTableMixin:
+    # noinspection PyMethodParameters
+    @declared_attr
+    def id(cls):
+        return Column(BigInteger, ForeignKey(Document.id), primary_key=True)
+
+
+class DataWipeDocument(JoinedTableMixin, Document):
+    """This represent a generic document."""
+
+    software = Column(CIText(), nullable=False)
+    software.comment = """Which software is used"""
+    success = Column(Boolean)
+    success.comment = """If the erase was success"""
+
+    def __str__(self) -> str:
+        return '{0.file_name}'.format(self)
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
index 7b15103a..23db2fa2 100644
--- a/ereuse_devicehub/resources/documents/schemas.py
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -2,27 +2,26 @@ from marshmallow.fields import DateTime, Integer, validate, Boolean
 from teal.marshmallow import SanitizedStr, URL
 from ereuse_devicehub.resources.schemas import Thing
 from ereuse_devicehub.resources.documents import models as m
-# from marshmallow import ValidationError, validates_schema
 
 
-class Document(Thing):
-    __doc__ = m.Document.__doc__
-    id = Integer(description=m.Document.id.comment, dump_only=True)
-    type = SanitizedStr(default='Document')
-    url = URL(description=m.Document.url.comment)
-    success = Boolean(description=m.Document.success.comment)
-    software = SanitizedStr(description=m.Document.software.comment)
+class DataWipeDocument(Thing):
+    __doc__ = m.DataWipeDocument.__doc__
+    id = Integer(description=m.DataWipeDocument.id.comment, dump_only=True)
+    type = SanitizedStr(default='DataWipeDocument')
+    url = URL(description=m.DataWipeDocument.url.comment)
+    success = Boolean(description=m.DataWipeDocument.success.comment)
+    software = SanitizedStr(description=m.DataWipeDocument.software.comment)
     date = DateTime(data_key='endTime', 
                     required=False, 
-                    description=m.Document.date.comment)
+                    description=m.DataWipeDocument.date.comment)
     id_document = SanitizedStr(data_key='documentId', 
                                default='', 
-                               description=m.Document.id_document.comment)
+                               description=m.DataWipeDocument.id_document.comment)
     file_name = SanitizedStr(data_key='filename',
                              default='',
-                             description=m.Document.file_name.comment, 
+                             description=m.DataWipeDocument.file_name.comment, 
                              validate=validate.Length(max=100))
     file_hash = SanitizedStr(data_key='hash',
                              default='',
-                             description=m.Document.file_hash.comment,
+                             description=m.DataWipeDocument.file_hash.comment,
                              validate=validate.Length(max=64))

From 7160b8819985443375ec9e90e381f8efd33701b9 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Thu, 29 Jul 2021 15:55:59 +0200
Subject: [PATCH 15/26] fixing migration downgrade

---
 ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index 7f99a5b3..5a03a35a 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -81,4 +81,5 @@ def upgrade():
 
 def downgrade():
     op.drop_table('data_wipe', schema=f'{get_inv()}')
+    op.drop_table('data_wipe_document', schema=f'{get_inv()}')
     op.drop_table('document', schema=f'{get_inv()}')

From bc95a618c18452c3686e37a872393468df825dc4 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Fri, 30 Jul 2021 14:36:10 +0200
Subject: [PATCH 16/26] adding result, software and url for DataWipe action

---
 .../resources/documents/device_row.py         | 111 +++++++++++++-----
 1 file changed, 82 insertions(+), 29 deletions(-)

diff --git a/ereuse_devicehub/resources/documents/device_row.py b/ereuse_devicehub/resources/documents/device_row.py
index 875906d7..1e8d1aa7 100644
--- a/ereuse_devicehub/resources/documents/device_row.py
+++ b/ereuse_devicehub/resources/documents/device_row.py
@@ -52,7 +52,8 @@ class DeviceRow(OrderedDict):
         self['Tag 2 Type'] = self['Tag 2 ID'] = self['Tag 2 Organization'] = ''
         self['Tag 3 Type'] = self['Tag 3 ID'] = self['Tag 3 Organization'] = ''
         for i, tag in zip(range(1, 3), device.tags):
-            self['Tag {} Type'.format(i)] = 'unamed' if tag.provider else 'named'
+            self['Tag {} Type'.format(
+                i)] = 'unamed' if tag.provider else 'named'
             self['Tag {} ID'.format(i)] = tag.id
             self['Tag {} Organization'.format(i)] = tag.org.name
 
@@ -70,11 +71,13 @@ class DeviceRow(OrderedDict):
         self['Updated in (web)'] = ''
 
         try:
-            self['Physical state'] = device.last_action_of(*states.Physical.actions()).t
+            self['Physical state'] = device.last_action_of(
+                *states.Physical.actions()).t
         except LookupError:
             self['Physical state'] = ''
         try:
-            self['Trading state'] = device.last_action_of(*states.Trading.actions()).t
+            self['Trading state'] = device.last_action_of(
+                *states.Trading.actions()).t
         except LookupError:
             self['Trading state'] = ''
         if isinstance(device, d.Computer):
@@ -108,7 +111,7 @@ class DeviceRow(OrderedDict):
             self['RAM Range'] = ''
             self['Data Storage Rate'] = ''
             self['Data Storage Range'] = ''
-        
+
         self['Price'] = none2str(device.price)
 
         benchram = get_action(device, 'BenchmarkRamSysbench')
@@ -120,7 +123,7 @@ class DeviceRow(OrderedDict):
     def components(self):
         """Function to get all components information of a device."""
         assert isinstance(self.device, d.Computer)
-        for ctype in self.ORDER_COMPONENTS: # ctype: str
+        for ctype in self.ORDER_COMPONENTS:  # ctype: str
             cmax = self.NUMS.get(ctype, 4)
             i = 1
             l_ctype = [ctype]
@@ -148,9 +151,11 @@ class DeviceRow(OrderedDict):
             self['{} {} Model'.format(ctype, i)] = ''
             self['{} {} Serial Number'.format(ctype, i)] = ''
         else:
-            self['{} {} Manufacturer'.format(ctype, i)] = none2str(component.manufacturer)
+            self['{} {} Manufacturer'.format(ctype, i)] = none2str(
+                component.manufacturer)
             self['{} {} Model'.format(ctype, i)] = none2str(component.model)
-            self['{} {} Serial Number'.format(ctype, i)] = none2str(component.serial_number)
+            self['{} {} Serial Number'.format(ctype, i)] = none2str(
+                component.serial_number)
 
         if ctype == d.Processor.t:
             self.get_processor(ctype, i, component)
@@ -170,10 +175,12 @@ class DeviceRow(OrderedDict):
             self['{} {} Number of cores'.format(ctype, i)] = ''
             self['{} {} Speed (GHz)'.format(ctype, i)] = ''
             self['Benchmark {} {} (points)'.format(ctype, i)] = ''
-            self['Benchmark ProcessorSysbench {} {} (points)'.format(ctype, i)] = ''
+            self['Benchmark ProcessorSysbench {} {} (points)'.format(
+                ctype, i)] = ''
             return
 
-        self['{} {} Number of cores'.format(ctype, i)] = none2str(component.cores)
+        self['{} {} Number of cores'.format(
+            ctype, i)] = none2str(component.cores)
         self['{} {} Speed (GHz)'.format(ctype, i)] = none2str(component.speed)
 
         benchmark = get_action(component, 'BenchmarkProcessor')
@@ -184,9 +191,11 @@ class DeviceRow(OrderedDict):
 
         sysbench = get_action(component, 'BenchmarkProcessorSysbench')
         if not sysbench:
-            self['Benchmark ProcessorSysbench {} {} (points)'.format(ctype, i)] = ''
+            self['Benchmark ProcessorSysbench {} {} (points)'.format(
+                ctype, i)] = ''
             return
-        self['Benchmark ProcessorSysbench {} {} (points)'.format(ctype, i)] = sysbench.rate
+        self['Benchmark ProcessorSysbench {} {} (points)'.format(
+            ctype, i)] = sysbench.rate
 
     def get_ram(self, ctype, i, component):
         """Particular fields for component Ram Module."""
@@ -202,6 +211,7 @@ class DeviceRow(OrderedDict):
         """Particular fields for component DataStorage.
            A DataStorage can be HardDrive or SolidStateDrive.
         """
+
         if component is None:
             self['{} {} Size (MB)'.format(ctype, i)] = ''
             self['Erasure {} {}'.format(ctype, i)] = ''
@@ -209,6 +219,7 @@ class DeviceRow(OrderedDict):
             self['Erasure {} {} Size (MB)'.format(ctype, i)] = ''
             self['Erasure {} {} Software'.format(ctype, i)] = ''
             self['Erasure {} {} Result'.format(ctype, i)] = ''
+            self['Erasure {} {} Certificate URL'.format(ctype, i)] = ''
             self['Erasure {} {} Type'.format(ctype, i)] = ''
             self['Erasure {} {} Method'.format(ctype, i)] = ''
             self['Erasure {} {} Elapsed (hours)'.format(ctype, i)] = ''
@@ -222,7 +233,8 @@ class DeviceRow(OrderedDict):
             self['Test {} {} Type'.format(ctype, i)] = ''
             self['Test {} {} Result'.format(ctype, i)] = ''
             self['Test {} {} Power on (hours used)'.format(ctype, i)] = ''
-            self['Test {} {} Lifetime remaining (percentage)'.format(ctype, i)] = ''
+            self['Test {} {} Lifetime remaining (percentage)'.format(
+                ctype, i)] = ''
             return
 
         snapshot = get_action(component, 'Snapshot')
@@ -233,15 +245,38 @@ class DeviceRow(OrderedDict):
 
         self['{} {} Size (MB)'.format(ctype, i)] = none2str(component.size)
 
-        erasures = [a for a in component.actions if a.type in ['EraseBasic', 'EraseSectors']]
+        erasures = [a for a in component.actions if a.type in [
+            'EraseBasic', 'EraseSectors', 'DataWipe']]
         erasure = erasures[-1] if erasures else None
         if not erasure:
             self['Erasure {} {}'.format(ctype, i)] = none2str(component.hid)
             serial_number = none2str(component.serial_number)
-            self['Erasure {} {} Serial Number'.format(ctype, i)] = serial_number
-            self['Erasure {} {} Size (MB)'.format(ctype, i)] = none2str(component.size)
+            self['Erasure {} {} Serial Number'.format(
+                ctype, i)] = serial_number
+            self['Erasure {} {} Size (MB)'.format(
+                ctype, i)] = none2str(component.size)
             self['Erasure {} {} Software'.format(ctype, i)] = ''
             self['Erasure {} {} Result'.format(ctype, i)] = ''
+            self['Erasure {} {} Certificate URL'.format(ctype, i)] = ''
+            self['Erasure {} {} Type'.format(ctype, i)] = ''
+            self['Erasure {} {} Method'.format(ctype, i)] = ''
+            self['Erasure {} {} Elapsed (hours)'.format(ctype, i)] = ''
+            self['Erasure {} {} Date'.format(ctype, i)] = ''
+            self['Erasure {} {} Steps'.format(ctype, i)] = ''
+            self['Erasure {} {} Steps Start Time'.format(ctype, i)] = ''
+            self['Erasure {} {} Steps End Time'.format(ctype, i)] = ''
+        elif hasattr(erasure, 'type') and erasure.type == 'DataWipe':
+            self['Erasure {} {}'.format(ctype, i)] = none2str(component.hid)
+            serial_number = none2str(component.serial_number)
+            self['Erasure {} {} Serial Number'.format(
+                ctype, i)] = serial_number
+            self['Erasure {} {} Size (MB)'.format(
+                ctype, i)] = none2str(component.size)
+            self['Erasure {} {} Software'.format(
+                ctype, i)] = erasure.document.software
+            self['Erasure {} {} Result'.format(ctype, i)] = get_result(erasure)
+            self['Erasure {} {} Certificate URL'.format(
+                ctype, i)] = erasure.document.url.to_text()
             self['Erasure {} {} Type'.format(ctype, i)] = ''
             self['Erasure {} {} Method'.format(ctype, i)] = ''
             self['Erasure {} {} Elapsed (hours)'.format(ctype, i)] = ''
@@ -252,22 +287,31 @@ class DeviceRow(OrderedDict):
         else:
             self['Erasure {} {}'.format(ctype, i)] = none2str(component.hid)
             serial_number = none2str(component.serial_number)
-            self['Erasure {} {} Serial Number'.format(ctype, i)] = serial_number
-            self['Erasure {} {} Size (MB)'.format(ctype, i)] = none2str(component.size)
+            self['Erasure {} {} Serial Number'.format(
+                ctype, i)] = serial_number
+            self['Erasure {} {} Size (MB)'.format(
+                ctype, i)] = none2str(component.size)
             self['Erasure {} {} Software'.format(ctype, i)] = software
 
-            result = get_result(erasure.severity)
+            result = get_result(erasure)
             self['Erasure {} {} Result'.format(ctype, i)] = result
+            self['Erasure {} {} Certificate URL'.format(ctype, i)] = ''
             self['Erasure {} {} Type'.format(ctype, i)] = erasure.type
             self['Erasure {} {} Method'.format(ctype, i)] = erasure.method
-            self['Erasure {} {} Elapsed (hours)'.format(ctype, i)] = format(erasure.elapsed)
-            self['Erasure {} {} Date'.format(ctype, i)] = format(erasure.created)
+            self['Erasure {} {} Elapsed (hours)'.format(
+                ctype, i)] = format(erasure.elapsed)
+            self['Erasure {} {} Date'.format(
+                ctype, i)] = format(erasure.created)
             steps = ','.join((format(x) for x in erasure.steps))
             self['Erasure {} {} Steps'.format(ctype, i)] = steps
-            steps_start_time = ','.join((format(x.start_time) for x in erasure.steps))
-            self['Erasure {} {} Steps Start Time'.format(ctype, i)] = steps_start_time
-            steps_end_time = ','.join((format(x.end_time) for x in erasure.steps))
-            self['Erasure {} {} Steps End Time'.format(ctype, i)] = steps_end_time
+            steps_start_time = ','.join(
+                (format(x.start_time) for x in erasure.steps))
+            self['Erasure {} {} Steps Start Time'.format(
+                ctype, i)] = steps_start_time
+            steps_end_time = ','.join((format(x.end_time)
+                                       for x in erasure.steps))
+            self['Erasure {} {} Steps End Time'.format(
+                ctype, i)] = steps_end_time
 
         benchmark = get_action(component, 'BenchmarkDataStorage')
         if not benchmark:
@@ -285,12 +329,14 @@ class DeviceRow(OrderedDict):
             self['Test {} {} Type'.format(ctype, i)] = ''
             self['Test {} {} Result'.format(ctype, i)] = ''
             self['Test {} {} Power on (hours used)'.format(ctype, i)] = ''
-            self['Test {} {} Lifetime remaining (percentage)'.format(ctype, i)] = ''
+            self['Test {} {} Lifetime remaining (percentage)'.format(
+                ctype, i)] = ''
             return
 
         self['Test {} {} Software'.format(ctype, i)] = software
         self['Test {} {} Type'.format(ctype, i)] = test_storage.length.value
-        self['Test {} {} Result'.format(ctype, i)] = get_result(test_storage.severity)
+        self['Test {} {} Result'.format(ctype, i)] = get_result(
+            test_storage)
         self['Test {} {} Power on (hours used)'.format(ctype, i)] = none2str(
             test_storage.power_cycle_count)
         self['Test {} {} Lifetime remaining (percentage)'.format(ctype, i)] = none2str(
@@ -319,7 +365,8 @@ class StockRow(OrderedDict):
         self['Manufacturer'] = none2str(device.manufacturer)
         self['Registered in'] = format(device.created, '%c')
         try:
-            self['Physical state'] = device.last_action_of(*states.Physical.actions()).t
+            self['Physical state'] = device.last_action_of(
+                *states.Physical.actions()).t
         except LookupError:
             self['Physical state'] = ''
         try:
@@ -343,15 +390,21 @@ class StockRow(OrderedDict):
             self['Data Storage Range'] = rate.data_storage_range
 
 
-def get_result(severity):
+def get_result(erasure):
     """ For the csv is necessary simplify the message of results """
+    if hasattr(erasure, 'type') and erasure.type == 'DataWipe':
+        if erasure.document.success:
+            return 'Success'
+        return 'Failure'
+
+
     type_of_results = {
         Severity.Error: 'Failure',
         Severity.Warning: 'Success with Warnings',
         Severity.Notice: 'Success',
         Severity.Info: 'Success'
         }
-    return type_of_results[severity]
+    return type_of_results[erasure.severity]
 
 
 def none2str(string):

From e0c8d0204c5df6efc23592337db1ca785870e151 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Mon, 2 Aug 2021 10:46:35 +0200
Subject: [PATCH 17/26] fixing tests csv exports

---
 tests/files/basic.csv                        |  4 +-
 tests/files/proposal_extended_csv_report.csv |  6 +-
 tests/test_documents.py                      | 91 +++++++++++---------
 3 files changed, 53 insertions(+), 48 deletions(-)

diff --git a/tests/files/basic.csv b/tests/files/basic.csv
index 8a7851ec..b6707df9 100644
--- a/tests/files/basic.csv
+++ b/tests/files/basic.csv
@@ -1,2 +1,2 @@
-DevicehubID;DocumentID;Public Link;Lots;Tag 1 Type;Tag 1 ID;Tag 1 Organization;Tag 2 Type;Tag 2 ID;Tag 2 Organization;Tag 3 Type;Tag 3 ID;Tag 3 Organization;Device Hardware ID;Device Type;Device Chassis;Device Serial Number;Device Model;Device Manufacturer;Registered in;Registered (process);Updated in (software);Updated in (web);Physical state;Trading state;Processor;RAM (MB);Data Storage Size (MB);Processor 1;Processor 1 Manufacturer;Processor 1 Model;Processor 1 Serial Number;Processor 1 Number of cores;Processor 1 Speed (GHz);Benchmark Processor 1 (points);Benchmark ProcessorSysbench Processor 1 (points);Processor 2;Processor 2 Manufacturer;Processor 2 Model;Processor 2 Serial Number;Processor 2 Number of cores;Processor 2 Speed (GHz);Benchmark Processor 2 (points);Benchmark ProcessorSysbench Processor 2 (points);RamModule 1;RamModule 1 Manufacturer;RamModule 1 Model;RamModule 1 Serial Number;RamModule 1 Size (MB);RamModule 1 Speed (MHz);RamModule 2;RamModule 2 Manufacturer;RamModule 2 Model;RamModule 2 Serial Number;RamModule 2 Size (MB);RamModule 2 Speed (MHz);RamModule 3;RamModule 3 Manufacturer;RamModule 3 Model;RamModule 3 Serial Number;RamModule 3 Size (MB);RamModule 3 Speed (MHz);RamModule 4;RamModule 4 Manufacturer;RamModule 4 Model;RamModule 4 Serial Number;RamModule 4 Size (MB);RamModule 4 Speed (MHz);DataStorage 1;DataStorage 1 Manufacturer;DataStorage 1 Model;DataStorage 1 Serial Number;DataStorage 1 Size (MB);Erasure DataStorage 1;Erasure DataStorage 1 Serial Number;Erasure DataStorage 1 Size (MB);Erasure DataStorage 1 Software;Erasure DataStorage 1 Result;Erasure DataStorage 1 Type;Erasure DataStorage 1 Method;Erasure DataStorage 1 Elapsed (hours);Erasure DataStorage 1 Date;Erasure DataStorage 1 Steps;Erasure DataStorage 1 Steps Start Time;Erasure DataStorage 1 Steps End Time;Benchmark DataStorage 1 Read Speed (MB/s);Benchmark DataStorage 1 Writing speed (MB/s);Test DataStorage 1 Software;Test DataStorage 1 Type;Test DataStorage 1 Result;Test DataStorage 1 Power on (hours used);Test DataStorage 1 Lifetime remaining (percentage);DataStorage 2;DataStorage 2 Manufacturer;DataStorage 2 Model;DataStorage 2 Serial Number;DataStorage 2 Size (MB);Erasure DataStorage 2;Erasure DataStorage 2 Serial Number;Erasure DataStorage 2 Size (MB);Erasure DataStorage 2 Software;Erasure DataStorage 2 Result;Erasure DataStorage 2 Type;Erasure DataStorage 2 Method;Erasure DataStorage 2 Elapsed (hours);Erasure DataStorage 2 Date;Erasure DataStorage 2 Steps;Erasure DataStorage 2 Steps Start Time;Erasure DataStorage 2 Steps End Time;Benchmark DataStorage 2 Read Speed (MB/s);Benchmark DataStorage 2 Writing speed (MB/s);Test DataStorage 2 Software;Test DataStorage 2 Type;Test DataStorage 2 Result;Test DataStorage 2 Power on (hours used);Test DataStorage 2 Lifetime remaining (percentage);DataStorage 3;DataStorage 3 Manufacturer;DataStorage 3 Model;DataStorage 3 Serial Number;DataStorage 3 Size (MB);Erasure DataStorage 3;Erasure DataStorage 3 Serial Number;Erasure DataStorage 3 Size (MB);Erasure DataStorage 3 Software;Erasure DataStorage 3 Result;Erasure DataStorage 3 Type;Erasure DataStorage 3 Method;Erasure DataStorage 3 Elapsed (hours);Erasure DataStorage 3 Date;Erasure DataStorage 3 Steps;Erasure DataStorage 3 Steps Start Time;Erasure DataStorage 3 Steps End Time;Benchmark DataStorage 3 Read Speed (MB/s);Benchmark DataStorage 3 Writing speed (MB/s);Test DataStorage 3 Software;Test DataStorage 3 Type;Test DataStorage 3 Result;Test DataStorage 3 Power on (hours used);Test DataStorage 3 Lifetime remaining (percentage);DataStorage 4;DataStorage 4 Manufacturer;DataStorage 4 Model;DataStorage 4 Serial Number;DataStorage 4 Size (MB);Erasure DataStorage 4;Erasure DataStorage 4 Serial Number;Erasure DataStorage 4 Size (MB);Erasure DataStorage 4 Software;Erasure DataStorage 4 Result;Erasure DataStorage 4 Type;Erasure DataStorage 4 Method;Erasure DataStorage 4 Elapsed (hours);Erasure DataStorage 4 Date;Erasure DataStorage 4 Steps;Erasure DataStorage 4 Steps Start Time;Erasure DataStorage 4 Steps End Time;Benchmark DataStorage 4 Read Speed (MB/s);Benchmark DataStorage 4 Writing speed (MB/s);Test DataStorage 4 Software;Test DataStorage 4 Type;Test DataStorage 4 Result;Test DataStorage 4 Power on (hours used);Test DataStorage 4 Lifetime remaining (percentage);Motherboard 1;Motherboard 1 Manufacturer;Motherboard 1 Model;Motherboard 1 Serial Number;Display 1;Display 1 Manufacturer;Display 1 Model;Display 1 Serial Number;GraphicCard 1;GraphicCard 1 Manufacturer;GraphicCard 1 Model;GraphicCard 1 Serial Number;GraphicCard 1 Memory (MB);GraphicCard 2;GraphicCard 2 Manufacturer;GraphicCard 2 Model;GraphicCard 2 Serial Number;GraphicCard 2 Memory (MB);NetworkAdapter 1;NetworkAdapter 1 Manufacturer;NetworkAdapter 1 Model;NetworkAdapter 1 Serial Number;NetworkAdapter 2;NetworkAdapter 2 Manufacturer;NetworkAdapter 2 Model;NetworkAdapter 2 Serial Number;SoundCard 1;SoundCard 1 Manufacturer;SoundCard 1 Model;SoundCard 1 Serial Number;SoundCard 2;SoundCard 2 Manufacturer;SoundCard 2 Model;SoundCard 2 Serial Number;Device Rate;Device Range;Processor Rate;Processor Range;RAM Rate;RAM Range;Data Storage Rate;Data Storage Range;Price;Benchmark RamSysbench (points)
-93652;;http://localhost/devices/93652;;;;;;;;;;;desktop-d1mr-d1ml-d1s;Desktop;Microtower;d1s;d1ml;d1mr;Wed May 26 11:29:24 2021;Workbench 11.0;2021-05-26 11:29:24.703310+02:00;;;;p1ml;0;0;Processor 6: model p1ml, S/N p1s;p1mr;p1ml;p1s;;1.6;2410.0;;;;;;;;;;RamModule 5: model rm1ml, S/N rm1s;rm1mr;rm1ml;rm1s;;1333;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GraphicCard 4: model gc1ml, S/N gc1s;gc1mr;gc1ml;gc1s;;;;;;;;;;;;;;;;;;;;;;;1.0;VERY_LOW;1.0;VERY_LOW;1.0;VERY_LOW;1.0;VERY_LOW;;
+DevicehubID;DocumentID;Public Link;Lots;Tag 1 Type;Tag 1 ID;Tag 1 Organization;Tag 2 Type;Tag 2 ID;Tag 2 Organization;Tag 3 Type;Tag 3 ID;Tag 3 Organization;Device Hardware ID;Device Type;Device Chassis;Device Serial Number;Device Model;Device Manufacturer;Registered in;Registered (process);Updated in (software);Updated in (web);Physical state;Trading state;Processor;RAM (MB);Data Storage Size (MB);Processor 1;Processor 1 Manufacturer;Processor 1 Model;Processor 1 Serial Number;Processor 1 Number of cores;Processor 1 Speed (GHz);Benchmark Processor 1 (points);Benchmark ProcessorSysbench Processor 1 (points);Processor 2;Processor 2 Manufacturer;Processor 2 Model;Processor 2 Serial Number;Processor 2 Number of cores;Processor 2 Speed (GHz);Benchmark Processor 2 (points);Benchmark ProcessorSysbench Processor 2 (points);RamModule 1;RamModule 1 Manufacturer;RamModule 1 Model;RamModule 1 Serial Number;RamModule 1 Size (MB);RamModule 1 Speed (MHz);RamModule 2;RamModule 2 Manufacturer;RamModule 2 Model;RamModule 2 Serial Number;RamModule 2 Size (MB);RamModule 2 Speed (MHz);RamModule 3;RamModule 3 Manufacturer;RamModule 3 Model;RamModule 3 Serial Number;RamModule 3 Size (MB);RamModule 3 Speed (MHz);RamModule 4;RamModule 4 Manufacturer;RamModule 4 Model;RamModule 4 Serial Number;RamModule 4 Size (MB);RamModule 4 Speed (MHz);DataStorage 1;DataStorage 1 Manufacturer;DataStorage 1 Model;DataStorage 1 Serial Number;DataStorage 1 Size (MB);Erasure DataStorage 1;Erasure DataStorage 1 Serial Number;Erasure DataStorage 1 Size (MB);Erasure DataStorage 1 Software;Erasure DataStorage 1 Result;Erasure DataStorage 1 Certificate URL;Erasure DataStorage 1 Type;Erasure DataStorage 1 Method;Erasure DataStorage 1 Elapsed (hours);Erasure DataStorage 1 Date;Erasure DataStorage 1 Steps;Erasure DataStorage 1 Steps Start Time;Erasure DataStorage 1 Steps End Time;Benchmark DataStorage 1 Read Speed (MB/s);Benchmark DataStorage 1 Writing speed (MB/s);Test DataStorage 1 Software;Test DataStorage 1 Type;Test DataStorage 1 Result;Test DataStorage 1 Power on (hours used);Test DataStorage 1 Lifetime remaining (percentage);DataStorage 2;DataStorage 2 Manufacturer;DataStorage 2 Model;DataStorage 2 Serial Number;DataStorage 2 Size (MB);Erasure DataStorage 2;Erasure DataStorage 2 Serial Number;Erasure DataStorage 2 Size (MB);Erasure DataStorage 2 Software;Erasure DataStorage 2 Result;Erasure DataStorage 2 Certificate URL;Erasure DataStorage 2 Type;Erasure DataStorage 2 Method;Erasure DataStorage 2 Elapsed (hours);Erasure DataStorage 2 Date;Erasure DataStorage 2 Steps;Erasure DataStorage 2 Steps Start Time;Erasure DataStorage 2 Steps End Time;Benchmark DataStorage 2 Read Speed (MB/s);Benchmark DataStorage 2 Writing speed (MB/s);Test DataStorage 2 Software;Test DataStorage 2 Type;Test DataStorage 2 Result;Test DataStorage 2 Power on (hours used);Test DataStorage 2 Lifetime remaining (percentage);DataStorage 3;DataStorage 3 Manufacturer;DataStorage 3 Model;DataStorage 3 Serial Number;DataStorage 3 Size (MB);Erasure DataStorage 3;Erasure DataStorage 3 Serial Number;Erasure DataStorage 3 Size (MB);Erasure DataStorage 3 Software;Erasure DataStorage 3 Result;Erasure DataStorage 3 Certificate URL;Erasure DataStorage 3 Type;Erasure DataStorage 3 Method;Erasure DataStorage 3 Elapsed (hours);Erasure DataStorage 3 Date;Erasure DataStorage 3 Steps;Erasure DataStorage 3 Steps Start Time;Erasure DataStorage 3 Steps End Time;Benchmark DataStorage 3 Read Speed (MB/s);Benchmark DataStorage 3 Writing speed (MB/s);Test DataStorage 3 Software;Test DataStorage 3 Type;Test DataStorage 3 Result;Test DataStorage 3 Power on (hours used);Test DataStorage 3 Lifetime remaining (percentage);DataStorage 4;DataStorage 4 Manufacturer;DataStorage 4 Model;DataStorage 4 Serial Number;DataStorage 4 Size (MB);Erasure DataStorage 4;Erasure DataStorage 4 Serial Number;Erasure DataStorage 4 Size (MB);Erasure DataStorage 4 Software;Erasure DataStorage 4 Result;Erasure DataStorage 4 Certificate URL;Erasure DataStorage 4 Type;Erasure DataStorage 4 Method;Erasure DataStorage 4 Elapsed (hours);Erasure DataStorage 4 Date;Erasure DataStorage 4 Steps;Erasure DataStorage 4 Steps Start Time;Erasure DataStorage 4 Steps End Time;Benchmark DataStorage 4 Read Speed (MB/s);Benchmark DataStorage 4 Writing speed (MB/s);Test DataStorage 4 Software;Test DataStorage 4 Type;Test DataStorage 4 Result;Test DataStorage 4 Power on (hours used);Test DataStorage 4 Lifetime remaining (percentage);Motherboard 1;Motherboard 1 Manufacturer;Motherboard 1 Model;Motherboard 1 Serial Number;Display 1;Display 1 Manufacturer;Display 1 Model;Display 1 Serial Number;GraphicCard 1;GraphicCard 1 Manufacturer;GraphicCard 1 Model;GraphicCard 1 Serial Number;GraphicCard 1 Memory (MB);GraphicCard 2;GraphicCard 2 Manufacturer;GraphicCard 2 Model;GraphicCard 2 Serial Number;GraphicCard 2 Memory (MB);NetworkAdapter 1;NetworkAdapter 1 Manufacturer;NetworkAdapter 1 Model;NetworkAdapter 1 Serial Number;NetworkAdapter 2;NetworkAdapter 2 Manufacturer;NetworkAdapter 2 Model;NetworkAdapter 2 Serial Number;SoundCard 1;SoundCard 1 Manufacturer;SoundCard 1 Model;SoundCard 1 Serial Number;SoundCard 2;SoundCard 2 Manufacturer;SoundCard 2 Model;SoundCard 2 Serial Number;Device Rate;Device Range;Processor Rate;Processor Range;RAM Rate;RAM Range;Data Storage Rate;Data Storage Range;Price;Benchmark RamSysbench (points)
+93652;;http://localhost/devices/93652;;;;;;;;;;;desktop-d1mr-d1ml-d1s;Desktop;Microtower;d1s;d1ml;d1mr;Mon Aug  2 10:25:31 2021;Workbench 11.0;2021-08-02 10:25:31.457986+02:00;;;;p1ml;0;0;Processor 6: model p1ml, S/N p1s;p1mr;p1ml;p1s;;1.6;2410.0;;;;;;;;;;RamModule 5: model rm1ml, S/N rm1s;rm1mr;rm1ml;rm1s;;1333;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GraphicCard 4: model gc1ml, S/N gc1s;gc1mr;gc1ml;gc1s;;;;;;;;;;;;;;;;;;;;;;;1.0;VERY_LOW;1.0;VERY_LOW;1.0;VERY_LOW;1.0;VERY_LOW;;
diff --git a/tests/files/proposal_extended_csv_report.csv b/tests/files/proposal_extended_csv_report.csv
index 288745f1..d8bf0c56 100644
--- a/tests/files/proposal_extended_csv_report.csv
+++ b/tests/files/proposal_extended_csv_report.csv
@@ -1,3 +1,3 @@
-DevicehubID;DocumentID;Public Link;Lots;Tag 1 Type;Tag 1 ID;Tag 1 Organization;Tag 2 Type;Tag 2 ID;Tag 2 Organization;Tag 3 Type;Tag 3 ID;Tag 3 Organization;Device Hardware ID;Device Type;Device Chassis;Device Serial Number;Device Model;Device Manufacturer;Registered in;Registered (process);Updated in (software);Updated in (web);Physical state;Trading state;Processor;RAM (MB);Data Storage Size (MB);Processor 1;Processor 1 Manufacturer;Processor 1 Model;Processor 1 Serial Number;Processor 1 Number of cores;Processor 1 Speed (GHz);Benchmark Processor 1 (points);Benchmark ProcessorSysbench Processor 1 (points);Processor 2;Processor 2 Manufacturer;Processor 2 Model;Processor 2 Serial Number;Processor 2 Number of cores;Processor 2 Speed (GHz);Benchmark Processor 2 (points);Benchmark ProcessorSysbench Processor 2 (points);RamModule 1;RamModule 1 Manufacturer;RamModule 1 Model;RamModule 1 Serial Number;RamModule 1 Size (MB);RamModule 1 Speed (MHz);RamModule 2;RamModule 2 Manufacturer;RamModule 2 Model;RamModule 2 Serial Number;RamModule 2 Size (MB);RamModule 2 Speed (MHz);RamModule 3;RamModule 3 Manufacturer;RamModule 3 Model;RamModule 3 Serial Number;RamModule 3 Size (MB);RamModule 3 Speed (MHz);RamModule 4;RamModule 4 Manufacturer;RamModule 4 Model;RamModule 4 Serial Number;RamModule 4 Size (MB);RamModule 4 Speed (MHz);DataStorage 1;DataStorage 1 Manufacturer;DataStorage 1 Model;DataStorage 1 Serial Number;DataStorage 1 Size (MB);Erasure DataStorage 1;Erasure DataStorage 1 Serial Number;Erasure DataStorage 1 Size (MB);Erasure DataStorage 1 Software;Erasure DataStorage 1 Result;Erasure DataStorage 1 Type;Erasure DataStorage 1 Method;Erasure DataStorage 1 Elapsed (hours);Erasure DataStorage 1 Date;Erasure DataStorage 1 Steps;Erasure DataStorage 1 Steps Start Time;Erasure DataStorage 1 Steps End Time;Benchmark DataStorage 1 Read Speed (MB/s);Benchmark DataStorage 1 Writing speed (MB/s);Test DataStorage 1 Software;Test DataStorage 1 Type;Test DataStorage 1 Result;Test DataStorage 1 Power on (hours used);Test DataStorage 1 Lifetime remaining (percentage);DataStorage 2;DataStorage 2 Manufacturer;DataStorage 2 Model;DataStorage 2 Serial Number;DataStorage 2 Size (MB);Erasure DataStorage 2;Erasure DataStorage 2 Serial Number;Erasure DataStorage 2 Size (MB);Erasure DataStorage 2 Software;Erasure DataStorage 2 Result;Erasure DataStorage 2 Type;Erasure DataStorage 2 Method;Erasure DataStorage 2 Elapsed (hours);Erasure DataStorage 2 Date;Erasure DataStorage 2 Steps;Erasure DataStorage 2 Steps Start Time;Erasure DataStorage 2 Steps End Time;Benchmark DataStorage 2 Read Speed (MB/s);Benchmark DataStorage 2 Writing speed (MB/s);Test DataStorage 2 Software;Test DataStorage 2 Type;Test DataStorage 2 Result;Test DataStorage 2 Power on (hours used);Test DataStorage 2 Lifetime remaining (percentage);DataStorage 3;DataStorage 3 Manufacturer;DataStorage 3 Model;DataStorage 3 Serial Number;DataStorage 3 Size (MB);Erasure DataStorage 3;Erasure DataStorage 3 Serial Number;Erasure DataStorage 3 Size (MB);Erasure DataStorage 3 Software;Erasure DataStorage 3 Result;Erasure DataStorage 3 Type;Erasure DataStorage 3 Method;Erasure DataStorage 3 Elapsed (hours);Erasure DataStorage 3 Date;Erasure DataStorage 3 Steps;Erasure DataStorage 3 Steps Start Time;Erasure DataStorage 3 Steps End Time;Benchmark DataStorage 3 Read Speed (MB/s);Benchmark DataStorage 3 Writing speed (MB/s);Test DataStorage 3 Software;Test DataStorage 3 Type;Test DataStorage 3 Result;Test DataStorage 3 Power on (hours used);Test DataStorage 3 Lifetime remaining (percentage);DataStorage 4;DataStorage 4 Manufacturer;DataStorage 4 Model;DataStorage 4 Serial Number;DataStorage 4 Size (MB);Erasure DataStorage 4;Erasure DataStorage 4 Serial Number;Erasure DataStorage 4 Size (MB);Erasure DataStorage 4 Software;Erasure DataStorage 4 Result;Erasure DataStorage 4 Type;Erasure DataStorage 4 Method;Erasure DataStorage 4 Elapsed (hours);Erasure DataStorage 4 Date;Erasure DataStorage 4 Steps;Erasure DataStorage 4 Steps Start Time;Erasure DataStorage 4 Steps End Time;Benchmark DataStorage 4 Read Speed (MB/s);Benchmark DataStorage 4 Writing speed (MB/s);Test DataStorage 4 Software;Test DataStorage 4 Type;Test DataStorage 4 Result;Test DataStorage 4 Power on (hours used);Test DataStorage 4 Lifetime remaining (percentage);Motherboard 1;Motherboard 1 Manufacturer;Motherboard 1 Model;Motherboard 1 Serial Number;Display 1;Display 1 Manufacturer;Display 1 Model;Display 1 Serial Number;GraphicCard 1;GraphicCard 1 Manufacturer;GraphicCard 1 Model;GraphicCard 1 Serial Number;GraphicCard 1 Memory (MB);GraphicCard 2;GraphicCard 2 Manufacturer;GraphicCard 2 Model;GraphicCard 2 Serial Number;GraphicCard 2 Memory (MB);NetworkAdapter 1;NetworkAdapter 1 Manufacturer;NetworkAdapter 1 Model;NetworkAdapter 1 Serial Number;NetworkAdapter 2;NetworkAdapter 2 Manufacturer;NetworkAdapter 2 Model;NetworkAdapter 2 Serial Number;SoundCard 1;SoundCard 1 Manufacturer;SoundCard 1 Model;SoundCard 1 Serial Number;SoundCard 2;SoundCard 2 Manufacturer;SoundCard 2 Model;SoundCard 2 Serial Number;Device Rate;Device Range;Processor Rate;Processor Range;RAM Rate;RAM Range;Data Storage Rate;Data Storage Range;Price;Benchmark RamSysbench (points)
-93652;;http://localhost/devices/93652;;named;foo;FooOrg;;;;;;;laptop-asustek_computer_inc-1001pxd-b8oaas048285-14:da:e9:42:f6:7b;Laptop;Netbook;b8oaas048285;1001pxd;asustek computer inc.;Wed May 26 11:37:09 2021;Workbench 11.0a2;2021-05-26 11:37:10.462501+02:00;;;;intel atom cpu n455 @ 2.66ghz;1024;238475;Processor 6: model intel atom cpu n455 @ 2.66ghz, S/N None;intel corp.;intel atom cpu n455 @ 2.66ghz;;1;2.667;6666.24;164.0803;;;;;;;;;RamModule 10: model None, S/N None;;;;1024;667;;;;;;;;;;;;;;;;;;;HardDrive 11: model hts54322, S/N e2024242cv86mm;hitachi;hts54322;e2024242cv86mm;238475;harddrive-hitachi-hts54322-e2024242cv86mm;e2024242cv86mm;238475;Workbench 11.0a2;Success;EraseBasic;Shred;1:16:49;2021-05-26 11:37:09.584933+02:00;✓ – StepRandom 1:16:49;2018-07-03 11:15:22.257059+02:00;2018-07-03 12:32:11.843190+02:00;66.2;21.8;Workbench 11.0a2;Short;Failure;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;Motherboard 12: model 1001pxd, S/N eee0123456720;asustek computer inc.;1001pxd;eee0123456720;;;;;GraphicCard 7: model atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller, S/N None;intel corporation;atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller;;256;;;;;;NetworkAdapter 4: model ar9285 wireless network adapter, S/N 74:2f:68:8b:fd:c9;qualcomm atheros;ar9285 wireless network adapter;74:2f:68:8b:fd:c9;NetworkAdapter 5: model ar8152 v2.0 fast ethernet, S/N 14:da:e9:42:f6:7b;qualcomm atheros;ar8152 v2.0 fast ethernet;14:da:e9:42:f6:7b;SoundCard 8: model nm10/ich7 family high definition audio controller, S/N None;intel corporation;nm10/ich7 family high definition audio controller;;;;;;1.75;LOW;1.55;LOW;1.53;LOW;3.76;HIGH;52.50 €;15.7188
-J2MA2;;http://localhost/devices/J2MA2;;;;;;;;;;;laptop-asustek_computer_inc-1001pxd-b8oaas048287-14:da:e9:42:f6:7c;Laptop;Netbook;b8oaas048287;1001pxd;asustek computer inc.;Wed May 26 11:37:10 2021;Workbench 11.0b11;2021-05-26 11:37:10.149784+02:00;;;;intel atom cpu n455 @ 1.66ghz;2048;558558;Processor 17: model intel atom cpu n455 @ 1.66ghz, S/N None;intel corp.;intel atom cpu n455 @ 1.66ghz;;1;1.667;6666.24;164.0803;;;;;;;;;RamModule 20: model None, S/N None;;;;1024;667;RamModule 21: model 48594d503131325336344350362d53362020, S/N 4f43487b;hynix semiconductor;48594d503131325336344350362d53362020;4f43487b;1024;667;;;;;;;;;;;;;HardDrive 22: model hts54322, S/N e2024242cv86hj;hitachi;hts54322;e2024242cv86hj;238475;harddrive-hitachi-hts54322-e2024242cv86hj;e2024242cv86hj;238475;Workbench 11.0b11;Success;EraseBasic;Shred;1:16:49;2021-05-26 11:37:10.062739+02:00;✓ – StepRandom 1:16:49;2018-07-03 11:15:22.257059+02:00;2018-07-03 12:32:11.843190+02:00;66.2;21.8;Workbench 11.0b11;Extended;Failure;;;DataStorage 23: model wdc wd1600bevt-2, S/N wd-wx11a80w7430;western digital;wdc wd1600bevt-2;wd-wx11a80w7430;160041;datastorage-western_digital-wdc_wd1600bevt-2-wd-wx11a80w7430;wd-wx11a80w7430;160041;Workbench 11.0b11;Failure;EraseBasic;Shred;0:45:36;2021-05-26 11:37:10.065439+02:00;✓ – StepRandom 0:45:36;2019-10-23 09:49:54.410830+02:00;2019-10-23 10:35:31.400587+02:00;41.6;17.3;Workbench 11.0b11;Short;Success;5293;195 days, 12:00:00;SolidStateDrive 24: model wdc wd1600bevt-2, S/N wd-wx11a80w7430;western digital;wdc wd1600bevt-2;wd-wx11a80w7430;160042;solidstatedrive-western_digital-wdc_wd1600bevt-2-wd-wx11a80w7430;wd-wx11a80w7430;160042;Workbench 11.0b11;Success;EraseSectors;Badblocks;1:46:03;2021-05-26 11:37:10.070634+02:00;✓ – StepRandom 0:46:03,✓ – StepZero 1:00:00;2019-08-19 18:48:19.690458+02:00,2019-08-19 19:34:22.690458+02:00;2019-08-19 19:34:22.930562+02:00,2019-08-19 20:34:22.930562+02:00;41.1;17.1;Workbench 11.0b11;Short;Success;5231;194 days, 17:00:00;;;;;;;;;;;;;;;;;;;;;;;;;Motherboard 25: model 1001pxd, S/N eee0123456789;asustek computer inc.;1001pxd;eee0123456789;;"auo ""auo""";auo lcd monitor;;GraphicCard 18: model atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller, S/N None;intel corporation;atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller;;256;;;;;;NetworkAdapter 15: model ar9285 wireless network adapter, S/N 74:2f:68:8b:fd:c8;qualcomm atheros;ar9285 wireless network adapter;74:2f:68:8b:fd:c8;NetworkAdapter 16: model ar8152 v2.0 fast ethernet, S/N 14:da:e9:42:f6:7c;qualcomm atheros;ar8152 v2.0 fast ethernet;14:da:e9:42:f6:7c;SoundCard 9: model usb 2.0 uvc vga webcam, S/N 0x0001;azurewave;usb 2.0 uvc vga webcam;0x0001;SoundCard 19: model nm10/ich7 family high definition audio controller, S/N None;intel corporation;nm10/ich7 family high definition audio controller;;1.72;LOW;1.31;LOW;1.99;LOW;3.97;HIGH;51.60 €;15.7188
+DevicehubID;DocumentID;Public Link;Lots;Tag 1 Type;Tag 1 ID;Tag 1 Organization;Tag 2 Type;Tag 2 ID;Tag 2 Organization;Tag 3 Type;Tag 3 ID;Tag 3 Organization;Device Hardware ID;Device Type;Device Chassis;Device Serial Number;Device Model;Device Manufacturer;Registered in;Registered (process);Updated in (software);Updated in (web);Physical state;Trading state;Processor;RAM (MB);Data Storage Size (MB);Processor 1;Processor 1 Manufacturer;Processor 1 Model;Processor 1 Serial Number;Processor 1 Number of cores;Processor 1 Speed (GHz);Benchmark Processor 1 (points);Benchmark ProcessorSysbench Processor 1 (points);Processor 2;Processor 2 Manufacturer;Processor 2 Model;Processor 2 Serial Number;Processor 2 Number of cores;Processor 2 Speed (GHz);Benchmark Processor 2 (points);Benchmark ProcessorSysbench Processor 2 (points);RamModule 1;RamModule 1 Manufacturer;RamModule 1 Model;RamModule 1 Serial Number;RamModule 1 Size (MB);RamModule 1 Speed (MHz);RamModule 2;RamModule 2 Manufacturer;RamModule 2 Model;RamModule 2 Serial Number;RamModule 2 Size (MB);RamModule 2 Speed (MHz);RamModule 3;RamModule 3 Manufacturer;RamModule 3 Model;RamModule 3 Serial Number;RamModule 3 Size (MB);RamModule 3 Speed (MHz);RamModule 4;RamModule 4 Manufacturer;RamModule 4 Model;RamModule 4 Serial Number;RamModule 4 Size (MB);RamModule 4 Speed (MHz);DataStorage 1;DataStorage 1 Manufacturer;DataStorage 1 Model;DataStorage 1 Serial Number;DataStorage 1 Size (MB);Erasure DataStorage 1;Erasure DataStorage 1 Serial Number;Erasure DataStorage 1 Size (MB);Erasure DataStorage 1 Software;Erasure DataStorage 1 Result;Erasure DataStorage 1 Certificate URL;Erasure DataStorage 1 Type;Erasure DataStorage 1 Method;Erasure DataStorage 1 Elapsed (hours);Erasure DataStorage 1 Date;Erasure DataStorage 1 Steps;Erasure DataStorage 1 Steps Start Time;Erasure DataStorage 1 Steps End Time;Benchmark DataStorage 1 Read Speed (MB/s);Benchmark DataStorage 1 Writing speed (MB/s);Test DataStorage 1 Software;Test DataStorage 1 Type;Test DataStorage 1 Result;Test DataStorage 1 Power on (hours used);Test DataStorage 1 Lifetime remaining (percentage);DataStorage 2;DataStorage 2 Manufacturer;DataStorage 2 Model;DataStorage 2 Serial Number;DataStorage 2 Size (MB);Erasure DataStorage 2;Erasure DataStorage 2 Serial Number;Erasure DataStorage 2 Size (MB);Erasure DataStorage 2 Software;Erasure DataStorage 2 Result;Erasure DataStorage 2 Certificate URL;Erasure DataStorage 2 Type;Erasure DataStorage 2 Method;Erasure DataStorage 2 Elapsed (hours);Erasure DataStorage 2 Date;Erasure DataStorage 2 Steps;Erasure DataStorage 2 Steps Start Time;Erasure DataStorage 2 Steps End Time;Benchmark DataStorage 2 Read Speed (MB/s);Benchmark DataStorage 2 Writing speed (MB/s);Test DataStorage 2 Software;Test DataStorage 2 Type;Test DataStorage 2 Result;Test DataStorage 2 Power on (hours used);Test DataStorage 2 Lifetime remaining (percentage);DataStorage 3;DataStorage 3 Manufacturer;DataStorage 3 Model;DataStorage 3 Serial Number;DataStorage 3 Size (MB);Erasure DataStorage 3;Erasure DataStorage 3 Serial Number;Erasure DataStorage 3 Size (MB);Erasure DataStorage 3 Software;Erasure DataStorage 3 Result;Erasure DataStorage 3 Certificate URL;Erasure DataStorage 3 Type;Erasure DataStorage 3 Method;Erasure DataStorage 3 Elapsed (hours);Erasure DataStorage 3 Date;Erasure DataStorage 3 Steps;Erasure DataStorage 3 Steps Start Time;Erasure DataStorage 3 Steps End Time;Benchmark DataStorage 3 Read Speed (MB/s);Benchmark DataStorage 3 Writing speed (MB/s);Test DataStorage 3 Software;Test DataStorage 3 Type;Test DataStorage 3 Result;Test DataStorage 3 Power on (hours used);Test DataStorage 3 Lifetime remaining (percentage);DataStorage 4;DataStorage 4 Manufacturer;DataStorage 4 Model;DataStorage 4 Serial Number;DataStorage 4 Size (MB);Erasure DataStorage 4;Erasure DataStorage 4 Serial Number;Erasure DataStorage 4 Size (MB);Erasure DataStorage 4 Software;Erasure DataStorage 4 Result;Erasure DataStorage 4 Certificate URL;Erasure DataStorage 4 Type;Erasure DataStorage 4 Method;Erasure DataStorage 4 Elapsed (hours);Erasure DataStorage 4 Date;Erasure DataStorage 4 Steps;Erasure DataStorage 4 Steps Start Time;Erasure DataStorage 4 Steps End Time;Benchmark DataStorage 4 Read Speed (MB/s);Benchmark DataStorage 4 Writing speed (MB/s);Test DataStorage 4 Software;Test DataStorage 4 Type;Test DataStorage 4 Result;Test DataStorage 4 Power on (hours used);Test DataStorage 4 Lifetime remaining (percentage);Motherboard 1;Motherboard 1 Manufacturer;Motherboard 1 Model;Motherboard 1 Serial Number;Display 1;Display 1 Manufacturer;Display 1 Model;Display 1 Serial Number;GraphicCard 1;GraphicCard 1 Manufacturer;GraphicCard 1 Model;GraphicCard 1 Serial Number;GraphicCard 1 Memory (MB);GraphicCard 2;GraphicCard 2 Manufacturer;GraphicCard 2 Model;GraphicCard 2 Serial Number;GraphicCard 2 Memory (MB);NetworkAdapter 1;NetworkAdapter 1 Manufacturer;NetworkAdapter 1 Model;NetworkAdapter 1 Serial Number;NetworkAdapter 2;NetworkAdapter 2 Manufacturer;NetworkAdapter 2 Model;NetworkAdapter 2 Serial Number;SoundCard 1;SoundCard 1 Manufacturer;SoundCard 1 Model;SoundCard 1 Serial Number;SoundCard 2;SoundCard 2 Manufacturer;SoundCard 2 Model;SoundCard 2 Serial Number;Device Rate;Device Range;Processor Rate;Processor Range;RAM Rate;RAM Range;Data Storage Rate;Data Storage Range;Price;Benchmark RamSysbench (points)
+93652;;http://localhost/devices/93652;;named;foo;FooOrg;;;;;;;laptop-asustek_computer_inc-1001pxd-b8oaas048285-14:da:e9:42:f6:7b;Laptop;Netbook;b8oaas048285;1001pxd;asustek computer inc.;Mon Aug  2 10:27:27 2021;Workbench 11.0a2;2021-08-02 10:27:27.772331+02:00;;;;intel atom cpu n455 @ 2.66ghz;1024;238475;Processor 6: model intel atom cpu n455 @ 2.66ghz, S/N None;intel corp.;intel atom cpu n455 @ 2.66ghz;;1;2.667;6666.24;164.0803;;;;;;;;;RamModule 10: model None, S/N None;;;;1024;667;;;;;;;;;;;;;;;;;;;HardDrive 11: model hts54322, S/N e2024242cv86mm;hitachi;hts54322;e2024242cv86mm;238475;harddrive-hitachi-hts54322-e2024242cv86mm;e2024242cv86mm;238475;Workbench 11.0a2;Success;;EraseBasic;Shred;1:16:49;2021-08-02 10:27:27.163611+02:00;✓ – StepRandom 1:16:49;2018-07-03 11:15:22.257059+02:00;2018-07-03 12:32:11.843190+02:00;66.2;21.8;Workbench 11.0a2;Short;Failure;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;Motherboard 12: model 1001pxd, S/N eee0123456720;asustek computer inc.;1001pxd;eee0123456720;;;;;GraphicCard 7: model atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller, S/N None;intel corporation;atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller;;256;;;;;;NetworkAdapter 4: model ar9285 wireless network adapter, S/N 74:2f:68:8b:fd:c9;qualcomm atheros;ar9285 wireless network adapter;74:2f:68:8b:fd:c9;NetworkAdapter 5: model ar8152 v2.0 fast ethernet, S/N 14:da:e9:42:f6:7b;qualcomm atheros;ar8152 v2.0 fast ethernet;14:da:e9:42:f6:7b;SoundCard 8: model nm10/ich7 family high definition audio controller, S/N None;intel corporation;nm10/ich7 family high definition audio controller;;;;;;1.75;LOW;1.55;LOW;1.53;LOW;3.76;HIGH;52.50 €;15.7188
+J2MA2;;http://localhost/devices/J2MA2;;;;;;;;;;;laptop-asustek_computer_inc-1001pxd-b8oaas048287-14:da:e9:42:f6:7c;Laptop;Netbook;b8oaas048287;1001pxd;asustek computer inc.;Mon Aug  2 10:27:27 2021;Workbench 11.0b11;2021-08-02 10:27:27.528832+02:00;;;;intel atom cpu n455 @ 1.66ghz;2048;558558;Processor 17: model intel atom cpu n455 @ 1.66ghz, S/N None;intel corp.;intel atom cpu n455 @ 1.66ghz;;1;1.667;6666.24;164.0803;;;;;;;;;RamModule 20: model None, S/N None;;;;1024;667;RamModule 21: model 48594d503131325336344350362d53362020, S/N 4f43487b;hynix semiconductor;48594d503131325336344350362d53362020;4f43487b;1024;667;;;;;;;;;;;;;HardDrive 22: model hts54322, S/N e2024242cv86hj;hitachi;hts54322;e2024242cv86hj;238475;harddrive-hitachi-hts54322-e2024242cv86hj;e2024242cv86hj;238475;Workbench 11.0b11;Success;;EraseBasic;Shred;1:16:49;2021-08-02 10:27:27.458348+02:00;✓ – StepRandom 1:16:49;2018-07-03 11:15:22.257059+02:00;2018-07-03 12:32:11.843190+02:00;66.2;21.8;Workbench 11.0b11;Extended;Failure;;;DataStorage 23: model wdc wd1600bevt-2, S/N wd-wx11a80w7430;western digital;wdc wd1600bevt-2;wd-wx11a80w7430;160041;datastorage-western_digital-wdc_wd1600bevt-2-wd-wx11a80w7430;wd-wx11a80w7430;160041;Workbench 11.0b11;Failure;;EraseBasic;Shred;0:45:36;2021-08-02 10:27:27.460573+02:00;✓ – StepRandom 0:45:36;2019-10-23 09:49:54.410830+02:00;2019-10-23 10:35:31.400587+02:00;41.6;17.3;Workbench 11.0b11;Short;Success;5293;195 days, 12:00:00;SolidStateDrive 24: model wdc wd1600bevt-2, S/N wd-wx11a80w7430;western digital;wdc wd1600bevt-2;wd-wx11a80w7430;160042;solidstatedrive-western_digital-wdc_wd1600bevt-2-wd-wx11a80w7430;wd-wx11a80w7430;160042;Workbench 11.0b11;Success;;EraseSectors;Badblocks;1:46:03;2021-08-02 10:27:27.464663+02:00;✓ – StepRandom 0:46:03,✓ – StepZero 1:00:00;2019-08-19 18:48:19.690458+02:00,2019-08-19 19:34:22.690458+02:00;2019-08-19 19:34:22.930562+02:00,2019-08-19 20:34:22.930562+02:00;41.1;17.1;Workbench 11.0b11;Short;Success;5231;194 days, 17:00:00;;;;;;;;;;;;;;;;;;;;;;;;;;Motherboard 25: model 1001pxd, S/N eee0123456789;asustek computer inc.;1001pxd;eee0123456789;;"auo ""auo""";auo lcd monitor;;GraphicCard 18: model atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller, S/N None;intel corporation;atom processor d4xx/d5xx/n4xx/n5xx integrated graphics controller;;256;;;;;;NetworkAdapter 15: model ar9285 wireless network adapter, S/N 74:2f:68:8b:fd:c8;qualcomm atheros;ar9285 wireless network adapter;74:2f:68:8b:fd:c8;NetworkAdapter 16: model ar8152 v2.0 fast ethernet, S/N 14:da:e9:42:f6:7c;qualcomm atheros;ar8152 v2.0 fast ethernet;14:da:e9:42:f6:7c;SoundCard 9: model usb 2.0 uvc vga webcam, S/N 0x0001;azurewave;usb 2.0 uvc vga webcam;0x0001;SoundCard 19: model nm10/ich7 family high definition audio controller, S/N None;intel corporation;nm10/ich7 family high definition audio controller;;1.72;LOW;1.31;LOW;1.99;LOW;3.97;HIGH;51.60 €;15.7188
diff --git a/tests/test_documents.py b/tests/test_documents.py
index 1f89d96b..06984640 100644
--- a/tests/test_documents.py
+++ b/tests/test_documents.py
@@ -33,19 +33,22 @@ def test_erasure_certificate_public_one(user: UserClient, client: Client):
     snapshot, _ = user.post(s, res=Snapshot)
 
     doc, response = user.get(res=documents.DocumentDef.t,
-                               item='erasures/{}'.format(snapshot['device']['id']),
-                               accept=ANY)
+                             item='erasures/{}'.format(
+                                 snapshot['device']['id']),
+                             accept=ANY)
     assert 'html' in response.content_type
     assert '<html' in doc
     assert '2018' in doc
 
     doc, response = client.get(res=documents.DocumentDef.t,
-                               item='erasures/{}'.format(snapshot['device']['id']),
+                               item='erasures/{}'.format(
+                                   snapshot['device']['id']),
                                query=[('format', 'PDF')],
                                accept='application/pdf')
     assert 'application/pdf' == response.content_type
 
-    erasure = next(e for e in snapshot['actions'] if e['type'] == 'EraseSectors')
+    erasure = next(e for e in snapshot['actions']
+                   if e['type'] == 'EraseSectors')
 
     doc, response = client.get(res=documents.DocumentDef.t,
                                item='erasures/{}'.format(erasure['id']),
@@ -65,7 +68,8 @@ def test_erasure_certificate_private_query(user: UserClient):
 
     doc, response = user.get(res=documents.DocumentDef.t,
                              item='erasures/',
-                             query=[('filter', {'id': [snapshot['device']['id']]})],
+                             query=[
+                                 ('filter', {'id': [snapshot['device']['id']]})],
                              accept=ANY)
     assert 'html' in response.content_type
     assert '<html' in doc
@@ -74,7 +78,8 @@ def test_erasure_certificate_private_query(user: UserClient):
     doc, response = user.get(res=documents.DocumentDef.t,
                              item='erasures/',
                              query=[
-                                 ('filter', {'id': [snapshot['device']['id']]}),
+                                 ('filter', {
+                                  'id': [snapshot['device']['id']]}),
                                  ('format', 'PDF')
                              ],
                              accept='application/pdf')
@@ -92,19 +97,19 @@ def test_export_csv_permitions(user: UserClient, user2: UserClient, client: Clie
     """test export device information in a csv file with others users."""
     snapshot, _ = user.post(file('basic.snapshot'), res=Snapshot)
     csv_user, _ = user.get(res=documents.DocumentDef.t,
-                          item='devices/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})])
+                           item='devices/',
+                           accept='text/csv',
+                           query=[('filter', {'type': ['Computer']})])
 
     csv_user2, _ = user2.get(res=documents.DocumentDef.t,
-                          item='devices/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})])
+                             item='devices/',
+                             accept='text/csv',
+                             query=[('filter', {'type': ['Computer']})])
 
     _, res = client.get(res=documents.DocumentDef.t,
-                          item='devices/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})], status=401)
+                        item='devices/',
+                        accept='text/csv',
+                        query=[('filter', {'type': ['Computer']})], status=401)
     assert res.status_code == 401
 
     assert len(csv_user) > 0
@@ -122,30 +127,31 @@ def test_export_csv_actions(user: UserClient, user2: UserClient, client: Client)
                     "finalUserCode": "abcdefjhi",
                     "startTime": "2020-11-01T02:00:00+00:00",
                     "endTime": "2020-12-01T02:00:00+00:00"
-    }
+                    }
 
     user.post(res=Allocate, data=post_request)
     hdd = [c for c in acer['components'] if c['type'] == 'HardDrive'][0]
-    hdd_action = [a for a in hdd['actions'] if a['type'] == 'TestDataStorage'][0]
+    hdd_action = [a for a in hdd['actions']
+                  if a['type'] == 'TestDataStorage'][0]
     hdd_action['lifetime'] += 1000
     acer.pop('elapsed')
     acer['licence_version'] = '1.0.0'
     snapshot, _ = client.post(acer, res=Live)
 
     csv_user, _ = user.get(res=documents.DocumentDef.t,
-                          item='actions/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})])
+                           item='actions/',
+                           accept='text/csv',
+                           query=[('filter', {'type': ['Computer']})])
 
     csv_user2, _ = user2.get(res=documents.DocumentDef.t,
-                          item='actions/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})])
+                             item='actions/',
+                             accept='text/csv',
+                             query=[('filter', {'type': ['Computer']})])
 
     _, res = client.get(res=documents.DocumentDef.t,
-                          item='actions/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})], status=401)
+                        item='actions/',
+                        accept='text/csv',
+                        query=[('filter', {'type': ['Computer']})], status=401)
     assert res.status_code == 401
 
     assert len(csv_user) > 0
@@ -164,21 +170,22 @@ def test_live_export_csv2(user: UserClient, client: Client, app: Devicehub):
                     "finalUserCode": "abcdefjhi",
                     "startTime": "2020-11-01T02:00:00+00:00",
                     "endTime": "2020-12-01T02:00:00+00:00"
-    }
+                    }
 
     user.post(res=Allocate, data=post_request)
 
     acer = yaml2json('acer-happy.live-test1')
     live, _ = client.post(acer, res=Live)
     csv_user, _ = user.get(res=documents.DocumentDef.t,
-                          item='actions/',
-                          accept='text/csv',
-                          query=[('filter', {'type': ['Computer']})])
+                           item='actions/',
+                           accept='text/csv',
+                           query=[('filter', {'type': ['Computer']})])
 
     assert "4692" in csv_user
     assert "8692" in csv_user
     assert "DevicehubID" in csv_user
 
+
 @pytest.mark.mvp
 @pytest.mark.usefixtures(conftest.app_context.__name__)
 def test_live_example2(user: UserClient, client: Client, app: Devicehub):
@@ -211,6 +218,7 @@ def test_export_basic_snapshot(user: UserClient):
                           item='devices/',
                           accept='text/csv',
                           query=[('filter', {'type': ['Computer']})])
+    
     f = StringIO(csv_str)
     obj_csv = csv.reader(f, f, delimiter=';', quotechar='"')
     export_csv = list(obj_csv)
@@ -275,9 +283,6 @@ def test_export_extended(app: Devicehub, user: UserClient):
     obj_csv = csv.reader(f, f, delimiter=';', quotechar='"')
     export_csv = list(obj_csv)
 
-    ff= open('ba.csv', 'w')
-    ff.write(csv_str)
-    ff.close()
     # Open fixture csv and transform to list
     with Path(__file__).parent.joinpath('files').joinpath(
             'proposal_extended_csv_report.csv').open() as csv_file:
@@ -290,18 +295,18 @@ def test_export_extended(app: Devicehub, user: UserClient):
     assert fixture_csv[0] == export_csv[0], 'Headers are not equal'
     assert fixture_csv[1][:19] == export_csv[1][:19], 'Computer information are not equal'
     assert fixture_csv[1][20] == export_csv[1][20], 'Computer information are not equal'
-    assert fixture_csv[1][22:81] == export_csv[1][22:81], 'Computer information are not equal'
-    assert fixture_csv[1][82] == export_csv[1][82], 'Computer information are not equal'
-    assert fixture_csv[1][85:] == export_csv[1][85:], 'Computer information are not equal'
+    assert fixture_csv[1][22:82] == export_csv[1][22:82], 'Computer information are not equal'
+    assert fixture_csv[1][83] == export_csv[1][83], 'Computer information are not equal'
+    assert fixture_csv[1][86:] == export_csv[1][86:], 'Computer information are not equal'
     assert fixture_csv[2][:19] == export_csv[2][:19], 'Computer information are not equal'
     assert fixture_csv[2][20] == export_csv[2][20], 'Computer information are not equal'
-    assert fixture_csv[2][22:81] == export_csv[2][22:81], 'Computer information are not equal'
-    assert fixture_csv[2][82] == export_csv[2][82], 'Computer information are not equal'
-    assert fixture_csv[2][85:105] == export_csv[2][85:105], 'Computer information are not equal'
-    assert fixture_csv[2][106] == export_csv[2][106], 'Computer information are not equal'
-    assert fixture_csv[2][109:129] == export_csv[2][109:129], 'Computer information are not equal'
-    assert fixture_csv[2][130] == export_csv[2][130], 'Computer information are not equal'
-    assert fixture_csv[2][133:] == export_csv[2][133:], 'Computer information are not equal'
+    assert fixture_csv[2][22:82] == export_csv[2][22:82], 'Computer information are not equal'
+    assert fixture_csv[2][83] == export_csv[2][83], 'Computer information are not equal'
+    assert fixture_csv[2][86:106] == export_csv[2][86:106], 'Computer information are not equal'
+    assert fixture_csv[2][108] == export_csv[2][108], 'Computer information are not equal'
+    assert fixture_csv[2][110:130] == export_csv[2][110:130], 'Computer information are not equal'
+    assert fixture_csv[2][131] == export_csv[2][131], 'Computer information are not equal'
+    assert fixture_csv[2][134:] == export_csv[2][134:], 'Computer information are not equal'
 
 
 @pytest.mark.mvp

From 8aba022ef52690251b4b9884de0af492a4a6a302 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Mon, 2 Aug 2021 11:42:33 +0200
Subject: [PATCH 18/26] fixing test

---
 tests/test_documents.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_documents.py b/tests/test_documents.py
index 06984640..afb48023 100644
--- a/tests/test_documents.py
+++ b/tests/test_documents.py
@@ -304,9 +304,9 @@ def test_export_extended(app: Devicehub, user: UserClient):
     assert fixture_csv[2][83] == export_csv[2][83], 'Computer information are not equal'
     assert fixture_csv[2][86:106] == export_csv[2][86:106], 'Computer information are not equal'
     assert fixture_csv[2][108] == export_csv[2][108], 'Computer information are not equal'
-    assert fixture_csv[2][110:130] == export_csv[2][110:130], 'Computer information are not equal'
+    assert fixture_csv[2][111:131] == export_csv[2][111:131], 'Computer information are not equal'
     assert fixture_csv[2][131] == export_csv[2][131], 'Computer information are not equal'
-    assert fixture_csv[2][134:] == export_csv[2][134:], 'Computer information are not equal'
+    assert fixture_csv[2][136:] == export_csv[2][136:], 'Computer information are not equal'
 
 
 @pytest.mark.mvp

From bf3dbb329adf4f6ce4414abf6d5a43065c3a45ed Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Tue, 3 Aug 2021 11:29:56 +0200
Subject: [PATCH 19/26] update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01024d2d..8e179164 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ ml).
 
 ## testing
   [1.0.8-beta]
+- [bugfix] #161 fixing DataStorage with bigInteger
 
 ## [1.0.7-beta]
 - [addend] #158 support for encrypted snapshots data

From 6326c0b59d93fe7dbbfad8776b1038a4c240be15 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Tue, 3 Aug 2021 11:55:47 +0200
Subject: [PATCH 20/26] fixing reported_uncorrectable_errors

---
 .../versions/0103a9c96b2d_testdatastorage_bigintegers.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py b/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py
index 87d0626f..e4f8d866 100644
--- a/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py
+++ b/ereuse_devicehub/migrations/versions/0103a9c96b2d_testdatastorage_bigintegers.py
@@ -25,7 +25,7 @@ def get_inv():
 def upgrade():
     op.alter_column('test_data_storage', 'reallocated_sector_count', type_=sa.BigInteger(), schema=f'{get_inv()}')
     op.alter_column('test_data_storage', 'power_cycle_count', type_=sa.Integer(), schema=f'{get_inv()}')
-    op.alter_column('test_data_storage', '_reported_uncorrectable_errors', type_=sa.BigInteger(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'reported_uncorrectable_errors', type_=sa.BigInteger(), schema=f'{get_inv()}')
     op.alter_column('test_data_storage', 'current_pending_sector_count', type_=sa.BigInteger(), schema=f'{get_inv()}')
     op.alter_column('test_data_storage', 'offline_uncorrectable', type_=sa.BigInteger(), schema=f'{get_inv()}')
 
@@ -33,6 +33,6 @@ def upgrade():
 def downgrade():
     op.alter_column('test_data_storage', 'reallocated_sector_count', type_=sa.SmallInteger(), schema=f'{get_inv()}')
     op.alter_column('test_data_storage', 'power_cycle_count', type_=sa.SmallInteger(), schema=f'{get_inv()}')
-    op.alter_column('test_data_storage', '_reported_uncorrectable_errors', type_=sa.Integer(), schema=f'{get_inv()}')
+    op.alter_column('test_data_storage', 'reported_uncorrectable_errors', type_=sa.Integer(), schema=f'{get_inv()}')
     op.alter_column('test_data_storage', 'current_pending_sector_count', type_=sa.Integer(), schema=f'{get_inv()}')
     op.alter_column('test_data_storage', 'offline_uncorrectable', type_=sa.Integer(), schema=f'{get_inv()}')

From d123b4e8a8a9d8f8f5cb94348c36da69bee2571a Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Tue, 3 Aug 2021 13:13:35 +0200
Subject: [PATCH 21/26] up the version

---
 CHANGELOG.md                 | 7 +++++--
 ereuse_devicehub/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e179164..ca25c563 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,10 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ml).
 
 ## master
-  [1.0.7-beta]
+  [1.0.8-beta]
 
 ## testing
-  [1.0.8-beta]
+  [1.0.9-beta]
+
+
+## [1.0.8-beta]
 - [bugfix] #161 fixing DataStorage with bigInteger
 
 ## [1.0.7-beta]
diff --git a/ereuse_devicehub/__init__.py b/ereuse_devicehub/__init__.py
index 811df9bd..d44f1a11 100644
--- a/ereuse_devicehub/__init__.py
+++ b/ereuse_devicehub/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.8-beta"
+__version__ = "1.0.9-beta"

From eed117808cb5738c8180c9c2904fe097a5fcc478 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Wed, 4 Aug 2021 18:15:15 +0200
Subject: [PATCH 22/26] Fixing nullables in models

---
 .../migrations/versions/7ecb8ff7abad_documents.py             | 2 +-
 ereuse_devicehub/resources/documents/models.py                | 4 ++--
 ereuse_devicehub/resources/documents/schemas.py               | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index e75c3470..e111580d 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -70,7 +70,7 @@ def upgrade():
 
     # DataWipe table
     op.create_table('data_wipe',
-                    sa.Column('document_id', sa.BigInteger(), nullable=True),
+                    sa.Column('document_id', sa.BigInteger(), nullable=False),
                     sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
                     sa.ForeignKeyConstraint(['document_id'], [f'{get_inv()}.document.id'], ),
                     sa.ForeignKeyConstraint(['id'], [f'{get_inv()}.action.id'], ),
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index 85aeb03f..fc24f190 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -20,7 +20,7 @@ class Document(Thing):
     date = Column(db.DateTime, nullable=True)
     date.comment = """The date of document, some documents need to have one date
     """
-    id_document = Column(CIText(), nullable=False)
+    id_document = Column(CIText(), nullable=True)
     id_document.comment = """The id of one document like invoice so they can be linked."""
     owner_id = db.Column(UUID(as_uuid=True),
                          db.ForeignKey(User.id),
@@ -48,7 +48,7 @@ class JoinedTableMixin:
 class DataWipeDocument(JoinedTableMixin, Document):
     """This represent a generic document."""
 
-    software = Column(CIText(), nullable=False)
+    software = Column(CIText(), nullable=True)
     software.comment = """Which software is used"""
     success = Column(Boolean)
     success.comment = """If the erase was success"""
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
index 23db2fa2..7129dc50 100644
--- a/ereuse_devicehub/resources/documents/schemas.py
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -15,6 +15,7 @@ class DataWipeDocument(Thing):
                     required=False, 
                     description=m.DataWipeDocument.date.comment)
     id_document = SanitizedStr(data_key='documentId', 
+                               required=False,
                                default='', 
                                description=m.DataWipeDocument.id_document.comment)
     file_name = SanitizedStr(data_key='filename',

From ed43bd477c179f3916fd3ceb20af18485ccdbe8e Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Fri, 6 Aug 2021 11:08:58 +0200
Subject: [PATCH 23/26] we only need the data files

---
 .../migrations/versions/7ecb8ff7abad_documents.py             | 2 +-
 ereuse_devicehub/resources/documents/models.py                | 4 ++--
 ereuse_devicehub/resources/documents/schemas.py               | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index e111580d..087f272c 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -45,7 +45,7 @@ def upgrade():
                     sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=False),
                     sa.Column('file_name', sa.Unicode(), nullable=False),
                     sa.Column('file_hash', sa.Unicode(), nullable=False),
-                    sa.Column('url', sa.Unicode(), nullable=False),
+                    sa.Column('url', sa.Unicode(), nullable=True),
 
                     sa.ForeignKeyConstraint(['owner_id'], ['common.user.id'], ),
                     sa.PrimaryKeyConstraint('id'),
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index fc24f190..ee2ac071 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -31,7 +31,7 @@ class Document(Thing):
     file_name.comment = """This is the name of the file when user up the document."""
     file_hash = Column(db.CIText(), nullable=False)
     file_hash.comment = """This is the hash of the file produced from frontend."""
-    url = db.Column(URL(), nullable=False)
+    url = db.Column(URL(), nullable=True)
     url.comment = """This is the url where resides the document."""
 
     def __str__(self) -> str:
@@ -50,7 +50,7 @@ class DataWipeDocument(JoinedTableMixin, Document):
 
     software = Column(CIText(), nullable=True)
     software.comment = """Which software is used"""
-    success = Column(Boolean)
+    success = Column(Boolean, default=False)
     success.comment = """If the erase was success"""
 
     def __str__(self) -> str:
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
index 7129dc50..c9df245f 100644
--- a/ereuse_devicehub/resources/documents/schemas.py
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -8,8 +8,8 @@ class DataWipeDocument(Thing):
     __doc__ = m.DataWipeDocument.__doc__
     id = Integer(description=m.DataWipeDocument.id.comment, dump_only=True)
     type = SanitizedStr(default='DataWipeDocument')
-    url = URL(description=m.DataWipeDocument.url.comment)
-    success = Boolean(description=m.DataWipeDocument.success.comment)
+    url = URL(required= False, description=m.DataWipeDocument.url.comment)
+    success = Boolean(required=False, default=False, description=m.DataWipeDocument.success.comment)
     software = SanitizedStr(description=m.DataWipeDocument.software.comment)
     date = DateTime(data_key='endTime', 
                     required=False, 

From 9086bb628fffd3cbf4e58e29bc2950e25acd9770 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Tue, 10 Aug 2021 12:55:47 +0200
Subject: [PATCH 24/26] fixing change type for document_type

---
 .../migrations/versions/7ecb8ff7abad_documents.py           | 4 ++--
 ereuse_devicehub/resources/action/views/documents.py        | 1 -
 ereuse_devicehub/resources/documents/models.py              | 2 +-
 ereuse_devicehub/resources/documents/schemas.py             | 6 +++++-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
index 087f272c..83daaf61 100644
--- a/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
+++ b/ereuse_devicehub/migrations/versions/7ecb8ff7abad_documents.py
@@ -39,7 +39,7 @@ def upgrade():
                               comment='The last time Document recorded a change for \n    this thing.\n    '),
                     sa.Column('created', sa.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'),
                               nullable=False, comment='When Document created this.'),
-                    sa.Column('type', sa.Unicode(), nullable=False),
+                    sa.Column('document_type', sa.Unicode(), nullable=False),
                     sa.Column('date', sa.TIMESTAMP(timezone=True), nullable=True),
                     sa.Column('id_document', sa.Unicode(), nullable=True),
                     sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=False),
@@ -54,7 +54,7 @@ def upgrade():
     op.create_index('generic_document_id', 'document', ['id'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
     op.create_index(op.f('ix_document_created'), 'document', ['created'], unique=False, schema=f'{get_inv()}')
     op.create_index(op.f('ix_document_updated'), 'document', ['updated'], unique=False, schema=f'{get_inv()}')
-    op.create_index('document_type_index', 'document', ['type'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
+    op.create_index('document_type_index', 'document', ['document_type'], unique=False, postgresql_using='hash', schema=f'{get_inv()}')
 
 
     # DataWipeDocument table
diff --git a/ereuse_devicehub/resources/action/views/documents.py b/ereuse_devicehub/resources/action/views/documents.py
index 3f3aaa84..e1f685b5 100644
--- a/ereuse_devicehub/resources/action/views/documents.py
+++ b/ereuse_devicehub/resources/action/views/documents.py
@@ -29,7 +29,6 @@ class ErasedView():
         schema = sh_document()
         [data.pop(x, None) for x in ['severity', 'devices', 'name', 'description']]
         doc_data = schema.load(data)
-        doc_data['type'] = 'DataWipe'
         self.document = DataWipeDocument(**doc_data)
         db.session.add(self.document)
         
diff --git a/ereuse_devicehub/resources/documents/models.py b/ereuse_devicehub/resources/documents/models.py
index ee2ac071..a2b98511 100644
--- a/ereuse_devicehub/resources/documents/models.py
+++ b/ereuse_devicehub/resources/documents/models.py
@@ -16,7 +16,7 @@ class Document(Thing):
     id.comment = """The identifier of the device for this database. Used only
     internally for software; users should not use this.
     """
-    type = Column(Unicode(STR_SM_SIZE), nullable=False)
+    document_type = Column(Unicode(STR_SM_SIZE), nullable=False)
     date = Column(db.DateTime, nullable=True)
     date.comment = """The date of document, some documents need to have one date
     """
diff --git a/ereuse_devicehub/resources/documents/schemas.py b/ereuse_devicehub/resources/documents/schemas.py
index c9df245f..6500777a 100644
--- a/ereuse_devicehub/resources/documents/schemas.py
+++ b/ereuse_devicehub/resources/documents/schemas.py
@@ -1,4 +1,5 @@
 from marshmallow.fields import DateTime, Integer, validate, Boolean
+from marshmallow import post_load
 from teal.marshmallow import SanitizedStr, URL
 from ereuse_devicehub.resources.schemas import Thing
 from ereuse_devicehub.resources.documents import models as m
@@ -7,7 +8,6 @@ from ereuse_devicehub.resources.documents import models as m
 class DataWipeDocument(Thing):
     __doc__ = m.DataWipeDocument.__doc__
     id = Integer(description=m.DataWipeDocument.id.comment, dump_only=True)
-    type = SanitizedStr(default='DataWipeDocument')
     url = URL(required= False, description=m.DataWipeDocument.url.comment)
     success = Boolean(required=False, default=False, description=m.DataWipeDocument.success.comment)
     software = SanitizedStr(description=m.DataWipeDocument.software.comment)
@@ -26,3 +26,7 @@ class DataWipeDocument(Thing):
                              default='',
                              description=m.DataWipeDocument.file_hash.comment,
                              validate=validate.Length(max=64))
+
+    @post_load
+    def get_trade_document(self, data):
+        data['document_type'] = 'DataWipeDocument'

From b2390f30108ff5f3cf18fe9d18fb2c57c0a25544 Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Fri, 13 Aug 2021 17:16:31 +0200
Subject: [PATCH 25/26] adding more mimes type

---
 ereuse_devicehub/resources/documents/documents.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py
index f45cf808..7003ee91 100644
--- a/ereuse_devicehub/resources/documents/documents.py
+++ b/ereuse_devicehub/resources/documents/documents.py
@@ -275,7 +275,14 @@ class StampsView(View):
             ok = '100% coincidence. The attached file contains data 100% existing in \
                   to our backend'
             result = ('Bad', bad)
-            if file_check.mimetype in ['text/csv', 'application/pdf']:
+            import pdb; pdb.set_trace()
+            mime = ['text/csv', 'application/pdf', 'text/plain','text/markdown',
+                    'image/jpeg', 'image/png', 'text/html',
+                    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+                    'application/vnd.oasis.opendocument.spreadsheet',
+                    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+                    'application/msword']
+            if file_check.mimetype in mime:
                 if verify_hash(file_check):
                     result = ('Ok', ok)
 

From faa58df4b44cdd69ecf7d999e9df29f9671e422c Mon Sep 17 00:00:00 2001
From: Cayo Puigdefabregas <cayo@ribaguifi.com>
Date: Fri, 13 Aug 2021 17:18:48 +0200
Subject: [PATCH 26/26] clean pdbs

---
 ereuse_devicehub/resources/documents/documents.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ereuse_devicehub/resources/documents/documents.py b/ereuse_devicehub/resources/documents/documents.py
index 7003ee91..98cdf956 100644
--- a/ereuse_devicehub/resources/documents/documents.py
+++ b/ereuse_devicehub/resources/documents/documents.py
@@ -275,7 +275,6 @@ class StampsView(View):
             ok = '100% coincidence. The attached file contains data 100% existing in \
                   to our backend'
             result = ('Bad', bad)
-            import pdb; pdb.set_trace()
             mime = ['text/csv', 'application/pdf', 'text/plain','text/markdown',
                     'image/jpeg', 'image/png', 'text/html',
                     'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',