Draft on lot

README.md

@@ -22,7 +22,7 @@ Devicehub is built with [Teal](https://github.com/bustawin/teal) and
 The requirements are:
 
 - Python 3.5.3 or higher. In debian 9 is `# apt install python3-pip`.
-- PostgreSQL 9.6 or higher. In debian 9 is `# apt install postgresql`
+- PostgreSQL 9.6 or higher. In debian 9 is `# apt install postgresql-contrib`
 - passlib. In debian 9 is `# apt install python3-passlib`.
 
 Install Devicehub with *pip*: `pip3 install ereuse-devicehub -U --pre`.
@@ -39,6 +39,8 @@ postgres $ createdb devicehub  # Create main database
 postgres $ psql devicehub  # Access to the database
 postgres $ CREATE USER dhub WITH PASSWORD 'ereuse';  # Create user devicehub uses to access db
 postgres $ GRANT ALL PRIVILEGES ON DATABASE devicehub TO dhub;  # Give access to the db
+postgres $ CREATE EXTENSION pgcrypto SCHEMA public; # Enable pgcrypto
+postgres $ CREATE EXTENSION ltree SCHEMA public; # Enable ltree
 postgres $ \q
 exit
 ```

ereuse_devicehub/config.py

@@ -2,7 +2,7 @@ from distutils.version import StrictVersion
 from itertools import chain
 from typing import Set
 
-from ereuse_devicehub.resources import agent, device, event, inventory, tag, user
+from ereuse_devicehub.resources import agent, device, event, inventory, lot, tag, user
 from ereuse_devicehub.resources.enums import PriceSoftware, RatingSoftware
 from teal.auth import TokenAuth
 from teal.config import Config
@@ -16,7 +16,8 @@ class DevicehubConfig(Config):
                                      import_resource(user),
                                      import_resource(tag),
                                      import_resource(inventory),
-                                     import_resource(agent)))
+                                     import_resource(agent),
+                                     import_resource(lot)))
     PASSWORD_SCHEMES = {'pbkdf2_sha256'}  # type: Set[str]
     SQLALCHEMY_DATABASE_URI = 'postgresql://dhub:ereuse@localhost/devicehub'  # type: str
     SCHEMA = 'dhub'

ereuse_devicehub/db.py

@@ -1,7 +1,16 @@
+from sqlalchemy.dialects import postgresql
+
 from teal.db import SQLAlchemy as _SQLAlchemy
 
 
 class SQLAlchemy(_SQLAlchemy):
+    """
+    Superuser must create the required extensions in the public
+    schema of the database, as it is in the `search_path`
+    defined in teal.
+    """
+    UUID = postgresql.UUID
+
     def drop_all(self, bind='__all__', app=None):
         """A faster nuke-like option to drop everything."""
         self.drop_schema()

ereuse_devicehub/resources/event/models.py

@@ -51,7 +51,7 @@ class Event(Thing):
     incidence.comment = """
         Should this event be reviewed due some anomaly?
     """
-    closed = Column(Boolean, default=True, nullable=False)
+    closed = Column(Boolean, default=False, nullable=False)
     closed.comment = """
         Whether the author has finished the event.
         After this is set to True, no modifications are allowed.
@@ -100,7 +100,7 @@ class Event(Thing):
     author = relationship(User,
                           backref=backref('authored_events', lazy=True, collection_class=set),
                           primaryjoin=author_id == User.id)
-    """
+    author_id.comment = """
     The user that recorded this action in the system.
      
     This does not necessarily has to be the person that produced
@@ -118,7 +118,7 @@ class Event(Thing):
                                          lazy=True,
                                          collection_class=OrderedSet,
                                          order_by=lambda: Event.created),
-                         primaryjoin=agent_id == Agent.id, )
+                         primaryjoin=agent_id == Agent.id)
     agent_id.comment = """
     The direct performer or driver of the action. e.g. John wrote a book.
     

ereuse_devicehub/resources/event/schemas.py

@@ -24,7 +24,7 @@ from teal.resource import Schema
 
 class Event(Thing):
     id = UUID(dump_only=True)
-    name = String(default='', validate=Length(STR_BIG_SIZE), description=m.Event.name.comment)
+    name = String(default='', validate=Length(max=STR_BIG_SIZE), description=m.Event.name.comment)
     incidence = Boolean(default=False, description=m.Event.incidence.comment)
     closed = Boolean(missing=True, description=m.Event.closed.comment)
     error = Boolean(default=False, description=m.Event.error.comment)

ereuse_devicehub/resources/lot/__init__.py

@@ -0,0 +1,19 @@
+import pathlib
+
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.lot import schemas
+from ereuse_devicehub.resources.lot.views import LotView
+from teal.resource import Converters, Resource
+
+
+class LotDef(Resource):
+    SCHEMA = schemas.Lot
+    VIEW = LotView
+    AUTH = True
+    ID_CONVERTER = Converters.uuid
+
+    def init_db(self, db: 'db.SQLAlchemy'):
+        # Create functions
+        with pathlib.Path(__file__).parent.joinpath('dag.sql').open() as f:
+            sql = f.read()
+            db.session.execute(sql)

ereuse_devicehub/resources/lot/dag.sql

@@ -0,0 +1,110 @@
+CREATE OR REPLACE FUNCTION add_edge(parent_id uuid, child_id uuid)
+  /* Adds an edge between ``parent`` and ``child``.
+
+  Designed to work with Directed Acyclic Graphs (DAG)
+  (or said in another way, trees with multiple parents without cycles).
+
+  This method will raise an exception if:
+  - Parent is the same as child.
+  - Child contains the parent.
+  - Edge parent - child already exists.
+
+  Influenced by:
+  - https://www.codeproject.com/Articles/22824/A-Model-to-Represent-Directed-Acyclic-Graphs-DAG
+  - http://patshaughnessy.net/2017/12/12/installing-the-postgres-ltree-extension
+  - https://en.wikipedia.org/wiki/Directed_acyclic_graph
+   */
+  RETURNS void AS $$
+DECLARE
+  parent text := replace(CAST(parent_id as text), '-', '_');
+  child  text := replace(CAST(child_id as text), '-', '_');
+BEGIN
+  if parent = child
+  then
+    raise exception 'Cannot create edge: the parent is the same as the child.';
+  end if;
+
+  if exists(
+      select 1 from edge where edge.path ~ CAST('*.' || child || '.*.' || parent || '.*' as lquery)
+  )
+  then
+    raise exception 'Cannot create edge: child already contains parent.';
+  end if;
+
+  -- We have two subgraphs: the parent subgraph that goes from the parent to the root,
+  -- and the child subgraph, going from the child (which is the root of this subgraph)
+  -- to all the leafs.
+  -- We do the cartesian product from all the paths of the parent subgraph that end in the parent
+  -- WITH all the paths that start from the child that end to its leafs.
+  insert into edge (lot_id, path) (select distinct lot_id, fp.path ||
+                                                           subpath(edge.path, index(edge.path, text2ltree(child)))
+                                   from edge,
+                                        (select path
+                                         from edge
+                                         where path ~ CAST('*.' || parent AS lquery)) as fp
+                                   where edge.path ~ CAST('*.' || child || '.*' AS lquery));
+  -- Cleanup: old paths that start with the child (that where used above in the cartesian product)
+  -- have became a subset of the result of the cartesian product, thus being redundant.
+  delete from edge where edge.path ~ CAST(child || '.*' AS lquery);
+END
+$$
+LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION delete_edge(parent_id uuid, child_id uuid)
+  /* Deletes an edge between ``parent`` and ``child``.
+
+  Designed to work with DAG (See ``add_edge`` function).
+
+  This method will raise an exception if the relationship does not
+  exist.
+   */
+  RETURNS void AS $$
+DECLARE
+  parent text := replace(CAST(parent_id as text), '-', '_');
+  child  text := replace(CAST(child_id as text), '-', '_');
+  number int;
+BEGIN
+  -- to delete we remove from the path of the descendants of the child
+  -- (and the child) any ancestor coming from this edge.
+  -- When we added the edge we did a cartesian product. When removing
+  -- this part of the path we will have duplicate paths.
+
+  -- don't check uniqueness for path key until we delete duplicates
+  SET CONSTRAINTS edge_path_unique DEFERRED;
+
+  -- remove everything above the child lot_id in the path
+  -- this creates duplicates on path and lot_id
+  update edge
+  set path = subpath(path, index(path, text2ltree(child)))
+  where path ~ CAST('*.' || parent || '.' || child || '.*' AS lquery);
+
+  -- remove duplicates
+  -- we need an id field exclusively for this operation
+  -- from https://wiki.postgresql.org/wiki/Deleting_duplicates
+  DELETE
+  FROM edge
+  WHERE id IN (SELECT id
+               FROM (SELECT id, ROW_NUMBER() OVER (partition BY lot_id, path) AS rnum FROM edge) t
+               WHERE t.rnum > 1);
+
+  -- re-activate uniqueness check and perform check
+  SET CONSTRAINTS edge_path_unique IMMEDIATE;
+
+  -- After the update the one of the paths of the child will be
+  -- containing only the child.
+  -- This can only be when the child has no parent at all.
+  -- In case the child has more than one parent, remove this path
+  -- (note that we want it to remove it too from descendants of this
+  -- child, ex. 'child_id'.'desc1')
+  select COUNT(1) into number from edge where lot_id = child_id;
+  IF number > 1
+  THEN
+    delete from edge where path <@ text2ltree(child);
+  end if;
+
+END
+$$
+LANGUAGE plpgsql;
+
+
+

ereuse_devicehub/resources/lot/models.py

@@ -0,0 +1,101 @@
+import uuid
+from datetime import datetime
+from typing import Set
+
+from flask import g
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.sql import expression
+from sqlalchemy_utils import LtreeType
+from sqlalchemy_utils.types.ltree import LQUERY
+
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.device.models import Device
+from ereuse_devicehub.resources.models import STR_SIZE, Thing
+from ereuse_devicehub.resources.user.models import User
+
+
+class Lot(Thing):
+    id = db.Column(UUID(as_uuid=True),
+                   primary_key=True,
+                   server_default=db.text('gen_random_uuid()'))
+    name = db.Column(db.Unicode(STR_SIZE), nullable=False)
+    closed = db.Column(db.Boolean, default=False, nullable=False)
+    closed.comment = """
+            A closed lot cannot be modified anymore.
+        """
+    devices = db.relationship(Device,
+                              backref=db.backref('parents', lazy=True, collection_class=set),
+                              secondary=lambda: LotDevice.__table__,
+                              collection_class=set)
+
+    def __repr__(self) -> str:
+        return '<Lot {0.name} devices={0.devices!r}>'.format(self)
+
+    def add_child(self, child: 'Lot'):
+        """Adds a child to this lot."""
+        Edge.add(self.id, child.id)
+        db.session.refresh(self)  # todo is this useful?
+        db.session.refresh(child)
+
+    def remove_child(self, child: 'Lot'):
+        Edge.delete(self.id, child.id)
+
+    def __contains__(self, child: 'Lot'):
+        return Edge.has_lot(self.id, child.id)
+
+
+class LotDevice(db.Model):
+    device_id = db.Column(db.BigInteger, db.ForeignKey(Device.id), primary_key=True)
+    lot_id = db.Column(UUID(as_uuid=True), db.ForeignKey(Lot.id), primary_key=True)
+    created = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+    author_id = db.Column(UUID(as_uuid=True),
+                          db.ForeignKey(User.id),
+                          nullable=False,
+                          default=lambda: g.user.id)
+    author = db.relationship(User, primaryjoin=author_id == User.id)
+    author_id.comment = """
+        The user that put the device in the lot.
+    """
+
+
+class Edge(Thing):
+    id = db.Column(db.UUID(as_uuid=True),
+                   primary_key=True,
+                   server_default=db.text('gen_random_uuid()'))
+    lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False)
+    lot = db.relationship(Lot,
+                          backref=db.backref('edges', lazy=True, collection_class=set),
+                          primaryjoin=Lot.id == lot_id)
+    path = db.Column(LtreeType, unique=True, nullable=False)
+
+    __table_args__ = (
+        db.UniqueConstraint(path, name='edge_path_unique', deferrable=True, initially='immediate'),
+        db.Index('path_gist', path, postgresql_using='gist'),
+        db.Index('path_btree', path, postgresql_using='btree')
+    )
+
+    def children(self) -> Set['Edge']:
+        """Get the children edges."""
+        # From https://stackoverflow.com/a/41158890
+        exp = '*.{}.*{{1}}'.format(self.lot_id)
+        return set(self.query
+                   .filter(self.path.lquery(expression.cast(exp, LQUERY)))
+                   .distinct(self.__class__.lot_id)
+                   .all())
+
+    @classmethod
+    def add(cls, parent_id: uuid.UUID, child_id: uuid.UUID):
+        """Creates an edge between parent and child."""
+        db.session.execute(db.func.add_edge(str(parent_id), str(child_id)))
+
+    @classmethod
+    def delete(cls, parent_id: uuid.UUID, child_id: uuid.UUID):
+        """Deletes the edge between parent and child."""
+        db.session.execute(db.func.delete_edge(str(parent_id), str(child_id)))
+
+    @classmethod
+    def has_lot(cls, parent_id: uuid.UUID, child_id: uuid.UUID) -> bool:
+        return bool(db.session.execute(
+            "SELECT 1 from edge where path ~ '*.{}.*.{}.*'".format(
+                str(parent_id).replace('-', '_'), str(child_id).replace('-', '_'))
+        ).first())

ereuse_devicehub/resources/lot/schemas.py

@@ -0,0 +1,14 @@
+from marshmallow import fields as f
+
+from ereuse_devicehub.marshmallow import NestedOn
+from ereuse_devicehub.resources.device.schemas import Device
+from ereuse_devicehub.resources.lot import models as m
+from ereuse_devicehub.resources.models import STR_SIZE
+from ereuse_devicehub.resources.schemas import Thing
+
+
+class Lot(Thing):
+    id = f.UUID(dump_only=True)
+    name = f.String(validate=f.validate.Length(max=STR_SIZE))
+    closed = f.String(required=True, missing=False, description=m.Lot.closed.comment)
+    devices = f.String(NestedOn(Device, many=True, collection_class=set, only_query='id'))

ereuse_devicehub/resources/lot/views.py

@@ -0,0 +1,25 @@
+import uuid
+
+from flask import current_app as app, request
+
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.lot.models import Lot
+from teal.resource import View
+
+
+class LotView(View):
+    def post(self):
+        json = request.get_json(validate=False)
+        e = app.resources[json['type']].schema.load(json)
+        Model = db.Model._decl_class_registry.data[json['type']]()
+        lot = Model(**e)
+        db.session.add(lot)
+        db.session.commit()
+        ret = self.schema.jsonify(lot)
+        ret.status_code = 201
+        return ret
+
+    def one(self, id: uuid.UUID):
+        """Gets one event."""
+        event = Lot.query.filter_by(id=id).one()
+        return self.schema.jsonify(event)

ereuse_devicehub/resources/tag/model.py

@@ -38,7 +38,7 @@ class Tag(Thing):
         return value
 
     __table_args__ = (
-        UniqueConstraint(device_id, org_id, name='One tag per organization.'),
+        UniqueConstraint(device_id, org_id, name='one_tag_per_organization'),
     )
 
     def __repr__(self) -> str:

tests/test_basic.py

@@ -24,7 +24,8 @@ def test_api_docs(client: Client):
         '/tags/',
         '/snapshots/',
         '/users/login',
-        '/events/'
+        '/events/',
+        '/lots/'
     }
     assert docs['info'] == {'title': 'Devicehub', 'version': '0.2'}
     assert docs['components']['securitySchemes']['bearerAuth'] == {
@@ -35,4 +36,4 @@ def test_api_docs(client: Client):
         'scheme': 'basic',
         'name': 'Authorization'
     }
-    assert 76 == len(docs['definitions'])
+    assert 77 == len(docs['definitions'])

tests/test_lot.py

@@ -0,0 +1,65 @@
+import pytest
+from flask import g
+from sqlalchemy_utils import Ltree
+
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.device.models import Desktop
+from ereuse_devicehub.resources.enums import ComputerChassis
+from ereuse_devicehub.resources.lot.models import Edge, Lot, LotDevice
+from tests import conftest
+
+
+@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
+def test_lot_device_relationship():
+    device = Desktop(serial_number='foo',
+                     model='bar',
+                     manufacturer='foobar',
+                     chassis=ComputerChassis.Lunchbox)
+    lot = Lot(name='lot1')
+    lot.devices.add(device)
+    db.session.add(lot)
+    db.session.flush()
+
+    lot_device = LotDevice.query.one()  # type: LotDevice
+    assert lot_device.device_id == device.id
+    assert lot_device.lot_id == lot.id
+    assert lot_device.created
+    assert lot_device.author_id == g.user.id
+    assert device.parents == {lot}
+
+
+@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
+def test_add_edge():
+    child = Lot(name='child')
+    parent = Lot(name='parent')
+    db.session.add(child)
+    db.session.add(parent)
+    db.session.flush()
+    # todo edges should automatically be created when the lot is created
+    child.edges.add(Edge(path=Ltree(str(child.id).replace('-', '_'))))
+    parent.edges.add(Edge(path=Ltree(str(parent.id).replace('-', '_'))))
+    db.session.flush()
+
+    parent.add_child(child)
+
+    assert child in parent
+    assert len(child.edges) == 1
+    assert len(parent.edges) == 1
+
+    parent.remove_child(child)
+    assert child not in parent
+    assert len(child.edges) == 1
+    assert len(parent.edges) == 1
+
+    grandparent = Lot(name='grandparent')
+    db.session.add(grandparent)
+    db.session.flush()
+    grandparent.edges.add(Edge(path=Ltree(str(grandparent.id).replace('-', '_'))))
+    db.session.flush()
+
+    grandparent.add_child(parent)
+    parent.add_child(child)
+
+    assert parent in grandparent
+    assert child in parent
+    assert child in grandparent

tests/test_rate.py

@@ -8,9 +8,10 @@ from ereuse_devicehub.resources.enums import Bios, ComputerChassis, ImageMimeTyp
     RatingSoftware
 from ereuse_devicehub.resources.event.models import PhotoboxRate, WorkbenchRate
 from ereuse_devicehub.resources.image.models import Image, ImageList
+from tests import conftest
 
 
-@pytest.mark.usefixtures('auth_app_context')
+@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
 def test_workbench_rate_db():
     rate = WorkbenchRate(processor=0.1,
                          ram=1.0,
@@ -25,7 +26,7 @@ def test_workbench_rate_db():
     db.session.commit()
 
 
-@pytest.mark.usefixtures('auth_app_context')
+@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
 def test_photobox_rate_db():
     pc = Desktop(serial_number='24', chassis=ComputerChassis.Tower)
     image = Image(name='foo',