From f5d69070e609ad582e28cc6d8fcf98f1ca5e2633 Mon Sep 17 00:00:00 2001 From: Xavier Bustamante Talavera Date: Wed, 8 Aug 2018 21:25:53 +0200 Subject: [PATCH] Draft on lot --- README.md | 4 +- ereuse_devicehub/config.py | 5 +- ereuse_devicehub/db.py | 9 ++ ereuse_devicehub/resources/event/models.py | 6 +- ereuse_devicehub/resources/event/schemas.py | 2 +- ereuse_devicehub/resources/lot/__init__.py | 19 ++++ ereuse_devicehub/resources/lot/dag.sql | 110 ++++++++++++++++++++ ereuse_devicehub/resources/lot/models.py | 101 ++++++++++++++++++ ereuse_devicehub/resources/lot/schemas.py | 14 +++ ereuse_devicehub/resources/lot/views.py | 25 +++++ ereuse_devicehub/resources/tag/model.py | 2 +- tests/test_basic.py | 5 +- tests/test_lot.py | 65 ++++++++++++ tests/test_rate.py | 5 +- 14 files changed, 360 insertions(+), 12 deletions(-) create mode 100644 ereuse_devicehub/resources/lot/__init__.py create mode 100644 ereuse_devicehub/resources/lot/dag.sql create mode 100644 ereuse_devicehub/resources/lot/models.py create mode 100644 ereuse_devicehub/resources/lot/schemas.py create mode 100644 ereuse_devicehub/resources/lot/views.py create mode 100644 tests/test_lot.py diff --git a/README.md b/README.md index b8364792..01e06e66 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Devicehub is built with [Teal](https://github.com/bustawin/teal) and The requirements are: - Python 3.5.3 or higher. In debian 9 is `# apt install python3-pip`. -- PostgreSQL 9.6 or higher. In debian 9 is `# apt install postgresql` +- PostgreSQL 9.6 or higher. In debian 9 is `# apt install postgresql-contrib` - passlib. In debian 9 is `# apt install python3-passlib`. Install Devicehub with *pip*: `pip3 install ereuse-devicehub -U --pre`. @@ -39,6 +39,8 @@ postgres $ createdb devicehub # Create main database postgres $ psql devicehub # Access to the database postgres $ CREATE USER dhub WITH PASSWORD 'ereuse'; # Create user devicehub uses to access db postgres $ GRANT ALL PRIVILEGES ON DATABASE devicehub TO dhub; # Give access to the db +postgres $ CREATE EXTENSION pgcrypto SCHEMA public; # Enable pgcrypto +postgres $ CREATE EXTENSION ltree SCHEMA public; # Enable ltree postgres $ \q exit ``` diff --git a/ereuse_devicehub/config.py b/ereuse_devicehub/config.py index 4aa65f45..a0505213 100644 --- a/ereuse_devicehub/config.py +++ b/ereuse_devicehub/config.py @@ -2,7 +2,7 @@ from distutils.version import StrictVersion from itertools import chain from typing import Set -from ereuse_devicehub.resources import agent, device, event, inventory, tag, user +from ereuse_devicehub.resources import agent, device, event, inventory, lot, tag, user from ereuse_devicehub.resources.enums import PriceSoftware, RatingSoftware from teal.auth import TokenAuth from teal.config import Config @@ -16,7 +16,8 @@ class DevicehubConfig(Config): import_resource(user), import_resource(tag), import_resource(inventory), - import_resource(agent))) + import_resource(agent), + import_resource(lot))) PASSWORD_SCHEMES = {'pbkdf2_sha256'} # type: Set[str] SQLALCHEMY_DATABASE_URI = 'postgresql://dhub:ereuse@localhost/devicehub' # type: str SCHEMA = 'dhub' diff --git a/ereuse_devicehub/db.py b/ereuse_devicehub/db.py index 956ce55d..9c1661f1 100644 --- a/ereuse_devicehub/db.py +++ b/ereuse_devicehub/db.py @@ -1,7 +1,16 @@ +from sqlalchemy.dialects import postgresql + from teal.db import SQLAlchemy as _SQLAlchemy class SQLAlchemy(_SQLAlchemy): + """ + Superuser must create the required extensions in the public + schema of the database, as it is in the `search_path` + defined in teal. + """ + UUID = postgresql.UUID + def drop_all(self, bind='__all__', app=None): """A faster nuke-like option to drop everything.""" self.drop_schema() diff --git a/ereuse_devicehub/resources/event/models.py b/ereuse_devicehub/resources/event/models.py index 766c8bea..b532cb97 100644 --- a/ereuse_devicehub/resources/event/models.py +++ b/ereuse_devicehub/resources/event/models.py @@ -51,7 +51,7 @@ class Event(Thing): incidence.comment = """ Should this event be reviewed due some anomaly? """ - closed = Column(Boolean, default=True, nullable=False) + closed = Column(Boolean, default=False, nullable=False) closed.comment = """ Whether the author has finished the event. After this is set to True, no modifications are allowed. @@ -100,7 +100,7 @@ class Event(Thing): author = relationship(User, backref=backref('authored_events', lazy=True, collection_class=set), primaryjoin=author_id == User.id) - """ + author_id.comment = """ The user that recorded this action in the system. This does not necessarily has to be the person that produced @@ -118,7 +118,7 @@ class Event(Thing): lazy=True, collection_class=OrderedSet, order_by=lambda: Event.created), - primaryjoin=agent_id == Agent.id, ) + primaryjoin=agent_id == Agent.id) agent_id.comment = """ The direct performer or driver of the action. e.g. John wrote a book. diff --git a/ereuse_devicehub/resources/event/schemas.py b/ereuse_devicehub/resources/event/schemas.py index 5f1aedf8..b5080b15 100644 --- a/ereuse_devicehub/resources/event/schemas.py +++ b/ereuse_devicehub/resources/event/schemas.py @@ -24,7 +24,7 @@ from teal.resource import Schema class Event(Thing): id = UUID(dump_only=True) - name = String(default='', validate=Length(STR_BIG_SIZE), description=m.Event.name.comment) + name = String(default='', validate=Length(max=STR_BIG_SIZE), description=m.Event.name.comment) incidence = Boolean(default=False, description=m.Event.incidence.comment) closed = Boolean(missing=True, description=m.Event.closed.comment) error = Boolean(default=False, description=m.Event.error.comment) diff --git a/ereuse_devicehub/resources/lot/__init__.py b/ereuse_devicehub/resources/lot/__init__.py new file mode 100644 index 00000000..cd69c849 --- /dev/null +++ b/ereuse_devicehub/resources/lot/__init__.py @@ -0,0 +1,19 @@ +import pathlib + +from ereuse_devicehub.db import db +from ereuse_devicehub.resources.lot import schemas +from ereuse_devicehub.resources.lot.views import LotView +from teal.resource import Converters, Resource + + +class LotDef(Resource): + SCHEMA = schemas.Lot + VIEW = LotView + AUTH = True + ID_CONVERTER = Converters.uuid + + def init_db(self, db: 'db.SQLAlchemy'): + # Create functions + with pathlib.Path(__file__).parent.joinpath('dag.sql').open() as f: + sql = f.read() + db.session.execute(sql) diff --git a/ereuse_devicehub/resources/lot/dag.sql b/ereuse_devicehub/resources/lot/dag.sql new file mode 100644 index 00000000..3807bde5 --- /dev/null +++ b/ereuse_devicehub/resources/lot/dag.sql @@ -0,0 +1,110 @@ +CREATE OR REPLACE FUNCTION add_edge(parent_id uuid, child_id uuid) + /* Adds an edge between ``parent`` and ``child``. + + Designed to work with Directed Acyclic Graphs (DAG) + (or said in another way, trees with multiple parents without cycles). + + This method will raise an exception if: + - Parent is the same as child. + - Child contains the parent. + - Edge parent - child already exists. + + Influenced by: + - https://www.codeproject.com/Articles/22824/A-Model-to-Represent-Directed-Acyclic-Graphs-DAG + - http://patshaughnessy.net/2017/12/12/installing-the-postgres-ltree-extension + - https://en.wikipedia.org/wiki/Directed_acyclic_graph + */ + RETURNS void AS $$ +DECLARE + parent text := replace(CAST(parent_id as text), '-', '_'); + child text := replace(CAST(child_id as text), '-', '_'); +BEGIN + if parent = child + then + raise exception 'Cannot create edge: the parent is the same as the child.'; + end if; + + if exists( + select 1 from edge where edge.path ~ CAST('*.' || child || '.*.' || parent || '.*' as lquery) + ) + then + raise exception 'Cannot create edge: child already contains parent.'; + end if; + + -- We have two subgraphs: the parent subgraph that goes from the parent to the root, + -- and the child subgraph, going from the child (which is the root of this subgraph) + -- to all the leafs. + -- We do the cartesian product from all the paths of the parent subgraph that end in the parent + -- WITH all the paths that start from the child that end to its leafs. + insert into edge (lot_id, path) (select distinct lot_id, fp.path || + subpath(edge.path, index(edge.path, text2ltree(child))) + from edge, + (select path + from edge + where path ~ CAST('*.' || parent AS lquery)) as fp + where edge.path ~ CAST('*.' || child || '.*' AS lquery)); + -- Cleanup: old paths that start with the child (that where used above in the cartesian product) + -- have became a subset of the result of the cartesian product, thus being redundant. + delete from edge where edge.path ~ CAST(child || '.*' AS lquery); +END +$$ +LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_edge(parent_id uuid, child_id uuid) + /* Deletes an edge between ``parent`` and ``child``. + + Designed to work with DAG (See ``add_edge`` function). + + This method will raise an exception if the relationship does not + exist. + */ + RETURNS void AS $$ +DECLARE + parent text := replace(CAST(parent_id as text), '-', '_'); + child text := replace(CAST(child_id as text), '-', '_'); + number int; +BEGIN + -- to delete we remove from the path of the descendants of the child + -- (and the child) any ancestor coming from this edge. + -- When we added the edge we did a cartesian product. When removing + -- this part of the path we will have duplicate paths. + + -- don't check uniqueness for path key until we delete duplicates + SET CONSTRAINTS edge_path_unique DEFERRED; + + -- remove everything above the child lot_id in the path + -- this creates duplicates on path and lot_id + update edge + set path = subpath(path, index(path, text2ltree(child))) + where path ~ CAST('*.' || parent || '.' || child || '.*' AS lquery); + + -- remove duplicates + -- we need an id field exclusively for this operation + -- from https://wiki.postgresql.org/wiki/Deleting_duplicates + DELETE + FROM edge + WHERE id IN (SELECT id + FROM (SELECT id, ROW_NUMBER() OVER (partition BY lot_id, path) AS rnum FROM edge) t + WHERE t.rnum > 1); + + -- re-activate uniqueness check and perform check + SET CONSTRAINTS edge_path_unique IMMEDIATE; + + -- After the update the one of the paths of the child will be + -- containing only the child. + -- This can only be when the child has no parent at all. + -- In case the child has more than one parent, remove this path + -- (note that we want it to remove it too from descendants of this + -- child, ex. 'child_id'.'desc1') + select COUNT(1) into number from edge where lot_id = child_id; + IF number > 1 + THEN + delete from edge where path <@ text2ltree(child); + end if; + +END +$$ +LANGUAGE plpgsql; + + + diff --git a/ereuse_devicehub/resources/lot/models.py b/ereuse_devicehub/resources/lot/models.py new file mode 100644 index 00000000..f8cd4d77 --- /dev/null +++ b/ereuse_devicehub/resources/lot/models.py @@ -0,0 +1,101 @@ +import uuid +from datetime import datetime +from typing import Set + +from flask import g +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.sql import expression +from sqlalchemy_utils import LtreeType +from sqlalchemy_utils.types.ltree import LQUERY + +from ereuse_devicehub.db import db +from ereuse_devicehub.resources.device.models import Device +from ereuse_devicehub.resources.models import STR_SIZE, Thing +from ereuse_devicehub.resources.user.models import User + + +class Lot(Thing): + id = db.Column(UUID(as_uuid=True), + primary_key=True, + server_default=db.text('gen_random_uuid()')) + name = db.Column(db.Unicode(STR_SIZE), nullable=False) + closed = db.Column(db.Boolean, default=False, nullable=False) + closed.comment = """ + A closed lot cannot be modified anymore. + """ + devices = db.relationship(Device, + backref=db.backref('parents', lazy=True, collection_class=set), + secondary=lambda: LotDevice.__table__, + collection_class=set) + + def __repr__(self) -> str: + return ''.format(self) + + def add_child(self, child: 'Lot'): + """Adds a child to this lot.""" + Edge.add(self.id, child.id) + db.session.refresh(self) # todo is this useful? + db.session.refresh(child) + + def remove_child(self, child: 'Lot'): + Edge.delete(self.id, child.id) + + def __contains__(self, child: 'Lot'): + return Edge.has_lot(self.id, child.id) + + +class LotDevice(db.Model): + device_id = db.Column(db.BigInteger, db.ForeignKey(Device.id), primary_key=True) + lot_id = db.Column(UUID(as_uuid=True), db.ForeignKey(Lot.id), primary_key=True) + created = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + author_id = db.Column(UUID(as_uuid=True), + db.ForeignKey(User.id), + nullable=False, + default=lambda: g.user.id) + author = db.relationship(User, primaryjoin=author_id == User.id) + author_id.comment = """ + The user that put the device in the lot. + """ + + +class Edge(Thing): + id = db.Column(db.UUID(as_uuid=True), + primary_key=True, + server_default=db.text('gen_random_uuid()')) + lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False) + lot = db.relationship(Lot, + backref=db.backref('edges', lazy=True, collection_class=set), + primaryjoin=Lot.id == lot_id) + path = db.Column(LtreeType, unique=True, nullable=False) + + __table_args__ = ( + db.UniqueConstraint(path, name='edge_path_unique', deferrable=True, initially='immediate'), + db.Index('path_gist', path, postgresql_using='gist'), + db.Index('path_btree', path, postgresql_using='btree') + ) + + def children(self) -> Set['Edge']: + """Get the children edges.""" + # From https://stackoverflow.com/a/41158890 + exp = '*.{}.*{{1}}'.format(self.lot_id) + return set(self.query + .filter(self.path.lquery(expression.cast(exp, LQUERY))) + .distinct(self.__class__.lot_id) + .all()) + + @classmethod + def add(cls, parent_id: uuid.UUID, child_id: uuid.UUID): + """Creates an edge between parent and child.""" + db.session.execute(db.func.add_edge(str(parent_id), str(child_id))) + + @classmethod + def delete(cls, parent_id: uuid.UUID, child_id: uuid.UUID): + """Deletes the edge between parent and child.""" + db.session.execute(db.func.delete_edge(str(parent_id), str(child_id))) + + @classmethod + def has_lot(cls, parent_id: uuid.UUID, child_id: uuid.UUID) -> bool: + return bool(db.session.execute( + "SELECT 1 from edge where path ~ '*.{}.*.{}.*'".format( + str(parent_id).replace('-', '_'), str(child_id).replace('-', '_')) + ).first()) diff --git a/ereuse_devicehub/resources/lot/schemas.py b/ereuse_devicehub/resources/lot/schemas.py new file mode 100644 index 00000000..59529954 --- /dev/null +++ b/ereuse_devicehub/resources/lot/schemas.py @@ -0,0 +1,14 @@ +from marshmallow import fields as f + +from ereuse_devicehub.marshmallow import NestedOn +from ereuse_devicehub.resources.device.schemas import Device +from ereuse_devicehub.resources.lot import models as m +from ereuse_devicehub.resources.models import STR_SIZE +from ereuse_devicehub.resources.schemas import Thing + + +class Lot(Thing): + id = f.UUID(dump_only=True) + name = f.String(validate=f.validate.Length(max=STR_SIZE)) + closed = f.String(required=True, missing=False, description=m.Lot.closed.comment) + devices = f.String(NestedOn(Device, many=True, collection_class=set, only_query='id')) diff --git a/ereuse_devicehub/resources/lot/views.py b/ereuse_devicehub/resources/lot/views.py new file mode 100644 index 00000000..e2a8dcaf --- /dev/null +++ b/ereuse_devicehub/resources/lot/views.py @@ -0,0 +1,25 @@ +import uuid + +from flask import current_app as app, request + +from ereuse_devicehub.db import db +from ereuse_devicehub.resources.lot.models import Lot +from teal.resource import View + + +class LotView(View): + def post(self): + json = request.get_json(validate=False) + e = app.resources[json['type']].schema.load(json) + Model = db.Model._decl_class_registry.data[json['type']]() + lot = Model(**e) + db.session.add(lot) + db.session.commit() + ret = self.schema.jsonify(lot) + ret.status_code = 201 + return ret + + def one(self, id: uuid.UUID): + """Gets one event.""" + event = Lot.query.filter_by(id=id).one() + return self.schema.jsonify(event) diff --git a/ereuse_devicehub/resources/tag/model.py b/ereuse_devicehub/resources/tag/model.py index 30c90eac..1bb01364 100644 --- a/ereuse_devicehub/resources/tag/model.py +++ b/ereuse_devicehub/resources/tag/model.py @@ -38,7 +38,7 @@ class Tag(Thing): return value __table_args__ = ( - UniqueConstraint(device_id, org_id, name='One tag per organization.'), + UniqueConstraint(device_id, org_id, name='one_tag_per_organization'), ) def __repr__(self) -> str: diff --git a/tests/test_basic.py b/tests/test_basic.py index a57697ab..616f9af7 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -24,7 +24,8 @@ def test_api_docs(client: Client): '/tags/', '/snapshots/', '/users/login', - '/events/' + '/events/', + '/lots/' } assert docs['info'] == {'title': 'Devicehub', 'version': '0.2'} assert docs['components']['securitySchemes']['bearerAuth'] == { @@ -35,4 +36,4 @@ def test_api_docs(client: Client): 'scheme': 'basic', 'name': 'Authorization' } - assert 76 == len(docs['definitions']) + assert 77 == len(docs['definitions']) diff --git a/tests/test_lot.py b/tests/test_lot.py new file mode 100644 index 00000000..7c914600 --- /dev/null +++ b/tests/test_lot.py @@ -0,0 +1,65 @@ +import pytest +from flask import g +from sqlalchemy_utils import Ltree + +from ereuse_devicehub.db import db +from ereuse_devicehub.resources.device.models import Desktop +from ereuse_devicehub.resources.enums import ComputerChassis +from ereuse_devicehub.resources.lot.models import Edge, Lot, LotDevice +from tests import conftest + + +@pytest.mark.usefixtures(conftest.auth_app_context.__name__) +def test_lot_device_relationship(): + device = Desktop(serial_number='foo', + model='bar', + manufacturer='foobar', + chassis=ComputerChassis.Lunchbox) + lot = Lot(name='lot1') + lot.devices.add(device) + db.session.add(lot) + db.session.flush() + + lot_device = LotDevice.query.one() # type: LotDevice + assert lot_device.device_id == device.id + assert lot_device.lot_id == lot.id + assert lot_device.created + assert lot_device.author_id == g.user.id + assert device.parents == {lot} + + +@pytest.mark.usefixtures(conftest.auth_app_context.__name__) +def test_add_edge(): + child = Lot(name='child') + parent = Lot(name='parent') + db.session.add(child) + db.session.add(parent) + db.session.flush() + # todo edges should automatically be created when the lot is created + child.edges.add(Edge(path=Ltree(str(child.id).replace('-', '_')))) + parent.edges.add(Edge(path=Ltree(str(parent.id).replace('-', '_')))) + db.session.flush() + + parent.add_child(child) + + assert child in parent + assert len(child.edges) == 1 + assert len(parent.edges) == 1 + + parent.remove_child(child) + assert child not in parent + assert len(child.edges) == 1 + assert len(parent.edges) == 1 + + grandparent = Lot(name='grandparent') + db.session.add(grandparent) + db.session.flush() + grandparent.edges.add(Edge(path=Ltree(str(grandparent.id).replace('-', '_')))) + db.session.flush() + + grandparent.add_child(parent) + parent.add_child(child) + + assert parent in grandparent + assert child in parent + assert child in grandparent diff --git a/tests/test_rate.py b/tests/test_rate.py index d1bf05dd..ff196fce 100644 --- a/tests/test_rate.py +++ b/tests/test_rate.py @@ -8,9 +8,10 @@ from ereuse_devicehub.resources.enums import Bios, ComputerChassis, ImageMimeTyp RatingSoftware from ereuse_devicehub.resources.event.models import PhotoboxRate, WorkbenchRate from ereuse_devicehub.resources.image.models import Image, ImageList +from tests import conftest -@pytest.mark.usefixtures('auth_app_context') +@pytest.mark.usefixtures(conftest.auth_app_context.__name__) def test_workbench_rate_db(): rate = WorkbenchRate(processor=0.1, ram=1.0, @@ -25,7 +26,7 @@ def test_workbench_rate_db(): db.session.commit() -@pytest.mark.usefixtures('auth_app_context') +@pytest.mark.usefixtures(conftest.auth_app_context.__name__) def test_photobox_rate_db(): pc = Desktop(serial_number='24', chassis=ComputerChassis.Tower) image = Image(name='foo',