From 0a9fbb0226ed41ccb32f491232cf02c57546ea1f Mon Sep 17 00:00:00 2001 From: Xavier Bustamante Talavera Date: Tue, 13 Nov 2018 15:52:27 +0100 Subject: [PATCH] Delete lots; add LotParent view; use parents, all_devices relationships; return uiTree with list of lots and parents --- ereuse_devicehub/db.py | 11 +- ereuse_devicehub/resources/lot/models.py | 133 ++++++++++++++-------- ereuse_devicehub/resources/lot/models.pyi | 23 ++-- ereuse_devicehub/resources/lot/views.py | 42 +++---- tests/test_lot.py | 124 +++++++++++--------- 5 files changed, 188 insertions(+), 145 deletions(-) diff --git a/ereuse_devicehub/db.py b/ereuse_devicehub/db.py index 99f8b73c..6e17ad7b 100644 --- a/ereuse_devicehub/db.py +++ b/ereuse_devicehub/db.py @@ -1,5 +1,6 @@ from sqlalchemy import event from sqlalchemy.dialects import postgresql +from sqlalchemy.sql import expression from sqlalchemy_utils import view from teal.db import SchemaSQLAlchemy @@ -18,9 +19,6 @@ class SQLAlchemy(SchemaSQLAlchemy): self.drop_schema(schema='common') -db = SQLAlchemy(session_options={"autoflush": False}) - - def create_view(name, selectable): """Creates a view. @@ -29,7 +27,7 @@ def create_view(name, selectable): sqlalchemy-utils/blob/master/tests/test_views.py>`_ for an example on how to use. """ - table = view.create_table_from_selectable(name=name, selectable=selectable, metadata=None) + table = view.create_table_from_selectable(name, selectable) # We need to ensure views are created / destroyed before / after # SchemaSQLAlchemy's listeners execute @@ -37,3 +35,8 @@ def create_view(name, selectable): event.listen(db.metadata, 'after_create', view.CreateView(name, selectable), insert=True) event.listen(db.metadata, 'before_drop', view.DropView(name)) return table + + +db = SQLAlchemy(session_options={"autoflush": False}) +f = db.func +exp = expression diff --git a/ereuse_devicehub/resources/lot/models.py b/ereuse_devicehub/resources/lot/models.py index 8604965c..a8615358 100644 --- a/ereuse_devicehub/resources/lot/models.py +++ b/ereuse_devicehub/resources/lot/models.py @@ -7,13 +7,12 @@ from citext import CIText from flask import g from sqlalchemy import TEXT from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy.sql import expression as exp from sqlalchemy_utils import LtreeType from sqlalchemy_utils.types.ltree import LQUERY from teal.db import CASCADE_OWN, UUIDLtree from teal.resource import url_for_resource -from ereuse_devicehub.db import create_view, db +from ereuse_devicehub.db import create_view, db, exp, f from ereuse_devicehub.resources.device.models import Component, Device from ereuse_devicehub.resources.models import Thing from ereuse_devicehub.resources.user.models import User @@ -31,6 +30,7 @@ class Lot(Thing): devices = db.relationship(Device, backref=db.backref('lots', lazy=True, collection_class=set), secondary=lambda: LotDevice.__table__, + lazy=True, collection_class=set) """ The **children** devices that the lot has. @@ -38,6 +38,32 @@ class Lot(Thing): Note that the lot can have more devices, if they are inside descendant lots. """ + parents = db.relationship(lambda: Lot, + viewonly=True, + lazy=True, + collection_class=set, + secondary=lambda: LotParent.__table__, + primaryjoin=lambda: Lot.id == LotParent.child_id, + secondaryjoin=lambda: LotParent.parent_id == Lot.id, + cascade='refresh-expire', # propagate changes outside ORM + backref=db.backref('children', + viewonly=True, + lazy=True, + cascade='refresh-expire', + collection_class=set) + ) + """The parent lots.""" + + all_devices = db.relationship(Device, + viewonly=True, + lazy=True, + collection_class=set, + secondary=lambda: LotDeviceDescendants.__table__, + primaryjoin=lambda: Lot.id == LotDeviceDescendants.ancestor_lot_id, + secondaryjoin=lambda: LotDeviceDescendants.device_id == Device.id) + """All devices, including components, inside this lot and its + descendants. + """ def __init__(self, name: str, closed: bool = closed.default.arg, description: str = None) -> None: @@ -49,38 +75,11 @@ class Lot(Thing): super().__init__(id=uuid.uuid4(), name=name, closed=closed, description=description) Path(self) # Lots have always one edge per default. - def add_child(self, child): - """Adds a child to this lot.""" - if isinstance(child, Lot): - Path.add(self.id, child.id) - db.session.refresh(self) # todo is this useful? - db.session.refresh(child) - else: - assert isinstance(child, uuid.UUID) - Path.add(self.id, child) - db.session.refresh(self) # todo is this useful? - - def remove_child(self, child): - if isinstance(child, Lot): - Path.delete(self.id, child.id) - else: - assert isinstance(child, uuid.UUID) - Path.delete(self.id, child) - @property def url(self) -> urlutils.URL: """The URL where to GET this event.""" return urlutils.URL(url_for_resource(Lot, item_id=self.id)) - @property - def children(self): - """The children lots.""" - # From https://stackoverflow.com/a/41158890 - id = UUIDLtree.convert(self.id) - return self.query \ - .join(self.__class__.paths) \ - .filter(Path.path.lquery(exp.cast('*.{}.*{{1}}'.format(id), LQUERY))) - @property def descendants(self): return self.descendantsq(self.id) @@ -90,26 +89,45 @@ class Lot(Thing): _id = UUIDLtree.convert(id) return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY)) - @property - def parents(self): - return self.parentsq(self.id) - - @classmethod - def parentsq(cls, id: UUID): - """The parent lots.""" - id = UUIDLtree.convert(id) - i = db.func.index(Path.path, id) - parent_id = db.func.replace(exp.cast(db.func.subpath(Path.path, i - 1, i), TEXT), '_', '-') - join_clause = parent_id == exp.cast(Lot.id, TEXT) - return cls.query.join(Path, join_clause).filter( - Path.path.lquery(exp.cast('*{{1}}.{}.*'.format(id), LQUERY)) - ) - @classmethod def roots(cls): """Gets the lots that are not under any other lot.""" return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1) + def add_children(self, *children): + """Add children lots to this lot. + + This operation is highly costly as it forces refreshing + many models in session. + """ + for child in children: + if isinstance(child, Lot): + Path.add(self.id, child.id) + db.session.refresh(child) + else: + assert isinstance(child, uuid.UUID) + Path.add(self.id, child) + # We need to refresh the models involved in this operation + # outside the session / ORM control so the models + # that have relationships to this model + # with the cascade 'refresh-expire' can welcome the changes + db.session.refresh(self) + + def remove_children(self, *children): + """Remove children lots from this lot. + + This operation is highly costly as it forces refreshing + many models in session. + """ + for child in children: + if isinstance(child, Lot): + Path.delete(self.id, child.id) + db.session.refresh(child) + else: + assert isinstance(child, uuid.UUID) + Path.delete(self.id, child) + db.session.refresh(self) + def delete(self): """Deletes the lot. @@ -117,10 +135,15 @@ class Lot(Thing): devices orphan from this lot and then marks this lot for deletion. """ - for child in self.children: - self.remove_child(child) + self.remove_children(*self.children) db.session.delete(self) + def _refresh_models_with_relationships_to_lots(self): + session = db.Session.object_session(self) + for model in session: + if isinstance(model, (Device, Lot, Path)): + session.expire(model) + def __contains__(self, child: Union['Lot', Device]): if isinstance(child, Lot): return Path.has_lot(self.id, child.id) @@ -225,8 +248,8 @@ class LotDeviceDescendants(db.Model): """Query that gets the descendants of the ancestor lot.""" devices = db.select([ LotDevice.device_id, - _ancestor.c.id.label('ancestor_lot_id'), _desc.c.id.label('parent_lot_id'), + _ancestor.c.id.label('ancestor_lot_id'), None ]).select_from(_ancestor).select_from(lot_device).where(descendants) @@ -240,12 +263,22 @@ class LotDeviceDescendants(db.Model): components = db.select([ Component.id.label('device_id'), - _ancestor.c.id.label('ancestor_lot_id'), _desc.c.id.label('parent_lot_id'), + _ancestor.c.id.label('ancestor_lot_id'), LotDevice.device_id.label('device_parent_id'), ]).select_from(_ancestor).select_from(lot_device_component).where(descendants) + __table__ = create_view('lot_device_descendants', devices.union(components)) + + +class LotParent(db.Model): + i = f.index(Path.path, db.func.text2ltree(f.replace(exp.cast(Path.lot_id, TEXT), '-', '_'))) + __table__ = create_view( - name='lot_device_descendants', - selectable=devices.union(components) + 'lot_parent', + db.select([ + Path.lot_id.label('child_id'), + exp.cast(f.replace(exp.cast(f.subltree(Path.path, i - 1, i), TEXT), '_', '-'), + UUID).label('parent_id') + ]).select_from(Path).where(i > 0), ) diff --git a/ereuse_devicehub/resources/lot/models.pyi b/ereuse_devicehub/resources/lot/models.pyi index abf518df..2c820b0c 100644 --- a/ereuse_devicehub/resources/lot/models.pyi +++ b/ereuse_devicehub/resources/lot/models.pyi @@ -22,6 +22,8 @@ class Lot(Thing): devices = ... # type: relationship paths = ... # type: relationship description = ... # type: Column + all_devices = ... # type: relationship + parents = ... # type: relationship def __init__(self, name: str, closed: bool = closed.default.arg) -> None: super().__init__() @@ -30,22 +32,21 @@ class Lot(Thing): self.closed = ... # type: bool self.devices = ... # type: Set[Device] self.paths = ... # type: Set[Path] - description = ... # type: str + self.description = ... # type: str + self.all_devices = ... # type: Set[Device] + self.parents = ... # type: Set[Lot] + self.children = ... # type: Set[Lot] - def add_child(self, child: Union[Lot, uuid.UUID]): + def add_children(self, *children: Union[Lot, uuid.UUID]): pass - def remove_child(self, child: Union[Lot, uuid.UUID]): + def remove_children(self, *children: Union[Lot, uuid.UUID]): pass @classmethod def roots(cls) -> LotQuery: pass - @property - def children(self) -> LotQuery: - pass - @property def descendants(self) -> LotQuery: pass @@ -54,14 +55,6 @@ class Lot(Thing): def descendantsq(cls, id) -> LotQuery: pass - @property - def parents(self) -> LotQuery: - pass - - @classmethod - def parentsq(cls, id) -> LotQuery: - pass - @property def url(self) -> urlutils.URL: pass diff --git a/ereuse_devicehub/resources/lot/views.py b/ereuse_devicehub/resources/lot/views.py index 5f5c447b..32aaa179 100644 --- a/ereuse_devicehub/resources/lot/views.py +++ b/ereuse_devicehub/resources/lot/views.py @@ -1,7 +1,7 @@ import uuid from collections import deque from enum import Enum -from typing import List, Set +from typing import Dict, List, Set, Union import marshmallow as ma from flask import Response, jsonify, request @@ -67,10 +67,12 @@ class LotView(View): you can filter. """ if args['format'] == LotFormat.UiTree: - return jsonify({ - 'items': self.ui_tree(), + lots = self.schema.dump(Lot.query, many=True, nested=1) + ret = { + 'items': {l['id']: l for l in lots}, + 'tree': self.ui_tree(), 'url': request.path - }) + } else: query = Lot.query if args['search']: @@ -87,15 +89,7 @@ class LotView(View): }, 'url': request.path } - return jsonify(ret) - - @classmethod - def ui_tree(cls) -> List[dict]: - nodes = [] - for model in Path.query: # type: Path - path = deque(model.path.path.split('.')) - cls._p(nodes, path) - return nodes + return jsonify(ret) def delete(self, id): lot = Lot.query.filter_by(id=id).one() @@ -104,7 +98,15 @@ class LotView(View): return Response(status=204) @classmethod - def _p(cls, nodes: List[dict], path: deque): + def ui_tree(cls) -> List[Dict]: + tree = [] + for model in Path.query: # type: Path + path = deque(model.path.path.split('.')) + cls._p(tree, path) + return tree + + @classmethod + def _p(cls, nodes: List[Dict[str, Union[uuid.UUID, List]]], path: deque): """Recursively creates the nested lot structure. Every recursive step consumes path (a deque of lot_id), @@ -116,14 +118,8 @@ class LotView(View): # does lot_id exist already in node? node = next(part for part in nodes if lot_id == part['id']) except StopIteration: - lot = Lot.query.filter_by(id=lot_id).one() node = { 'id': lot_id, - 'name': lot.name, - 'url': lot.url.to_text(), - 'closed': lot.closed, - 'updated': lot.updated, - 'created': lot.created, 'nodes': [] } nodes.append(node) @@ -180,12 +176,10 @@ class LotChildrenView(LotBaseChildrenView): id = ma.fields.List(ma.fields.UUID()) def _post(self, lot: Lot, ids: Set[uuid.UUID]): - for id in ids: - lot.add_child(id) # todo what to do if child exists already? + lot.add_children(*ids) def _delete(self, lot: Lot, ids: Set[uuid.UUID]): - for id in ids: - lot.remove_child(id) + lot.remove_children(*ids) class LotDeviceView(LotBaseChildrenView): diff --git a/tests/test_lot.py b/tests/test_lot.py index 1891554d..21799a05 100644 --- a/tests/test_lot.py +++ b/tests/test_lot.py @@ -37,23 +37,33 @@ def test_lot_model_children(): l1, l2, l3 = lots db.session.add_all(lots) db.session.flush() + assert not l1.children + assert not l1.parents + assert not l2.children + assert not l2.parents + assert not l3.parents + assert not l3.children - l1.add_child(l2) - db.session.flush() + l1.add_children(l2) + assert l1.children == {l2} + assert l2.parents == {l1} - assert list(l1.children) == [l2] - - l2.add_child(l3) - assert list(l1.children) == [l2] + l2.add_children(l3) + assert l1.children == {l2} + assert l2.parents == {l1} + assert l2.children == {l3} + assert l3.parents == {l2} l2.delete() db.session.flush() - assert not list(l1.children) + assert not l1.children + assert not l3.parents l1.delete() db.session.flush() l3b = Lot.query.one() assert l3 == l3b + assert not l3.parents def test_lot_modify_patch_endpoint_and_delete(user: UserClient): @@ -87,8 +97,8 @@ def test_lot_device_relationship(): assert lot_device.created assert lot_device.author_id == g.user.id assert device.lots == {child} - # todo Device IN LOT does not work assert device in child + assert device in child.all_devices graphic = GraphicCard(serial_number='foo', model='bar') device.components.add(graphic) @@ -98,7 +108,7 @@ def test_lot_device_relationship(): parent = Lot('parent') db.session.add(parent) db.session.flush() - parent.add_child(child) + parent.add_children(child) assert child in parent @@ -111,13 +121,13 @@ def test_add_edge(): db.session.add(parent) db.session.flush() - parent.add_child(child) + parent.add_children(child) assert child in parent assert len(child.paths) == 1 assert len(parent.paths) == 1 - parent.remove_child(child) + parent.remove_children(child) assert child not in parent assert len(child.paths) == 1 assert len(parent.paths) == 1 @@ -126,8 +136,8 @@ def test_add_edge(): db.session.add(grandparent) db.session.flush() - grandparent.add_child(parent) - parent.add_child(child) + grandparent.add_children(parent) + parent.add_children(child) assert parent in grandparent assert child in parent @@ -148,31 +158,36 @@ def test_lot_multiple_parents(auth_app_context): db.session.add_all(lots) db.session.flush() - grandparent1.add_child(parent) + grandparent1.add_children(parent) assert parent in grandparent1 - parent.add_child(child) + parent.add_children(child) assert child in parent assert child in grandparent1 - grandparent2.add_child(parent) + grandparent2.add_children(parent) assert parent in grandparent1 assert parent in grandparent2 assert child in parent assert child in grandparent1 assert child in grandparent2 + p = parent.id + c = child.id + gp1 = grandparent1.id + gp2 = grandparent2.id + nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree() - assert nodes[0]['name'] == 'grandparent1' - assert nodes[0]['nodes'][0]['name'] == 'parent' - assert nodes[0]['nodes'][0]['nodes'][0]['name'] == 'child' + assert nodes[0]['id'] == gp1 + assert nodes[0]['nodes'][0]['id'] == p + assert nodes[0]['nodes'][0]['nodes'][0]['id'] == c assert nodes[0]['nodes'][0]['nodes'][0]['nodes'] == [] - assert nodes[1]['name'] == 'grandparent2' - assert nodes[1]['nodes'][0]['name'] == 'parent' - assert nodes[1]['nodes'][0]['nodes'][0]['name'] == 'child' + assert nodes[1]['id'] == gp2 + assert nodes[1]['nodes'][0]['id'] == p + assert nodes[1]['nodes'][0]['nodes'][0]['id'] == c assert nodes[1]['nodes'][0]['nodes'][0]['nodes'] == [] # Now remove all childs - grandparent1.remove_child(parent) + grandparent1.remove_children(parent) assert parent not in grandparent1 assert child in parent assert parent in grandparent2 @@ -180,14 +195,14 @@ def test_lot_multiple_parents(auth_app_context): assert child in grandparent2 nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree() - assert nodes[0]['name'] == 'grandparent1' + assert nodes[0]['id'] == gp1 assert nodes[0]['nodes'] == [] - assert nodes[1]['name'] == 'grandparent2' - assert nodes[1]['nodes'][0]['name'] == 'parent' - assert nodes[1]['nodes'][0]['nodes'][0]['name'] == 'child' + assert nodes[1]['id'] == gp2 + assert nodes[1]['nodes'][0]['id'] == p + assert nodes[1]['nodes'][0]['nodes'][0]['id'] == c assert nodes[1]['nodes'][0]['nodes'][0]['nodes'] == [] - grandparent2.remove_child(parent) + grandparent2.remove_children(parent) assert parent not in grandparent2 assert parent not in grandparent1 assert child not in grandparent2 @@ -195,27 +210,27 @@ def test_lot_multiple_parents(auth_app_context): assert child in parent nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree() - assert nodes[0]['name'] == 'grandparent1' + assert nodes[0]['id'] == gp1 assert nodes[0]['nodes'] == [] - assert nodes[1]['name'] == 'grandparent2' + assert nodes[1]['id'] == gp2 assert nodes[1]['nodes'] == [] - assert nodes[2]['name'] == 'parent' - assert nodes[2]['nodes'][0]['name'] == 'child' + assert nodes[2]['id'] == p + assert nodes[2]['nodes'][0]['id'] == c assert nodes[2]['nodes'][0]['nodes'] == [] - parent.remove_child(child) + parent.remove_children(child) assert child not in parent assert len(child.paths) == 1 assert len(parent.paths) == 1 nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree() - assert nodes[0]['name'] == 'grandparent1' + assert nodes[0]['id'] == gp1 assert nodes[0]['nodes'] == [] - assert nodes[1]['name'] == 'grandparent2' + assert nodes[1]['id'] == gp2 assert nodes[1]['nodes'] == [] - assert nodes[2]['name'] == 'parent' + assert nodes[2]['id'] == p assert nodes[2]['nodes'] == [] - assert nodes[3]['name'] == 'child' + assert nodes[3]['id'] == c assert nodes[3]['nodes'] == [] @@ -243,29 +258,29 @@ def test_lot_unite_graphs_and_find(): db.session.add_all(lots) db.session.flush() - l1.add_child(l2) + l1.add_children(l2) assert l2 in l1 - l3.add_child(l2) + l3.add_children(l2) assert l2 in l3 - l5.add_child(l7) + l5.add_children(l7) assert l7 in l5 - l4.add_child(l5) + l4.add_children(l5) assert l5 in l4 assert l7 in l4 - l5.add_child(l8) + l5.add_children(l8) assert l8 in l5 - l4.add_child(l6) + l4.add_children(l6) assert l6 in l4 - l6.add_child(l5) + l6.add_children(l5) assert l5 in l6 and l5 in l4 # We unite the two graphs - l2.add_child(l4) + l2.add_children(l4) assert l4 in l2 and l5 in l2 and l6 in l2 and l7 in l2 and l8 in l2 assert l4 in l3 and l5 in l3 and l6 in l3 and l7 in l3 and l8 in l3 # We remove the union - l2.remove_child(l4) + l2.remove_children(l4) assert l4 not in l2 and l5 not in l2 and l6 not in l2 and l7 not in l2 and l8 not in l2 assert l4 not in l3 and l5 not in l3 and l6 not in l3 and l7 not in l3 and l8 not in l3 @@ -279,7 +294,7 @@ def test_lot_roots(): db.session.flush() assert set(Lot.roots()) == {l1, l2, l3} - l1.add_child(l2) + l1.add_children(l2) assert set(Lot.roots()) == {l1, l3} @@ -306,11 +321,16 @@ def test_lot_post_add_children_view_ui_tree_normal(user: UserClient): assert child['parents'][0]['id'] == parent['id'] # Format UiTree - lots = user.get(res=Lot, query=[('format', 'UiTree')])[0]['items'] - assert 1 == len(lots) - assert lots[0]['name'] == 'Parent' - assert len(lots[0]['nodes']) == 1 - assert lots[0]['nodes'][0]['name'] == 'Child' + r = user.get(res=Lot, query=[('format', 'UiTree')])[0] + lots, nodes = r['items'], r['tree'] + assert 1 == len(nodes) + assert nodes[0]['id'] == parent['id'] + assert len(nodes[0]['nodes']) == 1 + assert nodes[0]['nodes'][0]['id'] == child['id'] + assert 2 == len(lots) + assert 'Parent' == lots[parent['id']]['name'] + assert 'Child' == lots[child['id']]['name'] + assert lots[child['id']]['parents'][0]['name'] == 'Parent' # Normal list format lots = user.get(res=Lot)[0]['items']