Delete lots; add LotParent view; use parents, all_devices relationships; return uiTree with list of lots and parents

ereuse_devicehub/db.py

@@ -1,5 +1,6 @@
 from sqlalchemy import event
 from sqlalchemy.dialects import postgresql
+from sqlalchemy.sql import expression
 from sqlalchemy_utils import view
 from teal.db import SchemaSQLAlchemy
 
@@ -18,9 +19,6 @@ class SQLAlchemy(SchemaSQLAlchemy):
         self.drop_schema(schema='common')
 
 
-db = SQLAlchemy(session_options={"autoflush": False})
-
-
 def create_view(name, selectable):
     """Creates a view.
 
@@ -29,7 +27,7 @@ def create_view(name, selectable):
     sqlalchemy-utils/blob/master/tests/test_views.py>`_ for an
     example on how to use.
     """
-    table = view.create_table_from_selectable(name=name, selectable=selectable, metadata=None)
+    table = view.create_table_from_selectable(name, selectable)
 
     # We need to ensure views are created / destroyed before / after
     # SchemaSQLAlchemy's listeners execute
@@ -37,3 +35,8 @@ def create_view(name, selectable):
     event.listen(db.metadata, 'after_create', view.CreateView(name, selectable), insert=True)
     event.listen(db.metadata, 'before_drop', view.DropView(name))
     return table
+
+
+db = SQLAlchemy(session_options={"autoflush": False})
+f = db.func
+exp = expression

ereuse_devicehub/resources/lot/models.py

@@ -7,13 +7,12 @@ from citext import CIText
 from flask import g
 from sqlalchemy import TEXT
 from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.sql import expression as exp
 from sqlalchemy_utils import LtreeType
 from sqlalchemy_utils.types.ltree import LQUERY
 from teal.db import CASCADE_OWN, UUIDLtree
 from teal.resource import url_for_resource
 
-from ereuse_devicehub.db import create_view, db
+from ereuse_devicehub.db import create_view, db, exp, f
 from ereuse_devicehub.resources.device.models import Component, Device
 from ereuse_devicehub.resources.models import Thing
 from ereuse_devicehub.resources.user.models import User
@@ -31,6 +30,7 @@ class Lot(Thing):
     devices = db.relationship(Device,
                               backref=db.backref('lots', lazy=True, collection_class=set),
                               secondary=lambda: LotDevice.__table__,
+                              lazy=True,
                               collection_class=set)
     """
     The **children** devices that the lot has.
@@ -38,6 +38,32 @@ class Lot(Thing):
     Note that the lot can have more devices, if they are inside 
     descendant lots.
     """
+    parents = db.relationship(lambda: Lot,
+                              viewonly=True,
+                              lazy=True,
+                              collection_class=set,
+                              secondary=lambda: LotParent.__table__,
+                              primaryjoin=lambda: Lot.id == LotParent.child_id,
+                              secondaryjoin=lambda: LotParent.parent_id == Lot.id,
+                              cascade='refresh-expire',  # propagate changes outside ORM
+                              backref=db.backref('children',
+                                                 viewonly=True,
+                                                 lazy=True,
+                                                 cascade='refresh-expire',
+                                                 collection_class=set)
+                              )
+    """The parent lots."""
+
+    all_devices = db.relationship(Device,
+                                  viewonly=True,
+                                  lazy=True,
+                                  collection_class=set,
+                                  secondary=lambda: LotDeviceDescendants.__table__,
+                                  primaryjoin=lambda: Lot.id == LotDeviceDescendants.ancestor_lot_id,
+                                  secondaryjoin=lambda: LotDeviceDescendants.device_id == Device.id)
+    """All devices, including components, inside this lot and its
+    descendants.
+    """
 
     def __init__(self, name: str, closed: bool = closed.default.arg,
                  description: str = None) -> None:
@@ -49,38 +75,11 @@ class Lot(Thing):
         super().__init__(id=uuid.uuid4(), name=name, closed=closed, description=description)
         Path(self)  # Lots have always one edge per default.
 
-    def add_child(self, child):
-        """Adds a child to this lot."""
-        if isinstance(child, Lot):
-            Path.add(self.id, child.id)
-            db.session.refresh(self)  # todo is this useful?
-            db.session.refresh(child)
-        else:
-            assert isinstance(child, uuid.UUID)
-            Path.add(self.id, child)
-            db.session.refresh(self)  # todo is this useful?
-
-    def remove_child(self, child):
-        if isinstance(child, Lot):
-            Path.delete(self.id, child.id)
-        else:
-            assert isinstance(child, uuid.UUID)
-            Path.delete(self.id, child)
-
     @property
     def url(self) -> urlutils.URL:
         """The URL where to GET this event."""
         return urlutils.URL(url_for_resource(Lot, item_id=self.id))
 
-    @property
-    def children(self):
-        """The children lots."""
-        # From https://stackoverflow.com/a/41158890
-        id = UUIDLtree.convert(self.id)
-        return self.query \
-            .join(self.__class__.paths) \
-            .filter(Path.path.lquery(exp.cast('*.{}.*{{1}}'.format(id), LQUERY)))
-
     @property
     def descendants(self):
         return self.descendantsq(self.id)
@@ -90,26 +89,45 @@ class Lot(Thing):
         _id = UUIDLtree.convert(id)
         return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY))
 
-    @property
-    def parents(self):
-        return self.parentsq(self.id)
-
-    @classmethod
-    def parentsq(cls, id: UUID):
-        """The parent lots."""
-        id = UUIDLtree.convert(id)
-        i = db.func.index(Path.path, id)
-        parent_id = db.func.replace(exp.cast(db.func.subpath(Path.path, i - 1, i), TEXT), '_', '-')
-        join_clause = parent_id == exp.cast(Lot.id, TEXT)
-        return cls.query.join(Path, join_clause).filter(
-            Path.path.lquery(exp.cast('*{{1}}.{}.*'.format(id), LQUERY))
-        )
-
     @classmethod
     def roots(cls):
         """Gets the lots that are not under any other lot."""
         return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1)
 
+    def add_children(self, *children):
+        """Add children lots to this lot.
+
+        This operation is highly costly as it forces refreshing
+        many models in session.
+        """
+        for child in children:
+            if isinstance(child, Lot):
+                Path.add(self.id, child.id)
+                db.session.refresh(child)
+            else:
+                assert isinstance(child, uuid.UUID)
+                Path.add(self.id, child)
+        # We need to refresh the models involved in this operation
+        # outside the session / ORM control so the models
+        # that have relationships to this model
+        # with the cascade 'refresh-expire' can welcome the changes
+        db.session.refresh(self)
+
+    def remove_children(self, *children):
+        """Remove children lots from this lot.
+
+        This operation is highly costly as it forces refreshing
+        many models in session.
+        """
+        for child in children:
+            if isinstance(child, Lot):
+                Path.delete(self.id, child.id)
+                db.session.refresh(child)
+            else:
+                assert isinstance(child, uuid.UUID)
+                Path.delete(self.id, child)
+        db.session.refresh(self)
+
     def delete(self):
         """Deletes the lot.
 
@@ -117,10 +135,15 @@ class Lot(Thing):
         devices orphan from this lot and then marks this lot
         for deletion.
         """
-        for child in self.children:
+        self.remove_children(*self.children)
-            self.remove_child(child)
         db.session.delete(self)
 
+    def _refresh_models_with_relationships_to_lots(self):
+        session = db.Session.object_session(self)
+        for model in session:
+            if isinstance(model, (Device, Lot, Path)):
+                session.expire(model)
+
     def __contains__(self, child: Union['Lot', Device]):
         if isinstance(child, Lot):
             return Path.has_lot(self.id, child.id)
@@ -225,8 +248,8 @@ class LotDeviceDescendants(db.Model):
     """Query that gets the descendants of the ancestor lot."""
     devices = db.select([
         LotDevice.device_id,
-        _ancestor.c.id.label('ancestor_lot_id'),
         _desc.c.id.label('parent_lot_id'),
+        _ancestor.c.id.label('ancestor_lot_id'),
         None
     ]).select_from(_ancestor).select_from(lot_device).where(descendants)
 
@@ -240,12 +263,22 @@ class LotDeviceDescendants(db.Model):
 
     components = db.select([
         Component.id.label('device_id'),
-        _ancestor.c.id.label('ancestor_lot_id'),
         _desc.c.id.label('parent_lot_id'),
+        _ancestor.c.id.label('ancestor_lot_id'),
         LotDevice.device_id.label('device_parent_id'),
     ]).select_from(_ancestor).select_from(lot_device_component).where(descendants)
 
+    __table__ = create_view('lot_device_descendants', devices.union(components))
+
+
+class LotParent(db.Model):
+    i = f.index(Path.path, db.func.text2ltree(f.replace(exp.cast(Path.lot_id, TEXT), '-', '_')))
+
     __table__ = create_view(
-        name='lot_device_descendants',
+        'lot_parent',
-        selectable=devices.union(components)
+        db.select([
+            Path.lot_id.label('child_id'),
+            exp.cast(f.replace(exp.cast(f.subltree(Path.path, i - 1, i), TEXT), '_', '-'),
+                     UUID).label('parent_id')
+        ]).select_from(Path).where(i > 0),
     )

ereuse_devicehub/resources/lot/models.pyi

@@ -22,6 +22,8 @@ class Lot(Thing):
     devices = ...  # type: relationship
     paths = ...  # type: relationship
     description = ...  # type: Column
+    all_devices = ...  # type: relationship
+    parents = ...  # type: relationship
 
     def __init__(self, name: str, closed: bool = closed.default.arg) -> None:
         super().__init__()
@@ -30,22 +32,21 @@ class Lot(Thing):
         self.closed = ...  # type: bool
         self.devices = ...  # type: Set[Device]
         self.paths = ...  # type: Set[Path]
-        description = ...  # type: str
+        self.description = ...  # type: str
+        self.all_devices = ...  # type: Set[Device]
+        self.parents = ...  # type: Set[Lot]
+        self.children = ...  # type: Set[Lot]
 
-    def add_child(self, child: Union[Lot, uuid.UUID]):
+    def add_children(self, *children: Union[Lot, uuid.UUID]):
         pass
 
-    def remove_child(self, child: Union[Lot, uuid.UUID]):
+    def remove_children(self, *children: Union[Lot, uuid.UUID]):
         pass
 
     @classmethod
     def roots(cls) -> LotQuery:
         pass
 
-    @property
-    def children(self) -> LotQuery:
-        pass
-
     @property
     def descendants(self) -> LotQuery:
         pass
@@ -54,14 +55,6 @@ class Lot(Thing):
     def descendantsq(cls, id) -> LotQuery:
         pass
 
-    @property
-    def parents(self) -> LotQuery:
-        pass
-
-    @classmethod
-    def parentsq(cls, id) -> LotQuery:
-        pass
-
     @property
     def url(self) -> urlutils.URL:
         pass

ereuse_devicehub/resources/lot/views.py

@@ -1,7 +1,7 @@
 import uuid
 from collections import deque
 from enum import Enum
-from typing import List, Set
+from typing import Dict, List, Set, Union
 
 import marshmallow as ma
 from flask import Response, jsonify, request
@@ -67,10 +67,12 @@ class LotView(View):
         you can filter.
         """
         if args['format'] == LotFormat.UiTree:
-            return jsonify({
+            lots = self.schema.dump(Lot.query, many=True, nested=1)
-                'items': self.ui_tree(),
+            ret = {
+                'items': {l['id']: l for l in lots},
+                'tree': self.ui_tree(),
                 'url': request.path
-            })
+            }
         else:
             query = Lot.query
             if args['search']:
@@ -87,15 +89,7 @@ class LotView(View):
                 },
                 'url': request.path
             }
-            return jsonify(ret)
+        return jsonify(ret)
-
-    @classmethod
-    def ui_tree(cls) -> List[dict]:
-        nodes = []
-        for model in Path.query:  # type: Path
-            path = deque(model.path.path.split('.'))
-            cls._p(nodes, path)
-        return nodes
 
     def delete(self, id):
         lot = Lot.query.filter_by(id=id).one()
@@ -104,7 +98,15 @@ class LotView(View):
         return Response(status=204)
 
     @classmethod
-    def _p(cls, nodes: List[dict], path: deque):
+    def ui_tree(cls) -> List[Dict]:
+        tree = []
+        for model in Path.query:  # type: Path
+            path = deque(model.path.path.split('.'))
+            cls._p(tree, path)
+        return tree
+
+    @classmethod
+    def _p(cls, nodes: List[Dict[str, Union[uuid.UUID, List]]], path: deque):
         """Recursively creates the nested lot structure.
 
         Every recursive step consumes path (a deque of lot_id),
@@ -116,14 +118,8 @@ class LotView(View):
             # does lot_id exist already in node?
             node = next(part for part in nodes if lot_id == part['id'])
         except StopIteration:
-            lot = Lot.query.filter_by(id=lot_id).one()
             node = {
                 'id': lot_id,
-                'name': lot.name,
-                'url': lot.url.to_text(),
-                'closed': lot.closed,
-                'updated': lot.updated,
-                'created': lot.created,
                 'nodes': []
             }
             nodes.append(node)
@@ -180,12 +176,10 @@ class LotChildrenView(LotBaseChildrenView):
         id = ma.fields.List(ma.fields.UUID())
 
     def _post(self, lot: Lot, ids: Set[uuid.UUID]):
-        for id in ids:
+        lot.add_children(*ids)
-            lot.add_child(id)  # todo what to do if child exists already?
 
     def _delete(self, lot: Lot, ids: Set[uuid.UUID]):
-        for id in ids:
+        lot.remove_children(*ids)
-            lot.remove_child(id)
 
 
 class LotDeviceView(LotBaseChildrenView):

tests/test_lot.py

@@ -37,23 +37,33 @@ def test_lot_model_children():
     l1, l2, l3 = lots
     db.session.add_all(lots)
     db.session.flush()
+    assert not l1.children
+    assert not l1.parents
+    assert not l2.children
+    assert not l2.parents
+    assert not l3.parents
+    assert not l3.children
 
-    l1.add_child(l2)
+    l1.add_children(l2)
-    db.session.flush()
+    assert l1.children == {l2}
+    assert l2.parents == {l1}
 
-    assert list(l1.children) == [l2]
+    l2.add_children(l3)
-
+    assert l1.children == {l2}
-    l2.add_child(l3)
+    assert l2.parents == {l1}
-    assert list(l1.children) == [l2]
+    assert l2.children == {l3}
+    assert l3.parents == {l2}
 
     l2.delete()
     db.session.flush()
-    assert not list(l1.children)
+    assert not l1.children
+    assert not l3.parents
 
     l1.delete()
     db.session.flush()
     l3b = Lot.query.one()
     assert l3 == l3b
+    assert not l3.parents
 
 
 def test_lot_modify_patch_endpoint_and_delete(user: UserClient):
@@ -87,8 +97,8 @@ def test_lot_device_relationship():
     assert lot_device.created
     assert lot_device.author_id == g.user.id
     assert device.lots == {child}
-    # todo Device IN LOT does not work
     assert device in child
+    assert device in child.all_devices
 
     graphic = GraphicCard(serial_number='foo', model='bar')
     device.components.add(graphic)
@@ -98,7 +108,7 @@ def test_lot_device_relationship():
     parent = Lot('parent')
     db.session.add(parent)
     db.session.flush()
-    parent.add_child(child)
+    parent.add_children(child)
     assert child in parent
 
 
@@ -111,13 +121,13 @@ def test_add_edge():
     db.session.add(parent)
     db.session.flush()
 
-    parent.add_child(child)
+    parent.add_children(child)
 
     assert child in parent
     assert len(child.paths) == 1
     assert len(parent.paths) == 1
 
-    parent.remove_child(child)
+    parent.remove_children(child)
     assert child not in parent
     assert len(child.paths) == 1
     assert len(parent.paths) == 1
@@ -126,8 +136,8 @@ def test_add_edge():
     db.session.add(grandparent)
     db.session.flush()
 
-    grandparent.add_child(parent)
+    grandparent.add_children(parent)
-    parent.add_child(child)
+    parent.add_children(child)
 
     assert parent in grandparent
     assert child in parent
@@ -148,31 +158,36 @@ def test_lot_multiple_parents(auth_app_context):
     db.session.add_all(lots)
     db.session.flush()
 
-    grandparent1.add_child(parent)
+    grandparent1.add_children(parent)
     assert parent in grandparent1
-    parent.add_child(child)
+    parent.add_children(child)
     assert child in parent
     assert child in grandparent1
-    grandparent2.add_child(parent)
+    grandparent2.add_children(parent)
     assert parent in grandparent1
     assert parent in grandparent2
     assert child in parent
     assert child in grandparent1
     assert child in grandparent2
 
+    p = parent.id
+    c = child.id
+    gp1 = grandparent1.id
+    gp2 = grandparent2.id
+
     nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree()
-    assert nodes[0]['name'] == 'grandparent1'
+    assert nodes[0]['id'] == gp1
-    assert nodes[0]['nodes'][0]['name'] == 'parent'
+    assert nodes[0]['nodes'][0]['id'] == p
-    assert nodes[0]['nodes'][0]['nodes'][0]['name'] == 'child'
+    assert nodes[0]['nodes'][0]['nodes'][0]['id'] == c
     assert nodes[0]['nodes'][0]['nodes'][0]['nodes'] == []
-    assert nodes[1]['name'] == 'grandparent2'
+    assert nodes[1]['id'] == gp2
-    assert nodes[1]['nodes'][0]['name'] == 'parent'
+    assert nodes[1]['nodes'][0]['id'] == p
-    assert nodes[1]['nodes'][0]['nodes'][0]['name'] == 'child'
+    assert nodes[1]['nodes'][0]['nodes'][0]['id'] == c
     assert nodes[1]['nodes'][0]['nodes'][0]['nodes'] == []
 
     # Now remove all childs
 
-    grandparent1.remove_child(parent)
+    grandparent1.remove_children(parent)
     assert parent not in grandparent1
     assert child in parent
     assert parent in grandparent2
@@ -180,14 +195,14 @@ def test_lot_multiple_parents(auth_app_context):
     assert child in grandparent2
 
     nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree()
-    assert nodes[0]['name'] == 'grandparent1'
+    assert nodes[0]['id'] == gp1
     assert nodes[0]['nodes'] == []
-    assert nodes[1]['name'] == 'grandparent2'
+    assert nodes[1]['id'] == gp2
-    assert nodes[1]['nodes'][0]['name'] == 'parent'
+    assert nodes[1]['nodes'][0]['id'] == p
-    assert nodes[1]['nodes'][0]['nodes'][0]['name'] == 'child'
+    assert nodes[1]['nodes'][0]['nodes'][0]['id'] == c
     assert nodes[1]['nodes'][0]['nodes'][0]['nodes'] == []
 
-    grandparent2.remove_child(parent)
+    grandparent2.remove_children(parent)
     assert parent not in grandparent2
     assert parent not in grandparent1
     assert child not in grandparent2
@@ -195,27 +210,27 @@ def test_lot_multiple_parents(auth_app_context):
     assert child in parent
 
     nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree()
-    assert nodes[0]['name'] == 'grandparent1'
+    assert nodes[0]['id'] == gp1
     assert nodes[0]['nodes'] == []
-    assert nodes[1]['name'] == 'grandparent2'
+    assert nodes[1]['id'] == gp2
     assert nodes[1]['nodes'] == []
-    assert nodes[2]['name'] == 'parent'
+    assert nodes[2]['id'] == p
-    assert nodes[2]['nodes'][0]['name'] == 'child'
+    assert nodes[2]['nodes'][0]['id'] == c
     assert nodes[2]['nodes'][0]['nodes'] == []
 
-    parent.remove_child(child)
+    parent.remove_children(child)
     assert child not in parent
     assert len(child.paths) == 1
     assert len(parent.paths) == 1
 
     nodes = auth_app_context.resources[Lot.t].VIEW.ui_tree()
-    assert nodes[0]['name'] == 'grandparent1'
+    assert nodes[0]['id'] == gp1
     assert nodes[0]['nodes'] == []
-    assert nodes[1]['name'] == 'grandparent2'
+    assert nodes[1]['id'] == gp2
     assert nodes[1]['nodes'] == []
-    assert nodes[2]['name'] == 'parent'
+    assert nodes[2]['id'] == p
     assert nodes[2]['nodes'] == []
-    assert nodes[3]['name'] == 'child'
+    assert nodes[3]['id'] == c
     assert nodes[3]['nodes'] == []
 
 
@@ -243,29 +258,29 @@ def test_lot_unite_graphs_and_find():
     db.session.add_all(lots)
     db.session.flush()
 
-    l1.add_child(l2)
+    l1.add_children(l2)
     assert l2 in l1
-    l3.add_child(l2)
+    l3.add_children(l2)
     assert l2 in l3
-    l5.add_child(l7)
+    l5.add_children(l7)
     assert l7 in l5
-    l4.add_child(l5)
+    l4.add_children(l5)
     assert l5 in l4
     assert l7 in l4
-    l5.add_child(l8)
+    l5.add_children(l8)
     assert l8 in l5
-    l4.add_child(l6)
+    l4.add_children(l6)
     assert l6 in l4
-    l6.add_child(l5)
+    l6.add_children(l5)
     assert l5 in l6 and l5 in l4
 
     # We unite the two graphs
-    l2.add_child(l4)
+    l2.add_children(l4)
     assert l4 in l2 and l5 in l2 and l6 in l2 and l7 in l2 and l8 in l2
     assert l4 in l3 and l5 in l3 and l6 in l3 and l7 in l3 and l8 in l3
 
     # We remove the union
-    l2.remove_child(l4)
+    l2.remove_children(l4)
     assert l4 not in l2 and l5 not in l2 and l6 not in l2 and l7 not in l2 and l8 not in l2
     assert l4 not in l3 and l5 not in l3 and l6 not in l3 and l7 not in l3 and l8 not in l3
 
@@ -279,7 +294,7 @@ def test_lot_roots():
     db.session.flush()
 
     assert set(Lot.roots()) == {l1, l2, l3}
-    l1.add_child(l2)
+    l1.add_children(l2)
     assert set(Lot.roots()) == {l1, l3}
 
 
@@ -306,11 +321,16 @@ def test_lot_post_add_children_view_ui_tree_normal(user: UserClient):
     assert child['parents'][0]['id'] == parent['id']
 
     # Format UiTree
-    lots = user.get(res=Lot, query=[('format', 'UiTree')])[0]['items']
+    r = user.get(res=Lot, query=[('format', 'UiTree')])[0]
-    assert 1 == len(lots)
+    lots, nodes = r['items'], r['tree']
-    assert lots[0]['name'] == 'Parent'
+    assert 1 == len(nodes)
-    assert len(lots[0]['nodes']) == 1
+    assert nodes[0]['id'] == parent['id']
-    assert lots[0]['nodes'][0]['name'] == 'Child'
+    assert len(nodes[0]['nodes']) == 1
+    assert nodes[0]['nodes'][0]['id'] == child['id']
+    assert 2 == len(lots)
+    assert 'Parent' == lots[parent['id']]['name']
+    assert 'Child' == lots[child['id']]['name']
+    assert lots[child['id']]['parents'][0]['name'] == 'Parent'
 
     # Normal list format
     lots = user.get(res=Lot)[0]['items']