diff --git a/ereuse_devicehub/db.py b/ereuse_devicehub/db.py index b074b614..99f8b73c 100644 --- a/ereuse_devicehub/db.py +++ b/ereuse_devicehub/db.py @@ -1,4 +1,6 @@ +from sqlalchemy import event from sqlalchemy.dialects import postgresql +from sqlalchemy_utils import view from teal.db import SchemaSQLAlchemy @@ -17,3 +19,21 @@ class SQLAlchemy(SchemaSQLAlchemy): db = SQLAlchemy(session_options={"autoflush": False}) + + +def create_view(name, selectable): + """Creates a view. + + This is an adaptation from sqlalchemy_utils.view. See + `the test on sqlalchemy-utils `_ for an + example on how to use. + """ + table = view.create_table_from_selectable(name=name, selectable=selectable, metadata=None) + + # We need to ensure views are created / destroyed before / after + # SchemaSQLAlchemy's listeners execute + # That is why insert=True in 'after_create' + event.listen(db.metadata, 'after_create', view.CreateView(name, selectable), insert=True) + event.listen(db.metadata, 'before_drop', view.DropView(name)) + return table diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py index 437882a6..23cfcf9e 100644 --- a/ereuse_devicehub/resources/device/models.py +++ b/ereuse_devicehub/resources/device/models.py @@ -16,8 +16,8 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship, validates from sqlalchemy.util import OrderedSet from sqlalchemy_utils import ColorType from stdnum import imei, meid -from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, check_lower, \ - check_range +from teal.db import CASCADE_DEL, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, \ + check_lower, check_range from teal.enums import Layouts from teal.marshmallow import ValidationError from teal.resource import url_for_resource @@ -428,7 +428,7 @@ class Component(Device): parent = relationship(Computer, backref=backref('components', lazy=True, - cascade=CASCADE, + cascade=CASCADE_DEL, order_by=lambda: Component.id, collection_class=OrderedSet), primaryjoin=parent_id == Computer.id) diff --git a/ereuse_devicehub/resources/device/views.py b/ereuse_devicehub/resources/device/views.py index d8e5e82c..7706a1c2 100644 --- a/ereuse_devicehub/resources/device/views.py +++ b/ereuse_devicehub/resources/device/views.py @@ -12,10 +12,10 @@ from teal.resource import View from ereuse_devicehub import auth from ereuse_devicehub.db import db from ereuse_devicehub.resources import search -from ereuse_devicehub.resources.device.models import Component, Computer, Device, Manufacturer +from ereuse_devicehub.resources.device.models import Device, Manufacturer from ereuse_devicehub.resources.device.search import DeviceSearch from ereuse_devicehub.resources.event.models import Rate -from ereuse_devicehub.resources.lot.models import Lot, LotDevice +from ereuse_devicehub.resources.lot.models import LotDeviceDescendants from ereuse_devicehub.resources.tag.model import Tag @@ -41,15 +41,10 @@ class TagQ(query.Query): class LotQ(query.Query): - id = query.Or(query.QueryField(Lot.descendantsq, fields.UUID())) + id = query.Or(query.Equal(LotDeviceDescendants.ancestor_lot_id, fields.UUID())) class Filters(query.Query): - _parent = Computer.__table__.alias() - _device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id) - _parent_device_in_lot = (Device.id == Component.id) & (Component.parent_id == _parent.c.id) \ - & (_parent.c.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id) - type = query.Or(OfType(Device.type)) model = query.ILike(Device.model) manufacturer = query.ILike(Device.manufacturer) @@ -59,7 +54,7 @@ class Filters(query.Query): # todo This part of the query is really slow # And forces usage of distinct, as it returns many rows # due to having multiple paths to the same - lot = query.Join(_device_inside_lot | _parent_device_in_lot, LotQ) + lot = query.Join(Device.id == LotDeviceDescendants.device_id, LotQ) class Sorting(query.Sort): diff --git a/ereuse_devicehub/resources/event/models.py b/ereuse_devicehub/resources/event/models.py index 7c398a3b..da5d65b0 100644 --- a/ereuse_devicehub/resources/event/models.py +++ b/ereuse_devicehub/resources/event/models.py @@ -18,7 +18,7 @@ from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy.orm import backref, relationship, validates from sqlalchemy.orm.events import AttributeEvents as Events from sqlalchemy.util import OrderedSet -from teal.db import ArrayOfEnum, CASCADE, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \ +from teal.db import ArrayOfEnum, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \ POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range from teal.enums import Country, Currency, Subdivision from teal.marshmallow import ValidationError @@ -219,7 +219,7 @@ class EventWithOneDevice(JoinedTableMixin, Event): device = relationship(Device, backref=backref('events_one', lazy=True, - cascade=CASCADE, + cascade=CASCADE_OWN, order_by=lambda: EventWithOneDevice.created, collection_class=OrderedSet), primaryjoin=Device.id == device_id) diff --git a/ereuse_devicehub/resources/lot/models.py b/ereuse_devicehub/resources/lot/models.py index fd8a8373..6e850cdc 100644 --- a/ereuse_devicehub/resources/lot/models.py +++ b/ereuse_devicehub/resources/lot/models.py @@ -1,5 +1,6 @@ import uuid from datetime import datetime +from typing import Union from boltons import urlutils from citext import CIText @@ -9,11 +10,11 @@ from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.sql import expression as exp from sqlalchemy_utils import LtreeType from sqlalchemy_utils.types.ltree import LQUERY -from teal.db import UUIDLtree +from teal.db import CASCADE_OWN, UUIDLtree from teal.resource import url_for_resource -from ereuse_devicehub.db import db -from ereuse_devicehub.resources.device.models import Device +from ereuse_devicehub.db import create_view, db +from ereuse_devicehub.resources.device.models import Component, Computer, Device from ereuse_devicehub.resources.models import Thing from ereuse_devicehub.resources.user.models import User @@ -89,6 +90,16 @@ class Lot(Thing): _id = UUIDLtree.convert(id) return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY)) + @classmethod + def device_in_lotq(cls): + parent = Computer.__table__.alias() + device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id) + parent_device_in_lot = (Device.id == Component.id) \ + & (Component.parent_id == parent.c.id) \ + & (parent.c.id == LotDevice.device_id) \ + & (Lot.id == LotDevice.lot_id) + return device_inside_lot | parent_device_in_lot + @property def parents(self): return self.parentsq(self.id) @@ -109,8 +120,28 @@ class Lot(Thing): """Gets the lots that are not under any other lot.""" return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1) - def __contains__(self, child: 'Lot'): - return Path.has_lot(self.id, child.id) + def delete(self): + """Deletes the lot. + + This method removes the children lots and children + devices orphan from this lot and then marks this lot + for deletion. + """ + for child in self.children: + self.remove_child(child) + db.session.delete(self) + + def __contains__(self, child: Union['Lot', Device]): + if isinstance(child, Lot): + return Path.has_lot(self.id, child.id) + elif isinstance(child, Device): + device = db.session.query(LotDeviceDescendants) \ + .filter(LotDeviceDescendants.device_id == child.id) \ + .filter(LotDeviceDescendants.ancestor_lot_id == self.id) \ + .one_or_none() + return device + else: + raise TypeError('Lot only contains devices and lots, not {}'.format(child.__class__)) def __repr__(self) -> str: return ''.format(self) @@ -136,7 +167,10 @@ class Path(db.Model): server_default=db.text('gen_random_uuid()')) lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False, index=True) lot = db.relationship(Lot, - backref=db.backref('paths', lazy=True, collection_class=set), + backref=db.backref('paths', + lazy=True, + collection_class=set, + cascade=CASCADE_OWN), primaryjoin=Lot.id == lot_id) path = db.Column(LtreeType, nullable=False) created = db.Column(db.TIMESTAMP(timezone=True), server_default=db.text('CURRENT_TIMESTAMP')) @@ -174,3 +208,54 @@ class Path(db.Model): "SELECT 1 from path where path ~ '*.{}.*.{}.*'".format(parent_id, child_id) ).first() ) + + +class LotDeviceDescendants(db.Model): + """A view facilitating querying inclusion between devices and lots, + including components. + + The view has 4 columns: + 1. The ID of the device. + 2. The ID of a lot containing the device. + 3. The ID of the lot that directly contains the device. + 4. If 1. is a component, the ID of the device that is inside the lot. + """ + + _ancestor = Lot.__table__.alias(name='ancestor') + """Ancestor lot table.""" + _desc = Lot.__table__.alias() + """Descendant lot table.""" + lot_device = _desc \ + .join(LotDevice, _desc.c.id == LotDevice.lot_id) \ + .join(Path, _desc.c.id == Path.lot_id) + """Join: Path -- Lot -- LotDevice""" + + descendants = "path.path ~ (CAST('*.'|| replace(CAST({}.id as text), '-', '_') " \ + "|| '.*' AS LQUERY))".format(_ancestor.name) + """Query that gets the descendants of the ancestor lot.""" + devices = db.select([ + LotDevice.device_id, + _ancestor.c.id.label('ancestor_lot_id'), + _desc.c.id.label('parent_lot_id'), + None + ]).select_from(_ancestor).select_from(lot_device).where(descendants) + + # Components + _parent_device = Device.__table__.alias(name='parent_device') + """The device that has the access to the lot.""" + lot_device_component = lot_device \ + .join(_parent_device, _parent_device.c.id == LotDevice.device_id) \ + .join(Component, _parent_device.c.id == Component.parent_id) + """Join: Path -- Lot -- LotDevice -- ParentDevice (Device) -- Component""" + + components = db.select([ + Component.id.label('device_id'), + _ancestor.c.id.label('ancestor_lot_id'), + _desc.c.id.label('parent_lot_id'), + LotDevice.device_id.label('device_parent_id'), + ]).select_from(_ancestor).select_from(lot_device_component).where(descendants) + + __table__ = create_view( + name='lot_device_descendants', + selectable=devices.union(components) + ) diff --git a/ereuse_devicehub/resources/lot/models.pyi b/ereuse_devicehub/resources/lot/models.pyi index 9ae74b8a..abf518df 100644 --- a/ereuse_devicehub/resources/lot/models.pyi +++ b/ereuse_devicehub/resources/lot/models.pyi @@ -1,6 +1,6 @@ import uuid from datetime import datetime -from typing import Iterable, Set, Union +from typing import Iterable, Optional, Set, Union from uuid import UUID from boltons import urlutils @@ -8,6 +8,7 @@ from sqlalchemy import Column from sqlalchemy.orm import Query, relationship from sqlalchemy_utils import Ltree +from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.models import Device from ereuse_devicehub.resources.models import Thing @@ -65,6 +66,9 @@ class Lot(Thing): def url(self) -> urlutils.URL: pass + def delete(self): + pass + class Path: id = ... # type: Column @@ -79,3 +83,17 @@ class Path: self.lot = ... # type: Lot self.path = ... # type: Ltree self.created = ... # type: datetime + + +class LotDeviceDescendants(db.Model): + device_id = ... # type: Column + ancestor_lot_id = ... # type: Column + parent_lot_id = ... # type: Column + device_parent_id = ... # type: Column + + def __init__(self) -> None: + super().__init__() + self.device_id = ... # type: int + self.ancestor_lot_id = ... # type: UUID + self.parent_lot_id = ... # type: UUID + self.device_parent_id = ... # type: Optional[int] diff --git a/ereuse_devicehub/resources/lot/views.py b/ereuse_devicehub/resources/lot/views.py index 5532a2e5..5f5c447b 100644 --- a/ereuse_devicehub/resources/lot/views.py +++ b/ereuse_devicehub/resources/lot/views.py @@ -97,6 +97,12 @@ class LotView(View): cls._p(nodes, path) return nodes + def delete(self, id): + lot = Lot.query.filter_by(id=id).one() + lot.delete() + db.session.commit() + return Response(status=204) + @classmethod def _p(cls, nodes: List[dict], path: deque): """Recursively creates the nested lot structure. diff --git a/ereuse_devicehub/resources/models.pyi b/ereuse_devicehub/resources/models.pyi index deb89a77..ee15b660 100644 --- a/ereuse_devicehub/resources/models.pyi +++ b/ereuse_devicehub/resources/models.pyi @@ -1,6 +1,6 @@ from datetime import datetime -from sqlalchemy import Column +from sqlalchemy import Column, Table from teal.db import Model STR_SIZE = 64 @@ -10,6 +10,7 @@ STR_XSM_SIZE = 16 class Thing(Model): + __table__ = ... # type: Table t = ... # type: str type = ... # type: str updated = ... # type: Column diff --git a/requirements.txt b/requirements.txt index e4a7271a..c3fcc22a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,9 +23,9 @@ python-stdnum==1.9 PyYAML==3.13 requests==2.19.1 requests-mock==1.5.2 -SQLAlchemy==1.2.11 -SQLAlchemy-Utils==0.33.3 -teal==0.2.0a29 +SQLAlchemy==1.2.14 +SQLAlchemy-Utils==0.33.6 +teal==0.2.0a30 webargs==4.0.0 Werkzeug==0.14.1 sqlalchemy-citext==1.3.post0 diff --git a/setup.py b/setup.py index 46202707..a1cdbfcc 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( long_description=long_description, long_description_content_type='text/markdown', install_requires=[ - 'teal>=0.2.0a29', # teal always first + 'teal>=0.2.0a30', # teal always first 'click', 'click-spinner', 'ereuse-utils[Naming]>=0.4b10', diff --git a/tests/test_lot.py b/tests/test_lot.py index 051db214..1891554d 100644 --- a/tests/test_lot.py +++ b/tests/test_lot.py @@ -23,7 +23,40 @@ In case of error, debug with: """ -def test_lot_modify_patch_endpoint(user: UserClient): +@pytest.mark.usefixtures(conftest.auth_app_context.__name__) +def test_lot_model_children(): + """Tests the property Lot.children + + l1 + | + l2 + | + l3 + """ + lots = Lot('1'), Lot('2'), Lot('3') + l1, l2, l3 = lots + db.session.add_all(lots) + db.session.flush() + + l1.add_child(l2) + db.session.flush() + + assert list(l1.children) == [l2] + + l2.add_child(l3) + assert list(l1.children) == [l2] + + l2.delete() + db.session.flush() + assert not list(l1.children) + + l1.delete() + db.session.flush() + l3b = Lot.query.one() + assert l3 == l3b + + +def test_lot_modify_patch_endpoint_and_delete(user: UserClient): """Creates and modifies lot properties through the endpoint""" l, _ = user.post({'name': 'foo', 'description': 'baz'}, res=Lot) assert l['name'] == 'foo' @@ -32,20 +65,17 @@ def test_lot_modify_patch_endpoint(user: UserClient): l_after, _ = user.get(res=Lot, item=l['id']) assert l_after['name'] == 'bar' assert l_after['description'] == 'bax' + user.delete(res=Lot, item=l['id'], status=204) + user.get(res=Lot, item=l['id'], status=404) -@pytest.mark.xfail(reason='No DEL endpoint') -def test_lot_delete_endpoint(user: UserClient): - pass - - -@pytest.mark.xfail(reason='the IN comparison does not work for device') @pytest.mark.usefixtures(conftest.auth_app_context.__name__) def test_lot_device_relationship(): device = Desktop(serial_number='foo', model='bar', manufacturer='foobar', chassis=ComputerChassis.Lunchbox) + device.components.add(GraphicCard(serial_number='foo', model='bar1', manufacturer='baz')) child = Lot('child') child.devices.add(device) db.session.add(child) @@ -253,21 +283,6 @@ def test_lot_roots(): assert set(Lot.roots()) == {l1, l3} -@pytest.mark.usefixtures(conftest.auth_app_context.__name__) -def test_lot_model_children(): - """Tests the property Lot.children""" - lots = Lot('1'), Lot('2'), Lot('3') - l1, l2, l3 = lots - db.session.add_all(lots) - db.session.flush() - - l1.add_child(l2) - db.session.flush() - - children = l1.children - assert list(children) == [l2] - - def test_post_get_lot(user: UserClient): """Tests submitting and retreiving a basic lot.""" l, _ = user.post({'name': 'Foo'}, res=Lot)