Delete lots; add LotDeviceDescendants view really fixing querying devices in lots

This commit is contained in:
Xavier Bustamante Talavera 2018-11-11 21:52:55 +01:00
parent bf2c61ad65
commit bcf59de383
11 changed files with 188 additions and 48 deletions

View File

@ -1,4 +1,6 @@
from sqlalchemy import event
from sqlalchemy.dialects import postgresql from sqlalchemy.dialects import postgresql
from sqlalchemy_utils import view
from teal.db import SchemaSQLAlchemy from teal.db import SchemaSQLAlchemy
@ -17,3 +19,21 @@ class SQLAlchemy(SchemaSQLAlchemy):
db = SQLAlchemy(session_options={"autoflush": False}) db = SQLAlchemy(session_options={"autoflush": False})
def create_view(name, selectable):
"""Creates a view.
This is an adaptation from sqlalchemy_utils.view. See
`the test on sqlalchemy-utils <https://github.com/kvesteri/
sqlalchemy-utils/blob/master/tests/test_views.py>`_ for an
example on how to use.
"""
table = view.create_table_from_selectable(name=name, selectable=selectable, metadata=None)
# We need to ensure views are created / destroyed before / after
# SchemaSQLAlchemy's listeners execute
# That is why insert=True in 'after_create'
event.listen(db.metadata, 'after_create', view.CreateView(name, selectable), insert=True)
event.listen(db.metadata, 'before_drop', view.DropView(name))
return table

View File

@ -16,8 +16,8 @@ from sqlalchemy.orm import ColumnProperty, backref, relationship, validates
from sqlalchemy.util import OrderedSet from sqlalchemy.util import OrderedSet
from sqlalchemy_utils import ColorType from sqlalchemy_utils import ColorType
from stdnum import imei, meid from stdnum import imei, meid
from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, check_lower, \ from teal.db import CASCADE_DEL, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, URL, \
check_range check_lower, check_range
from teal.enums import Layouts from teal.enums import Layouts
from teal.marshmallow import ValidationError from teal.marshmallow import ValidationError
from teal.resource import url_for_resource from teal.resource import url_for_resource
@ -428,7 +428,7 @@ class Component(Device):
parent = relationship(Computer, parent = relationship(Computer,
backref=backref('components', backref=backref('components',
lazy=True, lazy=True,
cascade=CASCADE, cascade=CASCADE_DEL,
order_by=lambda: Component.id, order_by=lambda: Component.id,
collection_class=OrderedSet), collection_class=OrderedSet),
primaryjoin=parent_id == Computer.id) primaryjoin=parent_id == Computer.id)

View File

@ -12,10 +12,10 @@ from teal.resource import View
from ereuse_devicehub import auth from ereuse_devicehub import auth
from ereuse_devicehub.db import db from ereuse_devicehub.db import db
from ereuse_devicehub.resources import search from ereuse_devicehub.resources import search
from ereuse_devicehub.resources.device.models import Component, Computer, Device, Manufacturer from ereuse_devicehub.resources.device.models import Device, Manufacturer
from ereuse_devicehub.resources.device.search import DeviceSearch from ereuse_devicehub.resources.device.search import DeviceSearch
from ereuse_devicehub.resources.event.models import Rate from ereuse_devicehub.resources.event.models import Rate
from ereuse_devicehub.resources.lot.models import Lot, LotDevice from ereuse_devicehub.resources.lot.models import LotDeviceDescendants
from ereuse_devicehub.resources.tag.model import Tag from ereuse_devicehub.resources.tag.model import Tag
@ -41,15 +41,10 @@ class TagQ(query.Query):
class LotQ(query.Query): class LotQ(query.Query):
id = query.Or(query.QueryField(Lot.descendantsq, fields.UUID())) id = query.Or(query.Equal(LotDeviceDescendants.ancestor_lot_id, fields.UUID()))
class Filters(query.Query): class Filters(query.Query):
_parent = Computer.__table__.alias()
_device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
_parent_device_in_lot = (Device.id == Component.id) & (Component.parent_id == _parent.c.id) \
& (_parent.c.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
type = query.Or(OfType(Device.type)) type = query.Or(OfType(Device.type))
model = query.ILike(Device.model) model = query.ILike(Device.model)
manufacturer = query.ILike(Device.manufacturer) manufacturer = query.ILike(Device.manufacturer)
@ -59,7 +54,7 @@ class Filters(query.Query):
# todo This part of the query is really slow # todo This part of the query is really slow
# And forces usage of distinct, as it returns many rows # And forces usage of distinct, as it returns many rows
# due to having multiple paths to the same # due to having multiple paths to the same
lot = query.Join(_device_inside_lot | _parent_device_in_lot, LotQ) lot = query.Join(Device.id == LotDeviceDescendants.device_id, LotQ)
class Sorting(query.Sort): class Sorting(query.Sort):

View File

@ -18,7 +18,7 @@ from sqlalchemy.ext.orderinglist import ordering_list
from sqlalchemy.orm import backref, relationship, validates from sqlalchemy.orm import backref, relationship, validates
from sqlalchemy.orm.events import AttributeEvents as Events from sqlalchemy.orm.events import AttributeEvents as Events
from sqlalchemy.util import OrderedSet from sqlalchemy.util import OrderedSet
from teal.db import ArrayOfEnum, CASCADE, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \ from teal.db import ArrayOfEnum, CASCADE_OWN, INHERIT_COND, IP, POLYMORPHIC_ID, \
POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range POLYMORPHIC_ON, StrictVersionType, URL, check_lower, check_range
from teal.enums import Country, Currency, Subdivision from teal.enums import Country, Currency, Subdivision
from teal.marshmallow import ValidationError from teal.marshmallow import ValidationError
@ -219,7 +219,7 @@ class EventWithOneDevice(JoinedTableMixin, Event):
device = relationship(Device, device = relationship(Device,
backref=backref('events_one', backref=backref('events_one',
lazy=True, lazy=True,
cascade=CASCADE, cascade=CASCADE_OWN,
order_by=lambda: EventWithOneDevice.created, order_by=lambda: EventWithOneDevice.created,
collection_class=OrderedSet), collection_class=OrderedSet),
primaryjoin=Device.id == device_id) primaryjoin=Device.id == device_id)

View File

@ -1,5 +1,6 @@
import uuid import uuid
from datetime import datetime from datetime import datetime
from typing import Union
from boltons import urlutils from boltons import urlutils
from citext import CIText from citext import CIText
@ -9,11 +10,11 @@ from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.sql import expression as exp from sqlalchemy.sql import expression as exp
from sqlalchemy_utils import LtreeType from sqlalchemy_utils import LtreeType
from sqlalchemy_utils.types.ltree import LQUERY from sqlalchemy_utils.types.ltree import LQUERY
from teal.db import UUIDLtree from teal.db import CASCADE_OWN, UUIDLtree
from teal.resource import url_for_resource from teal.resource import url_for_resource
from ereuse_devicehub.db import db from ereuse_devicehub.db import create_view, db
from ereuse_devicehub.resources.device.models import Device from ereuse_devicehub.resources.device.models import Component, Computer, Device
from ereuse_devicehub.resources.models import Thing from ereuse_devicehub.resources.models import Thing
from ereuse_devicehub.resources.user.models import User from ereuse_devicehub.resources.user.models import User
@ -89,6 +90,16 @@ class Lot(Thing):
_id = UUIDLtree.convert(id) _id = UUIDLtree.convert(id)
return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY)) return (cls.id == Path.lot_id) & Path.path.lquery(exp.cast('*.{}.*'.format(_id), LQUERY))
@classmethod
def device_in_lotq(cls):
parent = Computer.__table__.alias()
device_inside_lot = (Device.id == LotDevice.device_id) & (Lot.id == LotDevice.lot_id)
parent_device_in_lot = (Device.id == Component.id) \
& (Component.parent_id == parent.c.id) \
& (parent.c.id == LotDevice.device_id) \
& (Lot.id == LotDevice.lot_id)
return device_inside_lot | parent_device_in_lot
@property @property
def parents(self): def parents(self):
return self.parentsq(self.id) return self.parentsq(self.id)
@ -109,8 +120,28 @@ class Lot(Thing):
"""Gets the lots that are not under any other lot.""" """Gets the lots that are not under any other lot."""
return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1) return cls.query.join(cls.paths).filter(db.func.nlevel(Path.path) == 1)
def __contains__(self, child: 'Lot'): def delete(self):
"""Deletes the lot.
This method removes the children lots and children
devices orphan from this lot and then marks this lot
for deletion.
"""
for child in self.children:
self.remove_child(child)
db.session.delete(self)
def __contains__(self, child: Union['Lot', Device]):
if isinstance(child, Lot):
return Path.has_lot(self.id, child.id) return Path.has_lot(self.id, child.id)
elif isinstance(child, Device):
device = db.session.query(LotDeviceDescendants) \
.filter(LotDeviceDescendants.device_id == child.id) \
.filter(LotDeviceDescendants.ancestor_lot_id == self.id) \
.one_or_none()
return device
else:
raise TypeError('Lot only contains devices and lots, not {}'.format(child.__class__))
def __repr__(self) -> str: def __repr__(self) -> str:
return '<Lot {0.name} devices={0.devices!r}>'.format(self) return '<Lot {0.name} devices={0.devices!r}>'.format(self)
@ -136,7 +167,10 @@ class Path(db.Model):
server_default=db.text('gen_random_uuid()')) server_default=db.text('gen_random_uuid()'))
lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False, index=True) lot_id = db.Column(db.UUID(as_uuid=True), db.ForeignKey(Lot.id), nullable=False, index=True)
lot = db.relationship(Lot, lot = db.relationship(Lot,
backref=db.backref('paths', lazy=True, collection_class=set), backref=db.backref('paths',
lazy=True,
collection_class=set,
cascade=CASCADE_OWN),
primaryjoin=Lot.id == lot_id) primaryjoin=Lot.id == lot_id)
path = db.Column(LtreeType, nullable=False) path = db.Column(LtreeType, nullable=False)
created = db.Column(db.TIMESTAMP(timezone=True), server_default=db.text('CURRENT_TIMESTAMP')) created = db.Column(db.TIMESTAMP(timezone=True), server_default=db.text('CURRENT_TIMESTAMP'))
@ -174,3 +208,54 @@ class Path(db.Model):
"SELECT 1 from path where path ~ '*.{}.*.{}.*'".format(parent_id, child_id) "SELECT 1 from path where path ~ '*.{}.*.{}.*'".format(parent_id, child_id)
).first() ).first()
) )
class LotDeviceDescendants(db.Model):
"""A view facilitating querying inclusion between devices and lots,
including components.
The view has 4 columns:
1. The ID of the device.
2. The ID of a lot containing the device.
3. The ID of the lot that directly contains the device.
4. If 1. is a component, the ID of the device that is inside the lot.
"""
_ancestor = Lot.__table__.alias(name='ancestor')
"""Ancestor lot table."""
_desc = Lot.__table__.alias()
"""Descendant lot table."""
lot_device = _desc \
.join(LotDevice, _desc.c.id == LotDevice.lot_id) \
.join(Path, _desc.c.id == Path.lot_id)
"""Join: Path -- Lot -- LotDevice"""
descendants = "path.path ~ (CAST('*.'|| replace(CAST({}.id as text), '-', '_') " \
"|| '.*' AS LQUERY))".format(_ancestor.name)
"""Query that gets the descendants of the ancestor lot."""
devices = db.select([
LotDevice.device_id,
_ancestor.c.id.label('ancestor_lot_id'),
_desc.c.id.label('parent_lot_id'),
None
]).select_from(_ancestor).select_from(lot_device).where(descendants)
# Components
_parent_device = Device.__table__.alias(name='parent_device')
"""The device that has the access to the lot."""
lot_device_component = lot_device \
.join(_parent_device, _parent_device.c.id == LotDevice.device_id) \
.join(Component, _parent_device.c.id == Component.parent_id)
"""Join: Path -- Lot -- LotDevice -- ParentDevice (Device) -- Component"""
components = db.select([
Component.id.label('device_id'),
_ancestor.c.id.label('ancestor_lot_id'),
_desc.c.id.label('parent_lot_id'),
LotDevice.device_id.label('device_parent_id'),
]).select_from(_ancestor).select_from(lot_device_component).where(descendants)
__table__ = create_view(
name='lot_device_descendants',
selectable=devices.union(components)
)

View File

@ -1,6 +1,6 @@
import uuid import uuid
from datetime import datetime from datetime import datetime
from typing import Iterable, Set, Union from typing import Iterable, Optional, Set, Union
from uuid import UUID from uuid import UUID
from boltons import urlutils from boltons import urlutils
@ -8,6 +8,7 @@ from sqlalchemy import Column
from sqlalchemy.orm import Query, relationship from sqlalchemy.orm import Query, relationship
from sqlalchemy_utils import Ltree from sqlalchemy_utils import Ltree
from ereuse_devicehub.db import db
from ereuse_devicehub.resources.device.models import Device from ereuse_devicehub.resources.device.models import Device
from ereuse_devicehub.resources.models import Thing from ereuse_devicehub.resources.models import Thing
@ -65,6 +66,9 @@ class Lot(Thing):
def url(self) -> urlutils.URL: def url(self) -> urlutils.URL:
pass pass
def delete(self):
pass
class Path: class Path:
id = ... # type: Column id = ... # type: Column
@ -79,3 +83,17 @@ class Path:
self.lot = ... # type: Lot self.lot = ... # type: Lot
self.path = ... # type: Ltree self.path = ... # type: Ltree
self.created = ... # type: datetime self.created = ... # type: datetime
class LotDeviceDescendants(db.Model):
device_id = ... # type: Column
ancestor_lot_id = ... # type: Column
parent_lot_id = ... # type: Column
device_parent_id = ... # type: Column
def __init__(self) -> None:
super().__init__()
self.device_id = ... # type: int
self.ancestor_lot_id = ... # type: UUID
self.parent_lot_id = ... # type: UUID
self.device_parent_id = ... # type: Optional[int]

View File

@ -97,6 +97,12 @@ class LotView(View):
cls._p(nodes, path) cls._p(nodes, path)
return nodes return nodes
def delete(self, id):
lot = Lot.query.filter_by(id=id).one()
lot.delete()
db.session.commit()
return Response(status=204)
@classmethod @classmethod
def _p(cls, nodes: List[dict], path: deque): def _p(cls, nodes: List[dict], path: deque):
"""Recursively creates the nested lot structure. """Recursively creates the nested lot structure.

View File

@ -1,6 +1,6 @@
from datetime import datetime from datetime import datetime
from sqlalchemy import Column from sqlalchemy import Column, Table
from teal.db import Model from teal.db import Model
STR_SIZE = 64 STR_SIZE = 64
@ -10,6 +10,7 @@ STR_XSM_SIZE = 16
class Thing(Model): class Thing(Model):
__table__ = ... # type: Table
t = ... # type: str t = ... # type: str
type = ... # type: str type = ... # type: str
updated = ... # type: Column updated = ... # type: Column

View File

@ -23,9 +23,9 @@ python-stdnum==1.9
PyYAML==3.13 PyYAML==3.13
requests==2.19.1 requests==2.19.1
requests-mock==1.5.2 requests-mock==1.5.2
SQLAlchemy==1.2.11 SQLAlchemy==1.2.14
SQLAlchemy-Utils==0.33.3 SQLAlchemy-Utils==0.33.6
teal==0.2.0a29 teal==0.2.0a30
webargs==4.0.0 webargs==4.0.0
Werkzeug==0.14.1 Werkzeug==0.14.1
sqlalchemy-citext==1.3.post0 sqlalchemy-citext==1.3.post0

View File

@ -29,7 +29,7 @@ setup(
long_description=long_description, long_description=long_description,
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
install_requires=[ install_requires=[
'teal>=0.2.0a29', # teal always first 'teal>=0.2.0a30', # teal always first
'click', 'click',
'click-spinner', 'click-spinner',
'ereuse-utils[Naming]>=0.4b10', 'ereuse-utils[Naming]>=0.4b10',

View File

@ -23,7 +23,40 @@ In case of error, debug with:
""" """
def test_lot_modify_patch_endpoint(user: UserClient): @pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_model_children():
"""Tests the property Lot.children
l1
|
l2
|
l3
"""
lots = Lot('1'), Lot('2'), Lot('3')
l1, l2, l3 = lots
db.session.add_all(lots)
db.session.flush()
l1.add_child(l2)
db.session.flush()
assert list(l1.children) == [l2]
l2.add_child(l3)
assert list(l1.children) == [l2]
l2.delete()
db.session.flush()
assert not list(l1.children)
l1.delete()
db.session.flush()
l3b = Lot.query.one()
assert l3 == l3b
def test_lot_modify_patch_endpoint_and_delete(user: UserClient):
"""Creates and modifies lot properties through the endpoint""" """Creates and modifies lot properties through the endpoint"""
l, _ = user.post({'name': 'foo', 'description': 'baz'}, res=Lot) l, _ = user.post({'name': 'foo', 'description': 'baz'}, res=Lot)
assert l['name'] == 'foo' assert l['name'] == 'foo'
@ -32,20 +65,17 @@ def test_lot_modify_patch_endpoint(user: UserClient):
l_after, _ = user.get(res=Lot, item=l['id']) l_after, _ = user.get(res=Lot, item=l['id'])
assert l_after['name'] == 'bar' assert l_after['name'] == 'bar'
assert l_after['description'] == 'bax' assert l_after['description'] == 'bax'
user.delete(res=Lot, item=l['id'], status=204)
user.get(res=Lot, item=l['id'], status=404)
@pytest.mark.xfail(reason='No DEL endpoint')
def test_lot_delete_endpoint(user: UserClient):
pass
@pytest.mark.xfail(reason='the IN comparison does not work for device')
@pytest.mark.usefixtures(conftest.auth_app_context.__name__) @pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_device_relationship(): def test_lot_device_relationship():
device = Desktop(serial_number='foo', device = Desktop(serial_number='foo',
model='bar', model='bar',
manufacturer='foobar', manufacturer='foobar',
chassis=ComputerChassis.Lunchbox) chassis=ComputerChassis.Lunchbox)
device.components.add(GraphicCard(serial_number='foo', model='bar1', manufacturer='baz'))
child = Lot('child') child = Lot('child')
child.devices.add(device) child.devices.add(device)
db.session.add(child) db.session.add(child)
@ -253,21 +283,6 @@ def test_lot_roots():
assert set(Lot.roots()) == {l1, l3} assert set(Lot.roots()) == {l1, l3}
@pytest.mark.usefixtures(conftest.auth_app_context.__name__)
def test_lot_model_children():
"""Tests the property Lot.children"""
lots = Lot('1'), Lot('2'), Lot('3')
l1, l2, l3 = lots
db.session.add_all(lots)
db.session.flush()
l1.add_child(l2)
db.session.flush()
children = l1.children
assert list(children) == [l2]
def test_post_get_lot(user: UserClient): def test_post_get_lot(user: UserClient):
"""Tests submitting and retreiving a basic lot.""" """Tests submitting and retreiving a basic lot."""
l, _ = user.post({'name': 'Foo'}, res=Lot) l, _ = user.post({'name': 'Foo'}, res=Lot)