From aa45d1b904967176bfe9706325d9c229ff4f6251 Mon Sep 17 00:00:00 2001 From: Xavier Bustamante Talavera Date: Wed, 30 May 2018 12:49:40 +0200 Subject: [PATCH] Add Tag support; sync with tags; use SQLAlchemy's collection_class; set autoflush to false --- docs/tags.rst | 105 ++++++++ ereuse_devicehub/client.py | 10 +- ereuse_devicehub/config.py | 23 +- ereuse_devicehub/db.py | 2 +- ereuse_devicehub/marshmallow.py | 7 +- .../resources/device/exceptions.py | 13 + ereuse_devicehub/resources/device/models.py | 24 +- ereuse_devicehub/resources/device/schemas.py | 4 +- ereuse_devicehub/resources/device/sync.py | 223 +++++++++++------ ereuse_devicehub/resources/event/__init__.py | 10 + ereuse_devicehub/resources/event/models.py | 38 ++- ereuse_devicehub/resources/event/views.py | 12 +- ereuse_devicehub/resources/models.py | 7 - ereuse_devicehub/resources/tag/__init__.py | 46 ++++ ereuse_devicehub/resources/tag/model.py | 45 ++++ ereuse_devicehub/resources/tag/schema.py | 15 ++ ereuse_devicehub/resources/tag/view.py | 71 ++++++ ereuse_devicehub/resources/user/__init__.py | 50 +++- ereuse_devicehub/resources/user/models.py | 33 ++- setup.py | 8 +- tests/conftest.py | 18 +- tests/test_device.py | 225 ++++++++++++++++-- tests/test_organization.py | 16 ++ tests/test_snapshot.py | 30 +++ tests/test_tag.py | 121 ++++++++++ tests/test_user.py | 3 +- 26 files changed, 987 insertions(+), 172 deletions(-) create mode 100644 docs/tags.rst create mode 100644 ereuse_devicehub/resources/tag/__init__.py create mode 100644 ereuse_devicehub/resources/tag/model.py create mode 100644 ereuse_devicehub/resources/tag/schema.py create mode 100644 ereuse_devicehub/resources/tag/view.py create mode 100644 tests/test_organization.py create mode 100644 tests/test_tag.py diff --git a/docs/tags.rst b/docs/tags.rst new file mode 100644 index 00000000..5d37cfdd --- /dev/null +++ b/docs/tags.rst @@ -0,0 +1,105 @@ +Tags +==== +Devicehub can generate tags, which are synthetic identifiers that +identify a device in an organization. A tag has minimally two fields: +the ID and the Registration Number of the organization that generated +such ID. + +In Devicehub tags are created empty, this is without any device +associated, and they are associated or **linked** when they are assigned +to a device. In Devicehub you usually use the AndroidApp to link +tags and devices. + +The organization that created the tag in the Devicehub (which can be +impersonating the organization that generated the ID) is called the +**tag provider**. This is usual when dealing with other organizations +devices. + +A device can have many tags but a tag can only be linked to one device. +As for the actual implementation, you cannot unlink them. + +Devicehub users can design, generate and print tags, manually setting +an ID and an tag provider. Future Devicehub versions can allow +parametrizing an ID generator. + +Note that these virtual tags don't have to forcefully be printed or +have a physical representation (this is not imposed at system level). + +The eReuse.org tags (eTag) +-------------------------- +We recognize a special type of tag, the **eReuse.org tags (eTag)**. +These are tags defined by eReuse.org and that can be issued only +by tag providers that comply with the eReuse.org requisites. + +The eTags are designed to empower device exchange between +organizations and identification efficiency. They are built with durable +plastic and have a QR code, NFC chip and a written ID. + +These tags live in separate databases from Devicehubs, empowered by +the `eReuse.org Tag `_. By using this +software, eReuse.org certified tag providers can create and manage +the tags, and send them to Devicehubs of their choice. + +Tag ID design +~~~~~~~~~~~~~ +The eTag has a fixed schema for its ID: ``XXX-YYYYYYYYYYYYYY``, where: + +- *XX* is the **eReuse.org Tag Provider ID (eTagPId)**. +- *YYYYYYYYYYYY* is the ID of the tag in the provider.. + +The eTagPid identifies an official eReuse.org Tag provider; this ID +is managed by eReuse.org in a public repository. eTagPIds are made of +2 capital letters and numbers. + +The ID of the tag in the provider (*YYYYYYYYYYYYYY*) consists from +5 to 10 capital letters and numbers (registering a maximum of 10^12 +tags). + +As an example, ``FO-A4CZ2`` is a tag from the ``FO`` tag provider +and ID ``A4CZ2``. + +Creating tags +------------- +You need to create a tag before linking it to a device. There are +two ways of creating a tag: + +- By performing ``POST /tags?ids=...`` and passing a list of tag IDs + to create. All users can create tags this method, however they + cannot create eTags. Get more info at the endpoint docs. +- By executing in a terminal ``flask create-tags `` and passing + a list of IDs to create. Only an admin is supposed to use this method, + which allows them to create eTags. Get more info with + ``flask create-tags --help``. + +Note that tags cannot have a slash ``/``. + +Linking a tag +------------- +Linking a tag is joining the tag with the device. + +In Devicehub this process is done when performing a Snapshot (POST +Snapshot), by setting tag ids in ``snapshot['device']['tags']``. Future +implementation will allow setting to the organization to ensure +tags are inequivocally correct. + +Note that tags must exist in the database prior this. + +You can only link once, and consecutive Snapshots that have the same +tag will validate that the link is correct –so it is good praxis to +try to always provide the tag when performing a Snapshot. Tags help +too in finding devices when these don't generate a ``HID``. Find more +in the ``Snapshot`` docs. + +Getting a device through its tag +-------------------------------- +When performing ``GET /tags//device`` you will get directly the +device of such tag, as long as there are not two tags with the same +tag-id. In such case you should use ``GET /tags///device`` +to inequivocally get the correct device (to develop). + +Tags and migrations +------------------- +Tags travel with the devices they are linked when migrating them. Future +implementations can parameterize this. + +http://t.devicetag.io/TG-1234567890 \ No newline at end of file diff --git a/ereuse_devicehub/client.py b/ereuse_devicehub/client.py index 6aa595a8..b67c71e4 100644 --- a/ereuse_devicehub/client.py +++ b/ereuse_devicehub/client.py @@ -1,4 +1,4 @@ -from typing import Type, Union +from typing import Any, Iterable, Tuple, Type, Union from boltons.typeutils import issubclass from ereuse_utils.test import JSON @@ -23,7 +23,7 @@ class Client(TealClient): uri: str, res: str or Type[Thing] = None, status: Union[int, Type[HTTPException], Type[ValidationError]] = 200, - query: dict = {}, + query: Iterable[Tuple[str, Any]] = tuple(), accept=JSON, content_type=JSON, item=None, @@ -38,7 +38,7 @@ class Client(TealClient): def get(self, uri: str = '', res: Union[Type[Thing], str] = None, - query: dict = {}, + query: Iterable[Tuple[str, Any]] = tuple(), status: Union[int, Type[HTTPException], Type[ValidationError]] = 200, item: Union[int, str] = None, accept: str = JSON, @@ -51,7 +51,7 @@ class Client(TealClient): data: str or dict, uri: str = '', res: Union[Type[Thing], str] = None, - query: dict = {}, + query: Iterable[Tuple[str, Any]] = tuple(), status: Union[int, Type[HTTPException], Type[ValidationError]] = 201, content_type: str = JSON, accept: str = JSON, @@ -89,7 +89,7 @@ class UserClient(Client): uri: str, res: str = None, status: int or HTTPException = 200, - query: dict = {}, + query: Iterable[Tuple[str, Any]] = tuple(), accept=JSON, content_type=JSON, item=None, diff --git a/ereuse_devicehub/config.py b/ereuse_devicehub/config.py index 7fbd5962..124d9580 100644 --- a/ereuse_devicehub/config.py +++ b/ereuse_devicehub/config.py @@ -5,7 +5,8 @@ from ereuse_devicehub.resources.device import ComponentDef, ComputerDef, Desktop NetworkAdapterDef, ProcessorDef, RamModuleDef, ServerDef from ereuse_devicehub.resources.event import AddDef, EventDef, RemoveDef, SnapshotDef, TestDef, \ TestHardDriveDef -from ereuse_devicehub.resources.user import UserDef +from ereuse_devicehub.resources.tag import TagDef +from ereuse_devicehub.resources.user import OrganizationDef, UserDef from teal.config import Config @@ -13,9 +14,25 @@ class DevicehubConfig(Config): RESOURCE_DEFINITIONS = ( DeviceDef, ComputerDef, DesktopDef, LaptopDef, NetbookDef, ServerDef, MicrotowerDef, ComponentDef, GraphicCardDef, HardDriveDef, MotherboardDef, - NetworkAdapterDef, RamModuleDef, ProcessorDef, UserDef, EventDef, AddDef, RemoveDef, - SnapshotDef, TestDef, TestHardDriveDef + NetworkAdapterDef, RamModuleDef, ProcessorDef, UserDef, OrganizationDef, TagDef, EventDef, + AddDef, RemoveDef, SnapshotDef, TestDef, TestHardDriveDef ) PASSWORD_SCHEMES = {'pbkdf2_sha256'} SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/dh-db1' MIN_WORKBENCH = StrictVersion('11.0') + """ + The minimum version of eReuse.org Workbench that this Devicehub + accepts. We recommend not changing this value. + """ + ORGANIZATION_NAME = None # type: str + ORGANIZATION_TAX_ID = None # type: str + """ + The organization using this Devicehub. + + It is used by default, for example, when creating tags. + """ + + def __init__(self, db: str = None) -> None: + if not self.ORGANIZATION_NAME or not self.ORGANIZATION_TAX_ID: + raise ValueError('You need to set the main organization parameters.') + super().__init__(db) diff --git a/ereuse_devicehub/db.py b/ereuse_devicehub/db.py index f37be16b..3e02c2a7 100644 --- a/ereuse_devicehub/db.py +++ b/ereuse_devicehub/db.py @@ -1,3 +1,3 @@ from teal.db import SQLAlchemy -db = SQLAlchemy() +db = SQLAlchemy(session_options={"autoflush": False}) diff --git a/ereuse_devicehub/marshmallow.py b/ereuse_devicehub/marshmallow.py index 113b3bc8..3a754336 100644 --- a/ereuse_devicehub/marshmallow.py +++ b/ereuse_devicehub/marshmallow.py @@ -8,6 +8,7 @@ from teal.marshmallow import NestedOn as TealNestedOn class NestedOn(TealNestedOn): __doc__ = TealNestedOn.__doc__ - def __init__(self, nested, polymorphic_on='type', db: SQLAlchemy = db, default=missing_, - exclude=tuple(), only=None, **kwargs): - super().__init__(nested, polymorphic_on, db, default, exclude, only, **kwargs) + def __init__(self, nested, polymorphic_on='type', db: SQLAlchemy = db, collection_class=list, + default=missing_, exclude=tuple(), only=None, **kwargs): + super().__init__(nested, polymorphic_on, db, collection_class, default, exclude, only, + **kwargs) diff --git a/ereuse_devicehub/resources/device/exceptions.py b/ereuse_devicehub/resources/device/exceptions.py index 9be597e0..0b98c381 100644 --- a/ereuse_devicehub/resources/device/exceptions.py +++ b/ereuse_devicehub/resources/device/exceptions.py @@ -12,3 +12,16 @@ class NeedsId(ValidationError): def __init__(self): message = 'We couldn\'t get an ID for this device. Is this a custom PC?' super().__init__(message) + + +class DeviceIsInAnotherDevicehub(ValidationError): + def __init__(self, + tag_id, + message=None, + field_names=None, + fields=None, + data=None, + valid_data=None, + **kwargs): + message = message or 'Device {} is from another Devicehub.'.format(tag_id) + super().__init__(message, field_names, fields, data, valid_data, **kwargs) diff --git a/ereuse_devicehub/resources/device/models.py b/ereuse_devicehub/resources/device/models.py index f9801b17..76ec3b61 100644 --- a/ereuse_devicehub/resources/device/models.py +++ b/ereuse_devicehub/resources/device/models.py @@ -1,4 +1,5 @@ from contextlib import suppress +from itertools import chain from operator import attrgetter from typing import Dict, Set @@ -7,10 +8,10 @@ from sqlalchemy import BigInteger, Column, Float, ForeignKey, Integer, Sequence, Unicode, inspect from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import ColumnProperty, backref, relationship +from sqlalchemy.util import OrderedSet -from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing, \ - check_range -from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound +from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing +from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, check_range class Device(Thing): @@ -32,10 +33,7 @@ class Device(Thing): @property def events(self) -> list: """All the events performed to the device.""" - # Tried to use chain() but Marshmallow doesn't like it :-( - events = self.events_multiple + self.events_one - events.sort(key=attrgetter('id')) - return events + return sorted(chain(self.events_multiple, self.events_one), key=attrgetter('id')) def __init__(self, *args, **kw) -> None: super().__init__(*args, **kw) @@ -107,13 +105,14 @@ class Microtower(Computer): class Component(Device): id = Column(BigInteger, ForeignKey(Device.id), primary_key=True) # type: int - parent_id = Column(BigInteger, ForeignKey('computer.id')) + parent_id = Column(BigInteger, ForeignKey(Computer.id)) parent = relationship(Computer, backref=backref('components', lazy=True, cascade=CASCADE, - order_by=lambda: Component.id), - primaryjoin='Component.parent_id == Computer.id') # type: Device + order_by=lambda: Component.id, + collection_class=OrderedSet), + primaryjoin=parent_id == Computer.id) # type: Device def similar_one(self, parent: Computer, blacklist: Set[int]) -> 'Component': """ @@ -136,10 +135,7 @@ class Component(Device): @property def events(self) -> list: - events = super().events - events.extend(self.events_components) - events.sort(key=attrgetter('id')) - return events + return sorted(chain(super().events, self.events_components), key=attrgetter('id')) class JoinedComponentTableMixin: diff --git a/ereuse_devicehub/resources/device/schemas.py b/ereuse_devicehub/resources/device/schemas.py index c3718541..0f3490f1 100644 --- a/ereuse_devicehub/resources/device/schemas.py +++ b/ereuse_devicehub/resources/device/schemas.py @@ -1,5 +1,6 @@ from marshmallow.fields import Float, Integer, Str from marshmallow.validate import Length, OneOf, Range +from sqlalchemy.util import OrderedSet from ereuse_devicehub.marshmallow import NestedOn from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE @@ -12,6 +13,7 @@ class Device(Thing): hid = Str(dump_only=True, description='The Hardware ID is the unique ID traceability systems ' 'use to ID a device globally.') + tags = NestedOn('Tag', many=True, collection_class=OrderedSet) pid = Str(description='The PID identifies a device under a circuit or platform.', validate=Length(max=STR_SIZE)) gid = Str(description='The Giver ID links the device to the giver\'s (donor, seller)' @@ -34,7 +36,7 @@ class Device(Thing): class Computer(Device): - components = NestedOn('Component', many=True, dump_only=True) + components = NestedOn('Component', many=True, dump_only=True, collection_class=OrderedSet) pass diff --git a/ereuse_devicehub/resources/device/sync.py b/ereuse_devicehub/resources/device/sync.py index 206af5ad..8de4dac2 100644 --- a/ereuse_devicehub/resources/device/sync.py +++ b/ereuse_devicehub/resources/device/sync.py @@ -1,140 +1,194 @@ -import re from contextlib import suppress from itertools import groupby -from typing import Iterable, List, Set +from typing import Iterable, Set -from psycopg2.errorcodes import UNIQUE_VIOLATION -from sqlalchemy.exc import IntegrityError +from sqlalchemy import inspect +from sqlalchemy.util import OrderedSet from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.exceptions import NeedsId from ereuse_devicehub.resources.device.models import Component, Computer, Device -from ereuse_devicehub.resources.event.models import Add, Remove +from ereuse_devicehub.resources.event.models import Remove +from ereuse_devicehub.resources.tag.model import Tag from teal.db import ResourceNotFound +from teal.marshmallow import ValidationError class Sync: """Synchronizes the device and components with the database.""" - @classmethod - def run(cls, device: Device, - components: Iterable[Component] or None) -> (Device, List[Add or Remove]): + def run(self, + device: Device, + components: Iterable[Component] or None) -> (Device, OrderedSet): """ Synchronizes the device and components with the database. Identifies if the device and components exist in the database and updates / inserts them as necessary. + Passed-in parameters have to be transient, or said differently, + not-db-synced objects, or otherwise they would end-up being + added in the session. `Learn more... `_. + This performs Add / Remove as necessary. + :param device: The device to add / update to the database. :param components: Components that are inside of the device. This method performs Add and Remove events so the device ends up with these components. Components are added / updated accordingly. If this is empty, all components are removed. - If this is None, it means that there is - no info about components and the already - existing components of the device (in case - the device already exists) won't be touch. + If this is None, it means that we are not + providing info about the components, in which + case we keep the already existing components + of the device –we don't touch them. :return: A tuple of: + 1. The device from the database (with an ID) whose ``components`` field contain the db version of the passed-in components. 2. A list of Add / Remove (not yet added to session). """ - db_device, _ = cls.execute_register(device) - db_components, events = [], [] + db_device = self.execute_register(device) + db_components, events = OrderedSet(), OrderedSet() if components is not None: # We have component info (see above) blacklist = set() # type: Set[int] not_new_components = set() for component in components: - db_component, is_new = cls.execute_register(component, blacklist, parent=db_device) - db_components.append(db_component) + db_component, is_new = self.execute_register_component(component, + blacklist, + parent=db_device) + db_components.add(db_component) if not is_new: not_new_components.add(db_component) # We only want to perform Add/Remove to not new components - events = cls.add_remove(db_device, not_new_components) + events = self.add_remove(db_device, not_new_components) db_device.components = db_components return db_device, events - @classmethod - def execute_register(cls, device: Device, - blacklist: Set[int] = None, - parent: Computer = None) -> (Device, bool): + def execute_register_component(self, + component: Component, + blacklist: Set[int], + parent: Computer): """ - Synchronizes one device to the DB. + Synchronizes one component to the DB. - This method tries to create a device into the database, and - if it already exists it returns a "local synced version", - this is the same ``device`` you passed-in but with updated - values from the database one (like the id value). + This method is a specialization of :meth:`.execute_register` + but for components that are inside parents. - When we say "local" we mean that if, the device existed on the - database, we do not "touch" any of its values on the DB. + This method assumes components don't have tags, and it tries + to identify a non-hid component by finding a + :meth:`ereuse_devicehub.resources.device.models.Component. + similar_one`. - :param device: The device to synchronize to the DB. + :param component: The component to sync. :param blacklist: A set of components already found by Component.similar_one(). Pass-in an empty Set. :param parent: For components, the computer that contains them. Helper used by Component.similar_one(). :return: A tuple with: - 1. A synchronized device with the DB. It can be a new - device or an already existing one. - 2. A flag stating if the device is new or it existed - already in the DB. + - The synced component. See :meth:`.execute_register` + for more info. + - A flag stating if the device is new or it already + existed in the DB. + """ + assert inspect(component).transient, 'Component should not be synced from DB' + try: + if component.hid: + db_component = Device.query.filter_by(hid=component.hid).one() + else: + # Is there a component similar to ours? + db_component = component.similar_one(parent, blacklist) + # We blacklist this component so we + # ensure we don't get it again for another component + # with the same physical properties + blacklist.add(db_component.id) + except ResourceNotFound: + db.session.add(component) + db.session.flush() + db_component = component + is_new = True + else: + self.merge(component, db_component) + is_new = False + return db_component, is_new + + def execute_register(self, device: Device) -> Device: + """ + Synchronizes one device to the DB. + + This method tries to get an existing device using the HID + or one of the tags, and... + + - if it already exists it returns a "local synced version" + –the same ``device`` you passed-in but with updated values + from the database. In this case we do not + "touch" any of its values on the DB. + - If it did not exist, a new device is created in the db. + + This method validates that all passed-in tags (``device.tags``), + if linked, are linked to the same device, ditto for the hid. + Finally it links the tags with the device. + + If you pass-in a component that is inside a parent, use + :meth:`.execute_register_component` as it has more specialized + methods to handle them. + + :param device: The device to synchronize to the DB. :raise NeedsId: The device has not any identifier we can use. To still create the device use ``force_creation``. :raise DatabaseError: Any other error from the DB. + :return: The synced device from the db with the tags linked. """ - # Let's try to create the device - if not device.hid and not device.id: - # We won't be able to surely identify this device - if isinstance(device, Component): - with suppress(ResourceNotFound): - # Is there a component similar to ours? - db_component = device.similar_one(parent, blacklist) - # We blacklist this component so we - # ensure we don't get it again for another component - # with the same physical properties - blacklist.add(db_component.id) - return cls.merge(device, db_component), False - else: - raise NeedsId() - try: - with db.session.begin_nested(): - # Create transaction savepoint to auto-rollback on insertion err - # Let's try to insert or update - db.session.add(device) - db.session.flush() - except IntegrityError as e: - if e.orig.diag.sqlstate == UNIQUE_VIOLATION: - db.session.rollback() - # This device already exists in the DB - field, value = ( - x.replace('(', '').replace(')', '') - for x in re.findall('\(.*?\)', e.orig.diag.message_detail) - ) - db_device = Device.query.filter_by(**{field: value}).one() # type: Device - return cls.merge(device, db_device), False - else: - raise e - else: - return device, True # Our device is new + assert inspect(device).transient, 'Device cannot be already synced from DB' + assert all(inspect(tag).transient for tag in device.tags), 'Tags cannot be synced from DB' + if not device.tags and not device.hid: + # We cannot identify this device + raise NeedsId() + db_device = None + if device.hid: + with suppress(ResourceNotFound): + db_device = Device.query.filter_by(hid=device.hid).one() + tags = {Tag.query.filter_by(id=tag.id).one() for tag in device.tags} # type: Set[Tag] + linked_tags = {tag for tag in tags if tag.device_id} # type: Set[Tag] + if linked_tags: + sample_tag = next(iter(linked_tags)) + for tag in linked_tags: + if tag.device_id != sample_tag.device_id: + raise MismatchBetweenTags(tag, sample_tag) # Linked to different devices + if db_device: # Device from hid + if sample_tag.device_id != db_device.id: # Device from hid != device from tags + raise MismatchBetweenTagsAndHid(db_device.id, db_device.hid) + else: # There was no device from hid + db_device = sample_tag.device + if db_device: # Device from hid or tags + self.merge(device, db_device) + else: # Device is new and tags are not linked to a device + device.tags.clear() # We don't want to add the transient dummy tags + db.session.add(device) + db_device = device + db_device.tags |= tags # Union of tags the device had plus the (potentially) new ones + db.session.flush() + assert db_device is not None + return db_device - @classmethod - def merge(cls, device: Device, db_device: Device): + @staticmethod + def merge(device: Device, db_device: Device): """ Copies the physical properties of the device to the db_device. + + This method mutates db_device. """ for field_name, value in device.physical_properties.items(): if value is not None: setattr(db_device, field_name, value) - return db_device - @classmethod - def add_remove(cls, device: Device, - components: Set[Component]) -> List[Add or Remove]: + @staticmethod + def add_remove(device: Device, + components: Set[Component]) -> OrderedSet: """ Generates the Add and Remove events (but doesn't add them to session). @@ -149,7 +203,7 @@ class Sync: :return: A list of Add / Remove events. """ # Note that we create the Remove events before the Add ones - events = [] + events = OrderedSet() old_components = set(device.components) adding = components - old_components @@ -160,5 +214,24 @@ class Sync: for parent, _components in groupby(sorted(adding, key=g_parent), key=g_parent): if parent.id != 0: # Is not Computer Identity - events.append(Remove(device=parent, components=list(_components))) + events.add(Remove(device=parent, components=OrderedSet(_components))) return events + + +class MismatchBetweenTags(ValidationError): + def __init__(self, + tag: Tag, + other_tag: Tag, + field_names={'device.tags'}): + message = '{!r} and {!r} are linked to different devices.'.format(tag, other_tag) + super().__init__(message, field_names) + + +class MismatchBetweenTagsAndHid(ValidationError): + def __init__(self, + device_id: int, + hid: str, + field_names={'device.hid'}): + message = 'Tags are linked to device {} but hid refers to device {}.'.format(device_id, + hid) + super().__init__(message, field_names) diff --git a/ereuse_devicehub/resources/event/__init__.py b/ereuse_devicehub/resources/event/__init__.py index 827b87a7..02736217 100644 --- a/ereuse_devicehub/resources/event/__init__.py +++ b/ereuse_devicehub/resources/event/__init__.py @@ -1,3 +1,6 @@ +from typing import Callable, Iterable, Tuple + +from ereuse_devicehub.resources.device.sync import Sync from ereuse_devicehub.resources.event.schemas import Add, Event, Remove, Snapshot, Test, \ TestHardDrive from ereuse_devicehub.resources.event.views import EventView, SnapshotView @@ -23,6 +26,13 @@ class SnapshotDef(EventDef): SCHEMA = Snapshot VIEW = SnapshotView + def __init__(self, app, import_name=__package__, static_folder=None, static_url_path=None, + template_folder=None, url_prefix=None, subdomain=None, url_defaults=None, + root_path=None, cli_commands: Iterable[Tuple[Callable, str or None]] = tuple()): + super().__init__(app, import_name, static_folder, static_url_path, template_folder, + url_prefix, subdomain, url_defaults, root_path, cli_commands) + self.sync = Sync() + class TestDef(EventDef): SCHEMA = Test diff --git a/ereuse_devicehub/resources/event/models.py b/ereuse_devicehub/resources/event/models.py index 48050e92..ff79e861 100644 --- a/ereuse_devicehub/resources/event/models.py +++ b/ereuse_devicehub/resources/event/models.py @@ -7,17 +7,17 @@ from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, DateTime, E from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.orm import backref, relationship, validates +from sqlalchemy.util import OrderedSet from sqlalchemy_utils import ColorType from ereuse_devicehub.db import db from ereuse_devicehub.resources.device.models import Component, Device from ereuse_devicehub.resources.event.enums import Appearance, Bios, Functionality, Orientation, \ SoftwareType, StepTypes, TestHardDriveLength -from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing, \ - check_range +from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing from ereuse_devicehub.resources.user.models import User from teal.db import CASCADE, CASCADE_OWN, INHERIT_COND, POLYMORPHIC_ID, POLYMORPHIC_ON, \ - StrictVersionType + StrictVersionType, check_range class JoinedTableMixin: @@ -39,7 +39,10 @@ class Event(Thing): use_alter=True, name='snapshot_events')) snapshot = relationship('Snapshot', - backref=backref('events', lazy=True, cascade=CASCADE), + backref=backref('events', + lazy=True, + cascade=CASCADE, + collection_class=OrderedSet), primaryjoin='Event.snapshot_id == Snapshot.id') author_id = Column(UUID(as_uuid=True), @@ -47,14 +50,16 @@ class Event(Thing): nullable=False, default=lambda: g.user.id) author = relationship(User, - backref=backref('events', lazy=True), + backref=backref('events', lazy=True, collection_class=set), primaryjoin=author_id == User.id) components = relationship(Component, backref=backref('events_components', lazy=True, - order_by=lambda: Event.id), + order_by=lambda: Event.id, + collection_class=OrderedSet), secondary=lambda: EventComponent.__table__, - order_by=lambda: Device.id) + order_by=lambda: Device.id, + collection_class=OrderedSet) @declared_attr def __mapper_args__(cls): @@ -84,7 +89,8 @@ class EventWithOneDevice(Event): backref=backref('events_one', lazy=True, cascade=CASCADE, - order_by=lambda: EventWithOneDevice.id), + order_by=lambda: EventWithOneDevice.id, + collection_class=OrderedSet), primaryjoin=Device.id == device_id) def __repr__(self) -> str: @@ -98,7 +104,8 @@ class EventWithMultipleDevices(Event): devices = relationship(Device, backref=backref('events_multiple', lazy=True, - order_by=lambda: EventWithMultipleDevices.id), + order_by=lambda: EventWithMultipleDevices.id, + collection_class=OrderedSet), secondary=lambda: EventDevice.__table__, order_by=lambda: Device.id) @@ -193,15 +200,22 @@ class SnapshotRequest(db.Model): id = Column(BigInteger, ForeignKey(Snapshot.id), primary_key=True) request = Column(JSON, nullable=False) - snapshot = relationship(Snapshot, backref=backref('request', lazy=True, uselist=False, - cascade=CASCADE_OWN)) + snapshot = relationship(Snapshot, + backref=backref('request', + lazy=True, + uselist=False, + cascade=CASCADE_OWN)) class Test(JoinedTableMixin, EventWithOneDevice): elapsed = Column(Interval, nullable=False) success = Column(Boolean, nullable=False) - snapshot = relationship(Snapshot, backref=backref('tests', lazy=True, cascade=CASCADE_OWN)) + snapshot = relationship(Snapshot, backref=backref('tests', + lazy=True, + cascade=CASCADE_OWN, + order_by=Event.id, + collection_class=OrderedSet)) class TestHardDrive(Test): diff --git a/ereuse_devicehub/resources/event/views.py b/ereuse_devicehub/resources/event/views.py index 6a4d65f9..24afd5de 100644 --- a/ereuse_devicehub/resources/event/views.py +++ b/ereuse_devicehub/resources/event/views.py @@ -1,9 +1,10 @@ from distutils.version import StrictVersion from flask import request +from sqlalchemy.util import OrderedSet from ereuse_devicehub.db import db -from ereuse_devicehub.resources.device.sync import Sync +from ereuse_devicehub.resources.device.models import Device from ereuse_devicehub.resources.event.enums import SoftwareType from ereuse_devicehub.resources.event.models import Event, Snapshot, TestHardDrive from teal.resource import View @@ -30,16 +31,15 @@ class SnapshotView(View): # Note that if we set the device / components into the snapshot # model object, when we flush them to the db we will flush # snapshot, and we want to wait to flush snapshot at the end - device = s.pop('device') + device = s.pop('device') # type: Device components = s.pop('components') if s['software'] == SoftwareType.Workbench else None # noinspection PyArgumentList snapshot = Snapshot(**s) - snapshot.device, snapshot.events = Sync.run(device, components) + snapshot.device, snapshot.events = self.resource_def.sync.run(device, components) snapshot.components = snapshot.device.components # commit will change the order of the components by what - # the DB wants. Let's get a copy of the list so we preserve - # order - ordered_components = [c for c in snapshot.components] + # the DB wants. Let's get a copy of the list so we preserve order + ordered_components = OrderedSet(x for x in snapshot.components) db.session.add(snapshot) db.session.commit() # todo we are setting snapshot dirty again with this components but diff --git a/ereuse_devicehub/resources/models.py b/ereuse_devicehub/resources/models.py index 1a62a362..acdd8bb9 100644 --- a/ereuse_devicehub/resources/models.py +++ b/ereuse_devicehub/resources/models.py @@ -1,7 +1,5 @@ from datetime import datetime -from sqlalchemy import CheckConstraint - from ereuse_devicehub.db import db STR_SIZE = 64 @@ -9,11 +7,6 @@ STR_BIG_SIZE = 128 STR_SM_SIZE = 32 -def check_range(column: str, min=1, max=None) -> CheckConstraint: - constraint = '>= {}'.format(min) if max is None else 'BETWEEN {} AND {}'.format(min, max) - return CheckConstraint('{} {}'.format(column, constraint)) - - class Thing(db.Model): __abstract__ = True updated = db.Column(db.DateTime, onupdate=datetime.utcnow) diff --git a/ereuse_devicehub/resources/tag/__init__.py b/ereuse_devicehub/resources/tag/__init__.py new file mode 100644 index 00000000..ea4e90cf --- /dev/null +++ b/ereuse_devicehub/resources/tag/__init__.py @@ -0,0 +1,46 @@ +from typing import Tuple + +from click import argument, option + +from ereuse_devicehub.db import db +from ereuse_devicehub.resources.tag.model import Tag +from ereuse_devicehub.resources.tag.schema import Tag as TagS +from ereuse_devicehub.resources.tag.view import TagView, get_device_from_tag +from teal.resource import Resource +from teal.teal import Teal + + +class TagDef(Resource): + SCHEMA = TagS + VIEW = TagView + + def __init__(self, app: Teal, import_name=__package__, static_folder=None, + static_url_path=None, + template_folder=None, url_prefix=None, subdomain=None, url_defaults=None, + root_path=None): + cli_commands = ( + (self.create_tags, 'create-tags'), + ) + super().__init__(app, import_name, static_folder, static_url_path, template_folder, + url_prefix, subdomain, url_defaults, root_path, cli_commands) + _get_device_from_tag = app.auth.requires_auth(get_device_from_tag) + self.add_url_rule('/<{}:{}>/device'.format(self.ID_CONVERTER.value, self.ID_NAME), + view_func=_get_device_from_tag, + methods={'GET'}) + + @option('--org', + help='The name of an existing organization in the DB. ' + 'By default the organization operating this Devicehub.') + @option('--provider', + help='The Base URL of the provider. ' + 'By default set to the actual Devicehub.') + @argument('ids', nargs=-1, required=True) + def create_tags(self, ids: Tuple[str], org: str = None, provider: str = None): + """Create TAGS and associates them to a specific PROVIDER.""" + tag_schema = TagS(only=('id', 'provider', 'org')) + + db.session.add_all( + Tag(**tag_schema.load({'id': tag_id, 'provider': provider, 'org': org})) + for tag_id in ids + ) + db.session.commit() diff --git a/ereuse_devicehub/resources/tag/model.py b/ereuse_devicehub/resources/tag/model.py new file mode 100644 index 00000000..8f9fdcd7 --- /dev/null +++ b/ereuse_devicehub/resources/tag/model.py @@ -0,0 +1,45 @@ +from sqlalchemy import BigInteger, Column, ForeignKey, Unicode, UniqueConstraint +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import backref, relationship, validates + +from ereuse_devicehub.resources.device.models import Device +from ereuse_devicehub.resources.models import Thing +from ereuse_devicehub.resources.user.models import Organization +from teal.db import DB_CASCADE_SET_NULL, URL +from teal.marshmallow import ValidationError + + +class Tag(Thing): + id = Column(Unicode(), primary_key=True) + org_id = Column(UUID(as_uuid=True), + ForeignKey(Organization.id), + primary_key=True, + # If we link with the Organization object this instance + # will be set as persistent and added to session + # which is something we don't want to enforce by default + default=lambda: Organization.get_default_org().id) + org = relationship(Organization, + backref=backref('tags', lazy=True), + primaryjoin=Organization.id == org_id, + collection_class=set) + provider = Column(URL(), + comment='The tag provider URL. If None, the provider is this Devicehub.') + device_id = Column(BigInteger, + # We don't want to delete the tag on device deletion, only set to null + ForeignKey(Device.id, ondelete=DB_CASCADE_SET_NULL)) + device = relationship(Device, + backref=backref('tags', lazy=True, collection_class=set), + primaryjoin=Device.id == device_id) + + @validates('id') + def does_not_contain_slash(self, _, value: str): + if '/' in value: + raise ValidationError('Tags cannot contain slashes (/).') + return value + + __table_args__ = ( + UniqueConstraint(device_id, org_id, name='One tag per organization.'), + ) + + def __repr__(self) -> str: + return ''.format(self) diff --git a/ereuse_devicehub/resources/tag/schema.py b/ereuse_devicehub/resources/tag/schema.py new file mode 100644 index 00000000..00ca28eb --- /dev/null +++ b/ereuse_devicehub/resources/tag/schema.py @@ -0,0 +1,15 @@ +from marshmallow.fields import String + +from ereuse_devicehub.marshmallow import NestedOn +from ereuse_devicehub.resources.device.schemas import Device +from ereuse_devicehub.resources.schemas import Thing +from teal.marshmallow import URL + + +class Tag(Thing): + id = String(description='The ID of the tag.', + validator=lambda x: '/' not in x, + required=True) + provider = URL(description='The provider URL.') + device = NestedOn(Device, description='The device linked to this tag.') + org = String(description='The organization that issued the tag.') diff --git a/ereuse_devicehub/resources/tag/view.py b/ereuse_devicehub/resources/tag/view.py new file mode 100644 index 00000000..318e6d1a --- /dev/null +++ b/ereuse_devicehub/resources/tag/view.py @@ -0,0 +1,71 @@ +from flask import Response, current_app as app, request +from marshmallow import Schema +from marshmallow.fields import List, String, URL +from webargs.flaskparser import parser + +from ereuse_devicehub.resources.device.models import Device +from ereuse_devicehub.resources.tag import Tag +from teal.marshmallow import ValidationError +from teal.resource import View + + +class TagView(View): + class PostArgs(Schema): + ids = List(String(), required=True, description='A list of tags identifiers.') + org = String(description='The name of an existing organization in the DB. ' + 'If not set, the default organization is used.') + provider = URL(description='The Base URL of the provider. By default is this Devicehub.') + + post_args = PostArgs() + + def post(self): + """ + Creates tags. + + --- + parameters: + - name: tags + in: path + description: Number of tags to create. + """ + args = parser.parse(self.post_args, request, locations={'querystring'}) + # Ensure user is not POSTing an eReuse.org tag + # For now only allow them to be created through command-line + for id in args['ids']: + try: + provider, _id = id.split('-') + except ValueError: + pass + else: + if len(provider) == 2 and 5 <= len(_id) <= 10: + raise CannotCreateETag(id) + self.resource_def.create_tags(**args) + return Response(status=201) + + +def get_device_from_tag(id: str): + """ + Gets the device by passing a tag id. + + Example: /tags/23/device. + + :raise MultipleTagsPerId: More than one tag per Id. Please, use + the /tags///device URL to disambiguate. + """ + # todo this could be more efficient by Device.query... join with tag + device = Tag.query.filter_by(id=id).one().device + if device is None: + raise TagNotLinked(id) + return app.resources[Device.t].schema.jsonify(device) + + +class CannotCreateETag(ValidationError): + def __init__(self, id: str): + message = 'Only sysadmin can create an eReuse.org Tag ({})'.format(id) + super().__init__(message) + + +class TagNotLinked(ValidationError): + def __init__(self, id): + message = 'The tag {} is not linked to a device.'.format(id) + super().__init__(message, field_names=['device']) diff --git a/ereuse_devicehub/resources/user/__init__.py b/ereuse_devicehub/resources/user/__init__.py index 05a6111b..7e212555 100644 --- a/ereuse_devicehub/resources/user/__init__.py +++ b/ereuse_devicehub/resources/user/__init__.py @@ -1,10 +1,12 @@ from click import argument, option +from flask import current_app as app from ereuse_devicehub import devicehub from ereuse_devicehub.db import db -from ereuse_devicehub.resources.user.models import User +from ereuse_devicehub.resources.user.models import Organization, User from ereuse_devicehub.resources.user.schemas import User as UserS from ereuse_devicehub.resources.user.views import UserView, login +from teal.db import SQLAlchemy from teal.resource import Converters, Resource @@ -17,7 +19,7 @@ class UserDef(Resource): def __init__(self, app: 'devicehub.Devicehub', import_name=__package__, static_folder=None, static_url_path=None, template_folder=None, url_prefix=None, subdomain=None, url_defaults=None, root_path=None): - cli_commands = ((self.create_user, 'user create'),) + cli_commands = ((self.create_user, 'create-user'),) super().__init__(app, import_name, static_folder, static_url_path, template_folder, url_prefix, subdomain, url_defaults, root_path, cli_commands) self.add_url_rule('/login', view_func=login, methods={'POST'}) @@ -28,10 +30,40 @@ class UserDef(Resource): """ Creates an user. """ - with self.app.app_context(): - self.SCHEMA(only={'email', 'password'}, exclude=('token',)) \ - .load({'email': email, 'password': password}) - user = User(email=email, password=password) - db.session.add(user) - db.session.commit() - return self.schema.dump(user) + u = self.SCHEMA(only={'email', 'password'}, exclude=('token',)) \ + .load({'email': email, 'password': password}) + user = User(**u) + db.session.add(user) + db.session.commit() + return self.schema.dump(user) + + +class OrganizationDef(Resource): + __type__ = 'Organization' + ID_CONVERTER = Converters.uid + AUTH = True + + def __init__(self, app, import_name=__package__, static_folder=None, static_url_path=None, + template_folder=None, url_prefix=None, subdomain=None, url_defaults=None, + root_path=None): + cli_commands = ((self.create_org, 'create-org'),) + super().__init__(app, import_name, static_folder, static_url_path, template_folder, + url_prefix, subdomain, url_defaults, root_path, cli_commands) + + @argument('name') + @argument('tax_id') + @argument('country') + def create_org(self, **kw: dict) -> dict: + """ + Creates an organization. + COUNTRY has to be 2 characters as defined by + """ + org = Organization(**self.schema.load(kw)) + db.session.add(org) + db.session.commit() + return self.schema.dump(org) + + def init_db(self, db: SQLAlchemy): + """Creates the default organization.""" + org = Organization(**app.config.get_namespace('ORGANIZATION_')) + db.session.add(org) diff --git a/ereuse_devicehub/resources/user/models.py b/ereuse_devicehub/resources/user/models.py index f7ea7926..6200ddc7 100644 --- a/ereuse_devicehub/resources/user/models.py +++ b/ereuse_devicehub/resources/user/models.py @@ -1,11 +1,11 @@ from uuid import uuid4 -from flask import current_app -from sqlalchemy import Column, Unicode +from flask import current_app as app +from sqlalchemy import Column, Unicode, UniqueConstraint from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy_utils import EmailType, PasswordType +from sqlalchemy_utils import CountryType, EmailType, PasswordType -from ereuse_devicehub.resources.models import STR_SIZE, Thing +from ereuse_devicehub.resources.models import STR_SIZE, STR_SM_SIZE, Thing class User(Thing): @@ -14,7 +14,7 @@ class User(Thing): email = Column(EmailType, nullable=False, unique=True) password = Column(PasswordType(max_length=STR_SIZE, onload=lambda **kwargs: dict( - schemes=current_app.config['PASSWORD_SCHEMES'], + schemes=app.config['PASSWORD_SCHEMES'], **kwargs ))) """ @@ -26,4 +26,25 @@ class User(Thing): token = Column(UUID(as_uuid=True), default=uuid4, unique=True) def __repr__(self) -> str: - return '<{0.t} {0.id!r} email={0.email!r}>'.format(self) + return '<{0.t} {0.id} email={0.email}>'.format(self) + + +class Organization(Thing): + id = Column(UUID(as_uuid=True), default=uuid4, primary_key=True) + name = Column(Unicode(length=STR_SM_SIZE), unique=True) + tax_id = Column(Unicode(length=STR_SM_SIZE), + comment='The Tax / Fiscal ID of the organization, ' + 'e.g. the TIN in the US or the CIF/NIF in Spain.') + country = Column(CountryType, comment='Country issuing the tax_id number.') + + __table_args__ = ( + UniqueConstraint(tax_id, country, name='Registration Number per country.'), + ) + + @classmethod + def get_default_org(cls) -> 'Organization': + """Retrieves the default organization.""" + return Organization.query.filter_by(**app.config.get_namespace('ORGANIZATION_')).one() + + def __repr__(self) -> str: + return ''.format(self) diff --git a/setup.py b/setup.py index d94d390c..8bce8928 100644 --- a/setup.py +++ b/setup.py @@ -14,11 +14,15 @@ setup( 'marshmallow_enum', 'ereuse-utils [Naming]', 'psycopg2-binary', - 'sqlalchemy-utils' + 'sqlalchemy-utils', + 'requests', + 'requests-toolbelt', + 'hashids' ], tests_requires=[ 'pytest', - 'pytest-datadir' + 'pytest-datadir', + 'requests_mock' ], classifiers={ 'Development Status :: 4 - Beta', diff --git a/tests/conftest.py b/tests/conftest.py index 55041c01..09a33485 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ from ereuse_devicehub.client import Client, UserClient from ereuse_devicehub.config import DevicehubConfig from ereuse_devicehub.db import db from ereuse_devicehub.devicehub import Devicehub +from ereuse_devicehub.resources.tag import Tag from ereuse_devicehub.resources.user.models import User @@ -14,6 +15,8 @@ class TestConfig(DevicehubConfig): SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/dh_test' SCHEMA = 'test' TESTING = True + ORGANIZATION_NAME = 'FooOrg' + ORGANIZATION_TAX_ID = 'FooOrgId' @pytest.fixture(scope='module') @@ -28,7 +31,8 @@ def _app(config: TestConfig) -> Devicehub: @pytest.fixture() def app(request, _app: Devicehub) -> Devicehub: - db.create_all(app=_app) + with _app.app_context(): + _app.init_db() # More robust than 'yield' request.addfinalizer(lambda *args, **kw: db.drop_all(app=_app)) return _app @@ -80,6 +84,16 @@ def auth_app_context(app: Devicehub): def file(name: str) -> dict: - """Opens and parses a JSON file from the ``files`` subdir.""" + """Opens and parses a YAML file from the ``files`` subdir.""" with Path(__file__).parent.joinpath('files').joinpath(name + '.yaml').open() as f: return yaml.load(f) + + +@pytest.fixture() +def tag_id(app: Devicehub) -> str: + """Creates a tag and returns its id.""" + with app.app_context(): + t = Tag(id='foo') + db.session.add(t) + db.session.commit() + return t.id diff --git a/tests/test_device.py b/tests/test_device.py index be165cb6..87bfcb22 100644 --- a/tests/test_device.py +++ b/tests/test_device.py @@ -2,6 +2,9 @@ from datetime import timedelta from uuid import UUID import pytest +from ereuse_utils.naming import Naming +from pytest import raises +from sqlalchemy.util import OrderedSet from ereuse_devicehub.client import UserClient from ereuse_devicehub.db import db @@ -10,8 +13,10 @@ from ereuse_devicehub.resources.device.exceptions import NeedsId from ereuse_devicehub.resources.device.models import Component, Computer, Desktop, Device, \ GraphicCard, Laptop, Microtower, Motherboard, NetworkAdapter from ereuse_devicehub.resources.device.schemas import Device as DeviceS -from ereuse_devicehub.resources.device.sync import Sync +from ereuse_devicehub.resources.device.sync import MismatchBetweenTags, MismatchBetweenTagsAndHid, \ + Sync from ereuse_devicehub.resources.event.models import Remove, Test +from ereuse_devicehub.resources.tag.model import Tag from ereuse_devicehub.resources.user import User from teal.db import ResourceNotFound from tests.conftest import file @@ -23,23 +28,23 @@ def test_device_model(): Tests that the correctness of the device model and its relationships. """ pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s') - pc.components = components = [ - NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'), - GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500) - ] + net = NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s') + graphic = GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500) + pc.components.add(net) + pc.components.add(graphic) db.session.add(pc) db.session.commit() pc = Desktop.query.one() assert pc.serial_number == 'p1s' - assert pc.components == components + assert pc.components == OrderedSet([net, graphic]) network_adapter = NetworkAdapter.query.one() assert network_adapter.parent == pc # Removing a component from pc doesn't delete the component - del pc.components[0] + pc.components.remove(net) db.session.commit() pc = Device.query.first() # this is the same as querying for Desktop directly - assert pc.components[0].type == GraphicCard.__name__ + assert pc.components == {graphic} network_adapter = NetworkAdapter.query.one() assert network_adapter not in pc.components assert network_adapter.parent is None @@ -47,6 +52,7 @@ def test_device_model(): # Deleting the pc deletes everything gcard = GraphicCard.query.one() db.session.delete(pc) + db.session.flush() assert pc.id == 1 assert Desktop.query.first() is None db.session.commit() @@ -74,7 +80,8 @@ def test_physical_properties(): manufacturer='mr', width=2.0, pid='abc') - pc = Computer(components=[c]) + pc = Computer() + pc.components.add(c) db.session.add(pc) db.session.commit() assert c.physical_properties == { @@ -99,9 +106,10 @@ def test_component_similar_one(): snapshot = file('pc-components.db') d = snapshot['device'] snapshot['components'][0]['serial_number'] = snapshot['components'][1]['serial_number'] = None - pc = Computer(**d, components=[Component(**c) for c in snapshot['components']]) + pc = Computer(**d, components=OrderedSet(Component(**c) for c in snapshot['components'])) component1, component2 = pc.components # type: Component db.session.add(pc) + db.session.flush() # Let's create a new component named 'A' similar to 1 componentA = Component(model=component1.model, manufacturer=component1.manufacturer) similar_to_a = componentA.similar_one(pc, set()) @@ -124,10 +132,10 @@ def test_add_remove(): values = file('pc-components.db') pc = values['device'] c1, c2 = [Component(**c) for c in values['components']] - pc = Computer(**pc, components=[c1, c2]) + pc = Computer(**pc, components=OrderedSet([c1, c2])) db.session.add(pc) c3 = Component(serial_number='nc1') - pc2 = Computer(serial_number='s2', components=[c3]) + pc2 = Computer(serial_number='s2', components=OrderedSet([c3])) c4 = Component(serial_number='c4s') db.session.add(pc2) db.session.add(c4) @@ -141,45 +149,212 @@ def test_add_remove(): assert len(events) == 1 assert isinstance(events[0], Remove) assert events[0].device == pc2 - assert events[0].components == [c3] + assert events[0].components == OrderedSet([c3]) @pytest.mark.usefixtures('app_context') -def test_execute_register_computer(): +def test_sync_run_components_empty(): + """ + Syncs a device that has an empty components list. The system should + remove all the components from the device. + """ + s = file('pc-components.db') + pc = Computer(**s['device'], components=OrderedSet(Component(**c) for c in s['components'])) + db.session.add(pc) + db.session.commit() + + # Create a new transient non-db synced object + pc = Computer(**s['device']) + db_pc, _ = Sync().run(pc, components=OrderedSet()) + assert not db_pc.components + assert not pc.components + + +@pytest.mark.usefixtures('app_context') +def test_sync_run_components_none(): + """ + Syncs a device that has a None components. The system should + keep all the components from the device. + """ + s = file('pc-components.db') + pc = Computer(**s['device'], components=OrderedSet(Component(**c) for c in s['components'])) + db.session.add(pc) + db.session.commit() + + # Create a new transient non-db synced object + transient_pc = Computer(**s['device']) + db_pc, _ = Sync().run(transient_pc, components=None) + assert db_pc.components + assert db_pc.components == pc.components + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_computer_new_computer_no_tag(): + """ + Syncs a new computer with HID and without a tag, creating it. + :return: + """ # Case 1: device does not exist on DB pc = Computer(**file('pc-components.db')['device']) - db_pc, _ = Sync.execute_register(pc, set()) + db_pc = Sync().execute_register(pc) assert pc.physical_properties == db_pc.physical_properties @pytest.mark.usefixtures('app_context') -def test_execute_register_computer_existing(): +def test_sync_execute_register_computer_existing_no_tag(): + """ + Syncs an existing computer with HID and without a tag. + """ pc = Computer(**file('pc-components.db')['device']) db.session.add(pc) - db.session.commit() # We need two separate sessions - pc = Computer(**file('pc-components.db')['device']) + db.session.commit() + + pc = Computer(**file('pc-components.db')['device']) # Create a new transient non-db object # 1: device exists on DB - db_pc, _ = Sync.execute_register(pc, set()) + db_pc = Sync().execute_register(pc) assert pc.physical_properties == db_pc.physical_properties @pytest.mark.usefixtures('app_context') -def test_execute_register_computer_no_hid(): +def test_sync_execute_register_computer_no_hid_no_tag(): + """ + Syncs a computer without HID and no tag. + + This should fail as we don't have a way to identify it. + """ pc = Computer(**file('pc-components.db')['device']) # 1: device has no HID pc.hid = pc.model = None with pytest.raises(NeedsId): - Sync.execute_register(pc, set()) + Sync().execute_register(pc) + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_computer_tag_not_linked(): + """ + Syncs a new computer with HID and a non-linked tag. + + It is OK if the tag was not linked, it will be linked in this process. + """ + tag = Tag(id='FOO') + db.session.add(tag) + db.session.commit() + + # Create a new transient non-db object + pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([Tag(id='FOO')])) + returned_pc = Sync().execute_register(pc) + assert returned_pc == pc + assert tag.device == pc, 'Tag has to be linked' + assert Computer.query.one() == pc, 'Computer had to be set to db' + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_no_hid_tag_not_linked(tag_id: str): + """ + Validates registering a computer without HID and a non-linked tag. + + In this case it is ok still, as the non-linked tag proves that + the computer was not existing before (otherwise the tag would + be linked), and thus it creates a new computer. + """ + tag = Tag(id=tag_id) + pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([tag])) + returned_pc = Sync().execute_register(pc) + db.session.commit() + assert returned_pc == pc + db_tag = next(iter(returned_pc.tags)) + # they are not the same tags though + # tag is a transient obj and db_tag the one from the db + # they have the same pk though + assert tag != db_tag, 'They are not the same tags though' + assert db_tag.id == tag.id + assert Computer.query.one() == pc, 'Computer had to be set to db' + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_tag_does_not_exist(): + """ + Ensures not being able to register if the tag does not exist, + even if the device has HID or it existed before. + + Tags have to be created before trying to link them through a Snapshot. + """ + pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([Tag()])) + with raises(ResourceNotFound): + Sync().execute_register(pc) + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_tag_linked_same_device(): + """ + If the tag is linked to the device, regardless if it has HID, + the system should match the device through the tag. + (If it has HID it validates both HID and tag point at the same + device, this his checked in ). + """ + orig_pc = Computer(**file('pc-components.db')['device']) + db.session.add(Tag(id='foo', device=orig_pc)) + db.session.commit() + + pc = Computer(**file('pc-components.db')['device']) # Create a new transient non-db object + pc.tags.add(Tag(id='foo')) + db_pc = Sync().execute_register(pc) + assert db_pc.id == orig_pc.id + assert len(db_pc.tags) == 1 + assert next(iter(db_pc.tags)).id == 'foo' + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_tag_linked_other_device_mismatch_between_tags(): + """ + Checks that sync raises an error if finds that at least two passed-in + tags are not linked to the same device. + """ + pc1 = Computer(**file('pc-components.db')['device']) + db.session.add(Tag(id='foo-1', device=pc1)) + pc2 = Computer(**file('pc-components.db')['device']) + pc2.serial_number = 'pc2-serial' + pc2.hid = Naming.hid(pc2.manufacturer, pc2.serial_number, pc2.model) + db.session.add(Tag(id='foo-2', device=pc2)) + db.session.commit() + + pc1 = Computer(**file('pc-components.db')['device']) # Create a new transient non-db object + pc1.tags.add(Tag(id='foo-1')) + pc1.tags.add(Tag(id='foo-2')) + with raises(MismatchBetweenTags): + Sync().execute_register(pc1) + + +@pytest.mark.usefixtures('app_context') +def test_sync_execute_register_mismatch_between_tags_and_hid(): + """ + Checks that sync raises an error if it finds that the HID does + not point at the same device as the tag does. + + In this case we set HID -> pc1 but tag -> pc2 + """ + pc1 = Computer(**file('pc-components.db')['device']) + db.session.add(Tag(id='foo-1', device=pc1)) + pc2 = Computer(**file('pc-components.db')['device']) + pc2.serial_number = 'pc2-serial' + pc2.hid = Naming.hid(pc2.manufacturer, pc2.serial_number, pc2.model) + db.session.add(Tag(id='foo-2', device=pc2)) + db.session.commit() + + pc1 = Computer(**file('pc-components.db')['device']) # Create a new transient non-db object + pc1.tags.add(Tag(id='foo-2')) + with raises(MismatchBetweenTagsAndHid): + Sync().execute_register(pc1) def test_get_device(app: Devicehub, user: UserClient): """Checks GETting a Desktop with its components.""" with app.app_context(): pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s') - pc.components = [ + pc.components = OrderedSet([ NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'), GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500) - ] + ]) db.session.add(pc) db.session.add(Test(device=pc, elapsed=timedelta(seconds=4), @@ -209,10 +384,10 @@ def test_get_devices(app: Devicehub, user: UserClient): """Checks GETting multiple devices.""" with app.app_context(): pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s') - pc.components = [ + pc.components = OrderedSet([ NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'), GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500) - ] + ]) pc1 = Microtower(model='p2mo', manufacturer='p2ma', serial_number='p2s') pc2 = Laptop(model='p3mo', manufacturer='p3ma', serial_number='p3s') db.session.add_all((pc, pc1, pc2)) diff --git a/tests/test_organization.py b/tests/test_organization.py new file mode 100644 index 00000000..c5506ff6 --- /dev/null +++ b/tests/test_organization.py @@ -0,0 +1,16 @@ +import pytest + +from ereuse_devicehub.config import DevicehubConfig +from ereuse_devicehub.resources.user import Organization + + +@pytest.mark.usefixtures('app_context') +def test_default_org_exists(config: DevicehubConfig): + """ + Ensures that the default organization is created on app + initialization and that is accessible for the method + :meth:`ereuse_devicehub.resources.user.Organization.get_default_org`. + """ + assert Organization.query.filter_by(name=config.ORGANIZATION_NAME, + tax_id=config.ORGANIZATION_TAX_ID).one() + assert Organization.get_default_org().name == config.ORGANIZATION_NAME diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index c10adede..c953d371 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -9,8 +9,10 @@ from ereuse_devicehub.db import db from ereuse_devicehub.devicehub import Devicehub from ereuse_devicehub.resources.device.exceptions import NeedsId from ereuse_devicehub.resources.device.models import Device, Microtower +from ereuse_devicehub.resources.device.sync import MismatchBetweenTagsAndHid from ereuse_devicehub.resources.event.models import Appearance, Bios, Event, Functionality, \ Snapshot, SnapshotRequest, SoftwareType +from ereuse_devicehub.resources.tag import Tag from ereuse_devicehub.resources.user.models import User from tests.conftest import file @@ -249,3 +251,31 @@ def _test_snapshot_computer_no_hid(user: UserClient): user.post(s, res=Device) s['device']['id'] = 1 # Assign the ID of the placeholder user.post(s, res=Snapshot) + + +def test_snapshot_mismatch_id(): + """Tests uploading a device with an ID from another device.""" + # Note that this won't happen as in this new version + # the ID is not used in the Snapshot process + pass + + +def test_snapshot_tag_inner_tag(tag_id: str, user: UserClient, app: Devicehub): + """Tests a posting Snapshot with a local tag.""" + b = file('basic.snapshot') + b['device']['tags'] = [{'type': 'Tag', 'id': tag_id}] + snapshot_and_check(user, b) + with app.app_context(): + tag, *_ = Tag.query.all() # type: Tag + assert tag.device_id == 1, 'Tag should be linked to the first device' + + +def test_snapshot_tag_inner_tag_mismatch_between_tags_and_hid(user: UserClient, tag_id: str): + """Ensures one device cannot 'steal' the tag from another one.""" + pc1 = file('basic.snapshot') + pc1['device']['tags'] = [{'type': 'Tag', 'id': tag_id}] + user.post(pc1, res=Snapshot) + pc2 = file('1-device-with-components.snapshot') + user.post(pc2, res=Snapshot) # PC2 uploads well + pc2['device']['tags'] = [{'type': 'Tag', 'id': tag_id}] # Set tag from pc1 to pc2 + user.post(pc2, res=Snapshot, status=MismatchBetweenTagsAndHid) diff --git a/tests/test_tag.py b/tests/test_tag.py new file mode 100644 index 00000000..eb7c119c --- /dev/null +++ b/tests/test_tag.py @@ -0,0 +1,121 @@ +import pytest +from pytest import raises +from sqlalchemy.exc import IntegrityError + +from ereuse_devicehub.client import UserClient +from ereuse_devicehub.db import db +from ereuse_devicehub.devicehub import Devicehub +from ereuse_devicehub.resources.device.models import Computer +from ereuse_devicehub.resources.tag import Tag +from ereuse_devicehub.resources.tag.view import CannotCreateETag, TagNotLinked +from ereuse_devicehub.resources.user import Organization +from teal.db import MultipleResourcesFound, ResourceNotFound +from teal.marshmallow import ValidationError + + +@pytest.mark.usefixtures('app_context') +def test_create_tag(): + """Creates a tag specifying a custom organization.""" + org = Organization(name='Bar', tax_id='BarTax') + tag = Tag(id='bar-1', org=org) + db.session.add(tag) + db.session.commit() + + +@pytest.mark.usefixtures('app_context') +def test_create_tag_default_org(): + """Creates a tag using the default organization.""" + tag = Tag(id='foo-1') + assert not tag.org_id, 'org-id is set as default value so it should only load on flush' + # We don't want the organization to load, or it would make this + # object, from transient to new (added to session) + assert 'org' not in vars(tag), 'Organization should not have been loaded' + db.session.add(tag) + db.session.commit() + assert tag.org.name == 'FooOrg' # as defined in the settings + + +@pytest.mark.usefixtures('app_context') +def test_create_tag_no_slash(): + """Checks that no tags can be created that contain a slash.""" + with raises(ValidationError): + Tag(id='/') + + +@pytest.mark.usefixtures('app_context') +def test_create_two_same_tags(): + """Ensures there cannot be two tags with the same ID and organization.""" + db.session.add(Tag(id='foo-bar')) + db.session.add(Tag(id='foo-bar')) + with raises(IntegrityError): + db.session.commit() + db.session.rollback() + # And it works if tags are in different organizations + db.session.add(Tag(id='foo-bar')) + org2 = Organization(name='org 2', tax_id='tax id org 2') + db.session.add(Tag(id='foo-bar', org=org2)) + db.session.commit() + + +def test_tag_post(app: Devicehub, user: UserClient): + """Checks the POST method of creating a tag.""" + user.post(res=Tag, query=[('ids', 'foo')], data={}) + with app.app_context(): + assert Tag.query.filter_by(id='foo').one() + + +def test_tag_post_etag(user: UserClient): + """ + Ensures users cannot create eReuse.org tags through POST; + only terminal. + """ + user.post(res=Tag, query=[('ids', 'FO-123456')], data={}, status=CannotCreateETag) + # Although similar, these are not eTags and should pass + user.post(res=Tag, query=[ + ('ids', 'FO-0123-45'), + ('ids', 'FOO012345678910'), + ('ids', 'FO'), + ('ids', 'FO-'), + ('ids', 'FO-123'), + ('ids', 'FOO-123456') + ], data={}) + + +def test_tag_get_device_from_tag_endpoint(app: Devicehub, user: UserClient): + """Checks getting a linked device from a tag endpoint""" + with app.app_context(): + # Create a pc with a tag + tag = Tag(id='foo-bar') + pc = Computer(serial_number='sn1') + pc.tags.add(tag) + db.session.add(pc) + db.session.commit() + computer, _ = user.get(res=Tag, item='foo-bar/device') + assert computer['serialNumber'] == 'sn1' + + +def test_tag_get_device_from_tag_endpoint_no_linked(app: Devicehub, user: UserClient): + """As above, but when the tag is not linked.""" + with app.app_context(): + db.session.add(Tag(id='foo-bar')) + db.session.commit() + user.get(res=Tag, item='foo-bar/device', status=TagNotLinked) + + +def test_tag_get_device_from_tag_endpoint_no_tag(user: UserClient): + """As above, but when there is no tag with such ID.""" + user.get(res=Tag, item='foo-bar/device', status=ResourceNotFound) + + +def test_tag_get_device_from_tag_endpoint_multiple_tags(app: Devicehub, user: UserClient): + """ + As above, but when there are two tags with the same ID, the + system should not return any of both (to be deterministic) so + it should raise an exception. + """ + with app.app_context(): + db.session.add(Tag(id='foo-bar')) + org2 = Organization(name='org 2', tax_id='tax id org 2') + db.session.add(Tag(id='foo-bar', org=org2)) + db.session.commit() + user.get(res=Tag, item='foo-bar/device', status=MultipleResourcesFound) diff --git a/tests/test_user.py b/tests/test_user.py index 966424fa..a27415ef 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -34,10 +34,11 @@ def test_create_user_email_insensitive(app: Devicehub): with app.app_context(): user = User(email='FOO@foo.com') db.session.add(user) + db.session.commit() # We search in case insensitive manner u1 = User.query.filter_by(email='foo@foo.com').one() assert u1 == user - assert u1.email == 'FOO@foo.com' + assert u1.email == 'foo@foo.com' def test_hash_password(app: Devicehub):