Add Tag support; sync with tags; use SQLAlchemy's collection_class; set autoflush to false

docs/tags.rst

@@ -0,0 +1,105 @@
+Tags
+====
+Devicehub can generate tags, which are synthetic identifiers that
+identify a device in an organization. A tag has minimally two fields:
+the ID and the Registration Number of the organization that generated
+such ID.
+
+In Devicehub tags are created empty, this is without any device
+associated, and they are associated or **linked** when they are assigned
+to a device. In Devicehub you usually use the AndroidApp to link
+tags and devices.
+
+The organization that created the tag in the Devicehub (which can be
+impersonating the organization that generated the ID) is called the
+**tag provider**. This is usual when dealing with other organizations
+devices.
+
+A device can have many tags but a tag can only be linked to one device.
+As for the actual implementation, you cannot unlink them.
+
+Devicehub users can design, generate and print tags, manually setting
+an ID and an tag provider. Future Devicehub versions can allow
+parametrizing an ID generator.
+
+Note that these virtual tags don't have to forcefully be printed or
+have a physical representation (this is not imposed at system level).
+
+The eReuse.org tags (eTag)
+--------------------------
+We recognize a special type of tag, the **eReuse.org tags (eTag)**.
+These are tags defined by eReuse.org and that can be issued only
+by tag providers that comply with the eReuse.org requisites.
+
+The eTags are designed to empower device exchange between
+organizations and identification efficiency. They are built with durable
+plastic and have a QR code, NFC chip and a written ID.
+
+These tags live in separate databases from Devicehubs, empowered by
+the `eReuse.org Tag <https://github.com/ereuse/tag>`_. By using this
+software, eReuse.org certified tag providers can create and manage
+the tags, and send them to Devicehubs of their choice.
+
+Tag ID design
+~~~~~~~~~~~~~
+The eTag has a fixed schema for its ID: ``XXX-YYYYYYYYYYYYYY``, where:
+
+- *XX* is the **eReuse.org Tag Provider ID (eTagPId)**.
+- *YYYYYYYYYYYY* is the ID of the tag in the provider..
+
+The eTagPid identifies an official eReuse.org Tag provider; this ID
+is managed by eReuse.org in a public repository. eTagPIds are made of
+2 capital letters and numbers.
+
+The ID of the tag in the provider (*YYYYYYYYYYYYYY*) consists from
+5 to 10 capital letters and numbers (registering a maximum of 10^12
+tags).
+
+As an example, ``FO-A4CZ2`` is a tag from the ``FO`` tag provider
+and ID ``A4CZ2``.
+
+Creating tags
+-------------
+You need to create a tag before linking it to a device. There are
+two ways of creating a tag:
+
+- By performing ``POST /tags?ids=...`` and passing a list of tag IDs
+  to create. All users can create tags this method, however they
+  cannot create eTags. Get more info at the endpoint docs.
+- By executing in a terminal ``flask create-tags <ids>`` and passing
+  a list of IDs to create. Only an admin is supposed to use this method,
+  which allows them to create eTags. Get more info with
+  ``flask create-tags --help``.
+
+Note that tags cannot have a slash ``/``.
+
+Linking a tag
+-------------
+Linking a tag is joining the tag with the device.
+
+In Devicehub this process is done when performing a Snapshot (POST
+Snapshot), by setting tag ids in ``snapshot['device']['tags']``. Future
+implementation will allow setting to the organization to ensure
+tags are inequivocally correct.
+
+Note that tags must exist in the database prior this.
+
+You can only link once, and consecutive Snapshots that have the same
+tag will validate that the link is correct –so it is good praxis to
+try to always provide the tag when performing a Snapshot. Tags help
+too in finding devices when these don't generate a ``HID``. Find more
+in the ``Snapshot`` docs.
+
+Getting a device through its tag
+--------------------------------
+When performing ``GET /tags/<tag-id>/device`` you will get directly the
+device of such tag, as long as there are not two tags with the same
+tag-id. In such case you should use ``GET /tags/<ngo>/<tag-id>/device``
+to inequivocally get the correct device (to develop).
+
+Tags and migrations
+-------------------
+Tags travel with the devices they are linked when migrating them. Future
+implementations can parameterize this.
+
+http://t.devicetag.io/TG-1234567890

ereuse_devicehub/client.py

@@ -1,4 +1,4 @@
-from typing import Type, Union
+from typing import Any, Iterable, Tuple, Type, Union
 
 from boltons.typeutils import issubclass
 from ereuse_utils.test import JSON
@@ -23,7 +23,7 @@ class Client(TealClient):
              uri: str,
              res: str or Type[Thing] = None,
              status: Union[int, Type[HTTPException], Type[ValidationError]] = 200,
-             query: dict = {},
+             query: Iterable[Tuple[str, Any]] = tuple(),
              accept=JSON,
              content_type=JSON,
              item=None,
@@ -38,7 +38,7 @@ class Client(TealClient):
     def get(self,
             uri: str = '',
             res: Union[Type[Thing], str] = None,
-            query: dict = {},
+            query: Iterable[Tuple[str, Any]] = tuple(),
             status: Union[int, Type[HTTPException], Type[ValidationError]] = 200,
             item: Union[int, str] = None,
             accept: str = JSON,
@@ -51,7 +51,7 @@ class Client(TealClient):
              data: str or dict,
              uri: str = '',
              res: Union[Type[Thing], str] = None,
-             query: dict = {},
+             query: Iterable[Tuple[str, Any]] = tuple(),
              status: Union[int, Type[HTTPException], Type[ValidationError]] = 201,
              content_type: str = JSON,
              accept: str = JSON,
@@ -89,7 +89,7 @@ class UserClient(Client):
              uri: str,
              res: str = None,
              status: int or HTTPException = 200,
-             query: dict = {},
+             query: Iterable[Tuple[str, Any]] = tuple(),
              accept=JSON,
              content_type=JSON,
              item=None,

ereuse_devicehub/config.py

@@ -5,7 +5,8 @@ from ereuse_devicehub.resources.device import ComponentDef, ComputerDef, Desktop
     NetworkAdapterDef, ProcessorDef, RamModuleDef, ServerDef
 from ereuse_devicehub.resources.event import AddDef, EventDef, RemoveDef, SnapshotDef, TestDef, \
     TestHardDriveDef
-from ereuse_devicehub.resources.user import UserDef
+from ereuse_devicehub.resources.tag import TagDef
+from ereuse_devicehub.resources.user import OrganizationDef, UserDef
 from teal.config import Config
 
 
@@ -13,9 +14,25 @@ class DevicehubConfig(Config):
     RESOURCE_DEFINITIONS = (
         DeviceDef, ComputerDef, DesktopDef, LaptopDef, NetbookDef, ServerDef,
         MicrotowerDef, ComponentDef, GraphicCardDef, HardDriveDef, MotherboardDef,
-        NetworkAdapterDef, RamModuleDef, ProcessorDef, UserDef, EventDef, AddDef, RemoveDef,
+        NetworkAdapterDef, RamModuleDef, ProcessorDef, UserDef, OrganizationDef, TagDef, EventDef,
-        SnapshotDef, TestDef, TestHardDriveDef
+        AddDef, RemoveDef, SnapshotDef, TestDef, TestHardDriveDef
     )
     PASSWORD_SCHEMES = {'pbkdf2_sha256'}
     SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/dh-db1'
     MIN_WORKBENCH = StrictVersion('11.0')
+    """
+    The minimum version of eReuse.org Workbench that this Devicehub
+    accepts. We recommend not changing this value.
+    """
+    ORGANIZATION_NAME = None  # type: str
+    ORGANIZATION_TAX_ID = None  # type: str
+    """
+    The organization using this Devicehub.
+    
+    It is used by default, for example, when creating tags.
+    """
+
+    def __init__(self, db: str = None) -> None:
+        if not self.ORGANIZATION_NAME or not self.ORGANIZATION_TAX_ID:
+            raise ValueError('You need to set the main organization parameters.')
+        super().__init__(db)

ereuse_devicehub/db.py

@@ -1,3 +1,3 @@
 from teal.db import SQLAlchemy
 
-db = SQLAlchemy()
+db = SQLAlchemy(session_options={"autoflush": False})

ereuse_devicehub/marshmallow.py

@@ -8,6 +8,7 @@ from teal.marshmallow import NestedOn as TealNestedOn
 class NestedOn(TealNestedOn):
     __doc__ = TealNestedOn.__doc__
 
-    def __init__(self, nested, polymorphic_on='type', db: SQLAlchemy = db, default=missing_,
+    def __init__(self, nested, polymorphic_on='type', db: SQLAlchemy = db, collection_class=list,
-                 exclude=tuple(), only=None, **kwargs):
+                 default=missing_, exclude=tuple(), only=None, **kwargs):
-        super().__init__(nested, polymorphic_on, db, default, exclude, only, **kwargs)
+        super().__init__(nested, polymorphic_on, db, collection_class, default, exclude, only,
+                         **kwargs)

ereuse_devicehub/resources/device/exceptions.py

@@ -12,3 +12,16 @@ class NeedsId(ValidationError):
     def __init__(self):
         message = 'We couldn\'t get an ID for this device. Is this a custom PC?'
         super().__init__(message)
+
+
+class DeviceIsInAnotherDevicehub(ValidationError):
+    def __init__(self,
+                 tag_id,
+                 message=None,
+                 field_names=None,
+                 fields=None,
+                 data=None,
+                 valid_data=None,
+                 **kwargs):
+        message = message or 'Device {} is from another Devicehub.'.format(tag_id)
+        super().__init__(message, field_names, fields, data, valid_data, **kwargs)

ereuse_devicehub/resources/device/models.py

@@ -1,4 +1,5 @@
 from contextlib import suppress
+from itertools import chain
 from operator import attrgetter
 from typing import Dict, Set
 
@@ -7,10 +8,10 @@ from sqlalchemy import BigInteger, Column, Float, ForeignKey, Integer, Sequence,
     Unicode, inspect
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import ColumnProperty, backref, relationship
+from sqlalchemy.util import OrderedSet
 
-from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing, \
+from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing
-    check_range
+from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound, check_range
-from teal.db import CASCADE, POLYMORPHIC_ID, POLYMORPHIC_ON, ResourceNotFound
 
 
 class Device(Thing):
@@ -32,10 +33,7 @@ class Device(Thing):
     @property
     def events(self) -> list:
         """All the events performed to the device."""
-        # Tried to use chain() but Marshmallow doesn't like it :-(
+        return sorted(chain(self.events_multiple, self.events_one), key=attrgetter('id'))
-        events = self.events_multiple + self.events_one
-        events.sort(key=attrgetter('id'))
-        return events
 
     def __init__(self, *args, **kw) -> None:
         super().__init__(*args, **kw)
@@ -107,13 +105,14 @@ class Microtower(Computer):
 class Component(Device):
     id = Column(BigInteger, ForeignKey(Device.id), primary_key=True)  # type: int
 
-    parent_id = Column(BigInteger, ForeignKey('computer.id'))
+    parent_id = Column(BigInteger, ForeignKey(Computer.id))
     parent = relationship(Computer,
                           backref=backref('components',
                                           lazy=True,
                                           cascade=CASCADE,
-                                          order_by=lambda: Component.id),
+                                          order_by=lambda: Component.id,
-                          primaryjoin='Component.parent_id == Computer.id')  # type: Device
+                                          collection_class=OrderedSet),
+                          primaryjoin=parent_id == Computer.id)  # type: Device
 
     def similar_one(self, parent: Computer, blacklist: Set[int]) -> 'Component':
         """
@@ -136,10 +135,7 @@ class Component(Device):
 
     @property
     def events(self) -> list:
-        events = super().events
+        return sorted(chain(super().events, self.events_components), key=attrgetter('id'))
-        events.extend(self.events_components)
-        events.sort(key=attrgetter('id'))
-        return events
 
 
 class JoinedComponentTableMixin:

ereuse_devicehub/resources/device/schemas.py

@@ -1,5 +1,6 @@
 from marshmallow.fields import Float, Integer, Str
 from marshmallow.validate import Length, OneOf, Range
+from sqlalchemy.util import OrderedSet
 
 from ereuse_devicehub.marshmallow import NestedOn
 from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE
@@ -12,6 +13,7 @@ class Device(Thing):
     hid = Str(dump_only=True,
               description='The Hardware ID is the unique ID traceability systems '
                           'use to ID a device globally.')
+    tags = NestedOn('Tag', many=True, collection_class=OrderedSet)
     pid = Str(description='The PID identifies a device under a circuit or platform.',
               validate=Length(max=STR_SIZE))
     gid = Str(description='The Giver ID links the device to the giver\'s (donor, seller)'
@@ -34,7 +36,7 @@ class Device(Thing):
 
 
 class Computer(Device):
-    components = NestedOn('Component', many=True, dump_only=True)
+    components = NestedOn('Component', many=True, dump_only=True, collection_class=OrderedSet)
     pass
 
 

ereuse_devicehub/resources/device/sync.py

@@ -1,140 +1,194 @@
-import re
 from contextlib import suppress
 from itertools import groupby
-from typing import Iterable, List, Set
+from typing import Iterable, Set
 
-from psycopg2.errorcodes import UNIQUE_VIOLATION
+from sqlalchemy import inspect
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.util import OrderedSet
 
 from ereuse_devicehub.db import db
 from ereuse_devicehub.resources.device.exceptions import NeedsId
 from ereuse_devicehub.resources.device.models import Component, Computer, Device
-from ereuse_devicehub.resources.event.models import Add, Remove
+from ereuse_devicehub.resources.event.models import Remove
+from ereuse_devicehub.resources.tag.model import Tag
 from teal.db import ResourceNotFound
+from teal.marshmallow import ValidationError
 
 
 class Sync:
     """Synchronizes the device and components with the database."""
 
-    @classmethod
+    def run(self,
-    def run(cls, device: Device,
+            device: Device,
-            components: Iterable[Component] or None) -> (Device, List[Add or Remove]):
+            components: Iterable[Component] or None) -> (Device, OrderedSet):
         """
         Synchronizes the device and components with the database.
 
         Identifies if the device and components exist in the database
         and updates / inserts them as necessary.
 
+        Passed-in parameters have to be transient, or said differently,
+        not-db-synced objects, or otherwise they would end-up being
+        added in the session. `Learn more... <http://docs.sqlalchemy.org/
+        en/latest/orm/session_state_management.html#quickie-intro-to
+        -object-states>`_.
+
         This performs Add / Remove as necessary.
+
         :param device: The device to add / update to the database.
         :param components: Components that are inside of the device.
                            This method performs Add and Remove events
                            so the device ends up with these components.
                            Components are added / updated accordingly.
                            If this is empty, all components are removed.
-                           If this is None, it means that there is
+                           If this is None, it means that we are not
-                           no info about components and the already
+                           providing info about the components, in which
-                           existing components of the device (in case
+                           case we keep the already existing components
-                           the device already exists) won't be touch.
+                           of the device –we don't touch them.
         :return: A tuple of:
+
                  1. The device from the database (with an ID) whose
                     ``components`` field contain the db version
                     of the passed-in components.
                  2. A list of Add / Remove (not yet added to session).
         """
-        db_device, _ = cls.execute_register(device)
+        db_device = self.execute_register(device)
-        db_components, events = [], []
+        db_components, events = OrderedSet(), OrderedSet()
         if components is not None:  # We have component info (see above)
             blacklist = set()  # type: Set[int]
             not_new_components = set()
             for component in components:
-                db_component, is_new = cls.execute_register(component, blacklist, parent=db_device)
+                db_component, is_new = self.execute_register_component(component,
-                db_components.append(db_component)
+                                                                       blacklist,
+                                                                       parent=db_device)
+                db_components.add(db_component)
                 if not is_new:
                     not_new_components.add(db_component)
             # We only want to perform Add/Remove to not new components
-            events = cls.add_remove(db_device, not_new_components)
+            events = self.add_remove(db_device, not_new_components)
             db_device.components = db_components
         return db_device, events
 
-    @classmethod
+    def execute_register_component(self,
-    def execute_register(cls, device: Device,
+                                   component: Component,
-                         blacklist: Set[int] = None,
+                                   blacklist: Set[int],
-                         parent: Computer = None) -> (Device, bool):
+                                   parent: Computer):
         """
-        Synchronizes one device to the DB.
+        Synchronizes one component to the DB.
 
-        This method tries to create a device into the database, and
+        This method is a specialization of :meth:`.execute_register`
-        if it already exists it returns a "local synced version",
+        but for components that are inside parents.
-        this is the same ``device`` you passed-in but with updated
-        values from the database one (like the id value).
 
-        When we say "local" we mean that if, the device existed on the
+        This method assumes components don't have tags, and it tries
-        database, we do not "touch" any of its values on the DB.
+        to identify a non-hid component by finding a
+        :meth:`ereuse_devicehub.resources.device.models.Component.
+        similar_one`.
 
-        :param device: The device to synchronize to the DB.
+        :param component: The component to sync.
         :param blacklist: A set of components already found by
                           Component.similar_one(). Pass-in an empty Set.
         :param parent: For components, the computer that contains them.
                        Helper used by Component.similar_one().
         :return: A tuple with:
-                 1. A synchronized device with the DB. It can be a new
+                 - The synced component. See :meth:`.execute_register`
-                   device or an already existing one.
+                   for more info.
-                 2. A flag stating if the device is new or it existed
+                 - A flag stating if the device is new or it already
-                    already in the DB.
+                   existed in the DB.
-        :raise NeedsId: The device has not any identifier we can use.
-                        To still create the device use
-                        ``force_creation``.
-        :raise DatabaseError: Any other error from the DB.
         """
-        # Let's try to create the device
+        assert inspect(component).transient, 'Component should not be synced from DB'
-        if not device.hid and not device.id:
+        try:
-            # We won't be able to surely identify this device
+            if component.hid:
-            if isinstance(device, Component):
+                db_component = Device.query.filter_by(hid=component.hid).one()
-                with suppress(ResourceNotFound):
+            else:
                 # Is there a component similar to ours?
-                    db_component = device.similar_one(parent, blacklist)
+                db_component = component.similar_one(parent, blacklist)
                 # We blacklist this component so we
                 # ensure we don't get it again for another component
                 # with the same physical properties
                 blacklist.add(db_component.id)
-                    return cls.merge(device, db_component), False
+        except ResourceNotFound:
-            else:
+            db.session.add(component)
-                raise NeedsId()
-        try:
-            with db.session.begin_nested():
-                # Create transaction savepoint to auto-rollback on insertion err
-                # Let's try to insert or update
-                db.session.add(device)
             db.session.flush()
-        except IntegrityError as e:
+            db_component = component
-            if e.orig.diag.sqlstate == UNIQUE_VIOLATION:
+            is_new = True
-                db.session.rollback()
-                # This device already exists in the DB
-                field, value = (
-                    x.replace('(', '').replace(')', '')
-                    for x in re.findall('\(.*?\)', e.orig.diag.message_detail)
-                )
-                db_device = Device.query.filter_by(**{field: value}).one()  # type: Device
-                return cls.merge(device, db_device), False
         else:
-                raise e
+            self.merge(component, db_component)
-        else:
+            is_new = False
-            return device, True  # Our device is new
+        return db_component, is_new
 
-    @classmethod
+    def execute_register(self, device: Device) -> Device:
-    def merge(cls, device: Device, db_device: Device):
+        """
+        Synchronizes one device to the DB.
+
+        This method tries to get an existing device using the HID
+        or one of the tags, and...
+
+        - if it already exists it returns a "local synced version"
+          –the same ``device`` you passed-in but with updated values
+          from the database. In this case we do not
+          "touch" any of its values on the DB.
+        - If it did not exist, a new device is created in the db.
+
+        This method validates that all passed-in tags (``device.tags``),
+        if linked, are linked to the same device, ditto for the hid.
+        Finally it links the tags with the device.
+
+        If you pass-in a component that is inside a parent, use
+        :meth:`.execute_register_component` as it has more specialized
+        methods to handle them.
+
+        :param device: The device to synchronize to the DB.
+        :raise NeedsId: The device has not any identifier we can use.
+                        To still create the device use
+                        ``force_creation``.
+        :raise DatabaseError: Any other error from the DB.
+        :return: The synced device from the db with the tags linked.
+        """
+        assert inspect(device).transient, 'Device cannot be already synced from DB'
+        assert all(inspect(tag).transient for tag in device.tags), 'Tags cannot be synced from DB'
+        if not device.tags and not device.hid:
+            # We cannot identify this device
+            raise NeedsId()
+        db_device = None
+        if device.hid:
+            with suppress(ResourceNotFound):
+                db_device = Device.query.filter_by(hid=device.hid).one()
+        tags = {Tag.query.filter_by(id=tag.id).one() for tag in device.tags}  # type: Set[Tag]
+        linked_tags = {tag for tag in tags if tag.device_id}  # type: Set[Tag]
+        if linked_tags:
+            sample_tag = next(iter(linked_tags))
+            for tag in linked_tags:
+                if tag.device_id != sample_tag.device_id:
+                    raise MismatchBetweenTags(tag, sample_tag)  # Linked to different devices
+            if db_device:  # Device from hid
+                if sample_tag.device_id != db_device.id:  # Device from hid != device from tags
+                    raise MismatchBetweenTagsAndHid(db_device.id, db_device.hid)
+            else:  # There was no device from hid
+                db_device = sample_tag.device
+        if db_device:  # Device from hid or tags
+            self.merge(device, db_device)
+        else:  # Device is new and tags are not linked to a device
+            device.tags.clear()  # We don't want to add the transient dummy tags
+            db.session.add(device)
+            db_device = device
+        db_device.tags |= tags  # Union of tags the device had plus the (potentially) new ones
+        db.session.flush()
+        assert db_device is not None
+        return db_device
+
+    @staticmethod
+    def merge(device: Device, db_device: Device):
         """
         Copies the physical properties of the device to the db_device.
+
+        This method mutates db_device.
         """
         for field_name, value in device.physical_properties.items():
             if value is not None:
                 setattr(db_device, field_name, value)
-        return db_device
 
-    @classmethod
+    @staticmethod
-    def add_remove(cls, device: Device,
+    def add_remove(device: Device,
-                   components: Set[Component]) -> List[Add or Remove]:
+                   components: Set[Component]) -> OrderedSet:
         """
         Generates the Add and Remove events (but doesn't add them to
         session).
@@ -149,7 +203,7 @@ class Sync:
         :return: A list of Add / Remove events.
         """
         # Note that we create the Remove events before the Add ones
-        events = []
+        events = OrderedSet()
         old_components = set(device.components)
 
         adding = components - old_components
@@ -160,5 +214,24 @@ class Sync:
 
             for parent, _components in groupby(sorted(adding, key=g_parent), key=g_parent):
                 if parent.id != 0:  # Is not Computer Identity
-                    events.append(Remove(device=parent, components=list(_components)))
+                    events.add(Remove(device=parent, components=OrderedSet(_components)))
         return events
+
+
+class MismatchBetweenTags(ValidationError):
+    def __init__(self,
+                 tag: Tag,
+                 other_tag: Tag,
+                 field_names={'device.tags'}):
+        message = '{!r} and {!r} are linked to different devices.'.format(tag, other_tag)
+        super().__init__(message, field_names)
+
+
+class MismatchBetweenTagsAndHid(ValidationError):
+    def __init__(self,
+                 device_id: int,
+                 hid: str,
+                 field_names={'device.hid'}):
+        message = 'Tags are linked to device {} but hid refers to device {}.'.format(device_id,
+                                                                                     hid)
+        super().__init__(message, field_names)

ereuse_devicehub/resources/event/__init__.py

@@ -1,3 +1,6 @@
+from typing import Callable, Iterable, Tuple
+
+from ereuse_devicehub.resources.device.sync import Sync
 from ereuse_devicehub.resources.event.schemas import Add, Event, Remove, Snapshot, Test, \
     TestHardDrive
 from ereuse_devicehub.resources.event.views import EventView, SnapshotView
@@ -23,6 +26,13 @@ class SnapshotDef(EventDef):
     SCHEMA = Snapshot
     VIEW = SnapshotView
 
+    def __init__(self, app, import_name=__package__, static_folder=None, static_url_path=None,
+                 template_folder=None, url_prefix=None, subdomain=None, url_defaults=None,
+                 root_path=None, cli_commands: Iterable[Tuple[Callable, str or None]] = tuple()):
+        super().__init__(app, import_name, static_folder, static_url_path, template_folder,
+                         url_prefix, subdomain, url_defaults, root_path, cli_commands)
+        self.sync = Sync()
+
 
 class TestDef(EventDef):
     SCHEMA = Test

ereuse_devicehub/resources/event/models.py

@@ -7,17 +7,17 @@ from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, DateTime, E
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import backref, relationship, validates
+from sqlalchemy.util import OrderedSet
 from sqlalchemy_utils import ColorType
 
 from ereuse_devicehub.db import db
 from ereuse_devicehub.resources.device.models import Component, Device
 from ereuse_devicehub.resources.event.enums import Appearance, Bios, Functionality, Orientation, \
     SoftwareType, StepTypes, TestHardDriveLength
-from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing, \
+from ereuse_devicehub.resources.models import STR_BIG_SIZE, STR_SIZE, STR_SM_SIZE, Thing
-    check_range
 from ereuse_devicehub.resources.user.models import User
 from teal.db import CASCADE, CASCADE_OWN, INHERIT_COND, POLYMORPHIC_ID, POLYMORPHIC_ON, \
-    StrictVersionType
+    StrictVersionType, check_range
 
 
 class JoinedTableMixin:
@@ -39,7 +39,10 @@ class Event(Thing):
                                                 use_alter=True,
                                                 name='snapshot_events'))
     snapshot = relationship('Snapshot',
-                            backref=backref('events', lazy=True, cascade=CASCADE),
+                            backref=backref('events',
+                                            lazy=True,
+                                            cascade=CASCADE,
+                                            collection_class=OrderedSet),
                             primaryjoin='Event.snapshot_id == Snapshot.id')
 
     author_id = Column(UUID(as_uuid=True),
@@ -47,14 +50,16 @@ class Event(Thing):
                        nullable=False,
                        default=lambda: g.user.id)
     author = relationship(User,
-                          backref=backref('events', lazy=True),
+                          backref=backref('events', lazy=True, collection_class=set),
                           primaryjoin=author_id == User.id)
     components = relationship(Component,
                               backref=backref('events_components',
                                               lazy=True,
-                                              order_by=lambda: Event.id),
+                                              order_by=lambda: Event.id,
+                                              collection_class=OrderedSet),
                               secondary=lambda: EventComponent.__table__,
-                              order_by=lambda: Device.id)
+                              order_by=lambda: Device.id,
+                              collection_class=OrderedSet)
 
     @declared_attr
     def __mapper_args__(cls):
@@ -84,7 +89,8 @@ class EventWithOneDevice(Event):
                           backref=backref('events_one',
                                           lazy=True,
                                           cascade=CASCADE,
-                                          order_by=lambda: EventWithOneDevice.id),
+                                          order_by=lambda: EventWithOneDevice.id,
+                                          collection_class=OrderedSet),
                           primaryjoin=Device.id == device_id)
 
     def __repr__(self) -> str:
@@ -98,7 +104,8 @@ class EventWithMultipleDevices(Event):
     devices = relationship(Device,
                            backref=backref('events_multiple',
                                            lazy=True,
-                                           order_by=lambda: EventWithMultipleDevices.id),
+                                           order_by=lambda: EventWithMultipleDevices.id,
+                                           collection_class=OrderedSet),
                            secondary=lambda: EventDevice.__table__,
                            order_by=lambda: Device.id)
 
@@ -193,7 +200,10 @@ class SnapshotRequest(db.Model):
     id = Column(BigInteger, ForeignKey(Snapshot.id), primary_key=True)
     request = Column(JSON, nullable=False)
 
-    snapshot = relationship(Snapshot, backref=backref('request', lazy=True, uselist=False,
+    snapshot = relationship(Snapshot,
+                            backref=backref('request',
+                                            lazy=True,
+                                            uselist=False,
                                             cascade=CASCADE_OWN))
 
 
@@ -201,7 +211,11 @@ class Test(JoinedTableMixin, EventWithOneDevice):
     elapsed = Column(Interval, nullable=False)
     success = Column(Boolean, nullable=False)
 
-    snapshot = relationship(Snapshot, backref=backref('tests', lazy=True, cascade=CASCADE_OWN))
+    snapshot = relationship(Snapshot, backref=backref('tests',
+                                                      lazy=True,
+                                                      cascade=CASCADE_OWN,
+                                                      order_by=Event.id,
+                                                      collection_class=OrderedSet))
 
 
 class TestHardDrive(Test):

ereuse_devicehub/resources/event/views.py

@@ -1,9 +1,10 @@
 from distutils.version import StrictVersion
 
 from flask import request
+from sqlalchemy.util import OrderedSet
 
 from ereuse_devicehub.db import db
-from ereuse_devicehub.resources.device.sync import Sync
+from ereuse_devicehub.resources.device.models import Device
 from ereuse_devicehub.resources.event.enums import SoftwareType
 from ereuse_devicehub.resources.event.models import Event, Snapshot, TestHardDrive
 from teal.resource import View
@@ -30,16 +31,15 @@ class SnapshotView(View):
         # Note that if we set the device / components into the snapshot
         # model object, when we flush them to the db we will flush
         # snapshot, and we want to wait to flush snapshot at the end
-        device = s.pop('device')
+        device = s.pop('device')  # type: Device
         components = s.pop('components') if s['software'] == SoftwareType.Workbench else None
         # noinspection PyArgumentList
         snapshot = Snapshot(**s)
-        snapshot.device, snapshot.events = Sync.run(device, components)
+        snapshot.device, snapshot.events = self.resource_def.sync.run(device, components)
         snapshot.components = snapshot.device.components
         # commit will change the order of the components by what
-        # the DB wants. Let's get a copy of the list so we preserve
+        # the DB wants. Let's get a copy of the list so we preserve order
-        # order
+        ordered_components = OrderedSet(x for x in snapshot.components)
-        ordered_components = [c for c in snapshot.components]
         db.session.add(snapshot)
         db.session.commit()
         # todo we are setting snapshot dirty again with this components but

ereuse_devicehub/resources/models.py

@@ -1,7 +1,5 @@
 from datetime import datetime
 
-from sqlalchemy import CheckConstraint
-
 from ereuse_devicehub.db import db
 
 STR_SIZE = 64
@@ -9,11 +7,6 @@ STR_BIG_SIZE = 128
 STR_SM_SIZE = 32
 
 
-def check_range(column: str, min=1, max=None) -> CheckConstraint:
-    constraint = '>= {}'.format(min) if max is None else 'BETWEEN {} AND {}'.format(min, max)
-    return CheckConstraint('{} {}'.format(column, constraint))
-
-
 class Thing(db.Model):
     __abstract__ = True
     updated = db.Column(db.DateTime, onupdate=datetime.utcnow)

ereuse_devicehub/resources/tag/__init__.py

@@ -0,0 +1,46 @@
+from typing import Tuple
+
+from click import argument, option
+
+from ereuse_devicehub.db import db
+from ereuse_devicehub.resources.tag.model import Tag
+from ereuse_devicehub.resources.tag.schema import Tag as TagS
+from ereuse_devicehub.resources.tag.view import TagView, get_device_from_tag
+from teal.resource import Resource
+from teal.teal import Teal
+
+
+class TagDef(Resource):
+    SCHEMA = TagS
+    VIEW = TagView
+
+    def __init__(self, app: Teal, import_name=__package__, static_folder=None,
+                 static_url_path=None,
+                 template_folder=None, url_prefix=None, subdomain=None, url_defaults=None,
+                 root_path=None):
+        cli_commands = (
+            (self.create_tags, 'create-tags'),
+        )
+        super().__init__(app, import_name, static_folder, static_url_path, template_folder,
+                         url_prefix, subdomain, url_defaults, root_path, cli_commands)
+        _get_device_from_tag = app.auth.requires_auth(get_device_from_tag)
+        self.add_url_rule('/<{}:{}>/device'.format(self.ID_CONVERTER.value, self.ID_NAME),
+                          view_func=_get_device_from_tag,
+                          methods={'GET'})
+
+    @option('--org',
+            help='The name of an existing organization in the DB. '
+                 'By default the organization operating this Devicehub.')
+    @option('--provider',
+            help='The Base URL of the provider. '
+                 'By default set to the actual Devicehub.')
+    @argument('ids', nargs=-1, required=True)
+    def create_tags(self, ids: Tuple[str], org: str = None, provider: str = None):
+        """Create TAGS and associates them to a specific PROVIDER."""
+        tag_schema = TagS(only=('id', 'provider', 'org'))
+
+        db.session.add_all(
+            Tag(**tag_schema.load({'id': tag_id, 'provider': provider, 'org': org}))
+            for tag_id in ids
+        )
+        db.session.commit()

ereuse_devicehub/resources/tag/model.py

@@ -0,0 +1,45 @@
+from sqlalchemy import BigInteger, Column, ForeignKey, Unicode, UniqueConstraint
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy.orm import backref, relationship, validates
+
+from ereuse_devicehub.resources.device.models import Device
+from ereuse_devicehub.resources.models import Thing
+from ereuse_devicehub.resources.user.models import Organization
+from teal.db import DB_CASCADE_SET_NULL, URL
+from teal.marshmallow import ValidationError
+
+
+class Tag(Thing):
+    id = Column(Unicode(), primary_key=True)
+    org_id = Column(UUID(as_uuid=True),
+                    ForeignKey(Organization.id),
+                    primary_key=True,
+                    # If we link with the Organization object this instance
+                    # will be set as persistent and added to session
+                    # which is something we don't want to enforce by default
+                    default=lambda: Organization.get_default_org().id)
+    org = relationship(Organization,
+                       backref=backref('tags', lazy=True),
+                       primaryjoin=Organization.id == org_id,
+                       collection_class=set)
+    provider = Column(URL(),
+                      comment='The tag provider URL. If None, the provider is this Devicehub.')
+    device_id = Column(BigInteger,
+                       # We don't want to delete the tag on device deletion, only set to null
+                       ForeignKey(Device.id, ondelete=DB_CASCADE_SET_NULL))
+    device = relationship(Device,
+                          backref=backref('tags', lazy=True, collection_class=set),
+                          primaryjoin=Device.id == device_id)
+
+    @validates('id')
+    def does_not_contain_slash(self, _, value: str):
+        if '/' in value:
+            raise ValidationError('Tags cannot contain slashes (/).')
+        return value
+
+    __table_args__ = (
+        UniqueConstraint(device_id, org_id, name='One tag per organization.'),
+    )
+
+    def __repr__(self) -> str:
+        return '<Tag {0.id} org:{0.org_id} device:{0.device_id}>'.format(self)

ereuse_devicehub/resources/tag/schema.py

@@ -0,0 +1,15 @@
+from marshmallow.fields import String
+
+from ereuse_devicehub.marshmallow import NestedOn
+from ereuse_devicehub.resources.device.schemas import Device
+from ereuse_devicehub.resources.schemas import Thing
+from teal.marshmallow import URL
+
+
+class Tag(Thing):
+    id = String(description='The ID of the tag.',
+                validator=lambda x: '/' not in x,
+                required=True)
+    provider = URL(description='The provider URL.')
+    device = NestedOn(Device, description='The device linked to this tag.')
+    org = String(description='The organization that issued the tag.')

ereuse_devicehub/resources/tag/view.py

@@ -0,0 +1,71 @@
+from flask import Response, current_app as app, request
+from marshmallow import Schema
+from marshmallow.fields import List, String, URL
+from webargs.flaskparser import parser
+
+from ereuse_devicehub.resources.device.models import Device
+from ereuse_devicehub.resources.tag import Tag
+from teal.marshmallow import ValidationError
+from teal.resource import View
+
+
+class TagView(View):
+    class PostArgs(Schema):
+        ids = List(String(), required=True, description='A list of tags identifiers.')
+        org = String(description='The name of an existing organization in the DB. '
+                                 'If not set, the default organization is used.')
+        provider = URL(description='The Base URL of the provider. By default is this Devicehub.')
+
+    post_args = PostArgs()
+
+    def post(self):
+        """
+        Creates tags.
+
+        ---
+        parameters:
+          - name: tags
+            in: path
+            description: Number of tags to create.
+        """
+        args = parser.parse(self.post_args, request, locations={'querystring'})
+        # Ensure user is not POSTing an eReuse.org tag
+        # For now only allow them to be created through command-line
+        for id in args['ids']:
+            try:
+                provider, _id = id.split('-')
+            except ValueError:
+                pass
+            else:
+                if len(provider) == 2 and 5 <= len(_id) <= 10:
+                    raise CannotCreateETag(id)
+        self.resource_def.create_tags(**args)
+        return Response(status=201)
+
+
+def get_device_from_tag(id: str):
+    """
+    Gets the device by passing a tag id.
+
+    Example: /tags/23/device.
+
+    :raise MultipleTagsPerId: More than one tag per Id. Please, use
+           the /tags/<organization>/<id>/device URL to disambiguate.
+    """
+    # todo this could be more efficient by Device.query... join with tag
+    device = Tag.query.filter_by(id=id).one().device
+    if device is None:
+        raise TagNotLinked(id)
+    return app.resources[Device.t].schema.jsonify(device)
+
+
+class CannotCreateETag(ValidationError):
+    def __init__(self, id: str):
+        message = 'Only sysadmin can create an eReuse.org Tag ({})'.format(id)
+        super().__init__(message)
+
+
+class TagNotLinked(ValidationError):
+    def __init__(self, id):
+        message = 'The tag {} is not linked to a device.'.format(id)
+        super().__init__(message, field_names=['device'])

ereuse_devicehub/resources/user/__init__.py

@@ -1,10 +1,12 @@
 from click import argument, option
+from flask import current_app as app
 
 from ereuse_devicehub import devicehub
 from ereuse_devicehub.db import db
-from ereuse_devicehub.resources.user.models import User
+from ereuse_devicehub.resources.user.models import Organization, User
 from ereuse_devicehub.resources.user.schemas import User as UserS
 from ereuse_devicehub.resources.user.views import UserView, login
+from teal.db import SQLAlchemy
 from teal.resource import Converters, Resource
 
 
@@ -17,7 +19,7 @@ class UserDef(Resource):
     def __init__(self, app: 'devicehub.Devicehub', import_name=__package__, static_folder=None,
                  static_url_path=None, template_folder=None, url_prefix=None, subdomain=None,
                  url_defaults=None, root_path=None):
-        cli_commands = ((self.create_user, 'user create'),)
+        cli_commands = ((self.create_user, 'create-user'),)
         super().__init__(app, import_name, static_folder, static_url_path, template_folder,
                          url_prefix, subdomain, url_defaults, root_path, cli_commands)
         self.add_url_rule('/login', view_func=login, methods={'POST'})
@@ -28,10 +30,40 @@ class UserDef(Resource):
         """
         Creates an user.
         """
-        with self.app.app_context():
+        u = self.SCHEMA(only={'email', 'password'}, exclude=('token',)) \
-            self.SCHEMA(only={'email', 'password'}, exclude=('token',)) \
             .load({'email': email, 'password': password})
-            user = User(email=email, password=password)
+        user = User(**u)
         db.session.add(user)
         db.session.commit()
         return self.schema.dump(user)
+
+
+class OrganizationDef(Resource):
+    __type__ = 'Organization'
+    ID_CONVERTER = Converters.uid
+    AUTH = True
+
+    def __init__(self, app, import_name=__package__, static_folder=None, static_url_path=None,
+                 template_folder=None, url_prefix=None, subdomain=None, url_defaults=None,
+                 root_path=None):
+        cli_commands = ((self.create_org, 'create-org'),)
+        super().__init__(app, import_name, static_folder, static_url_path, template_folder,
+                         url_prefix, subdomain, url_defaults, root_path, cli_commands)
+
+    @argument('name')
+    @argument('tax_id')
+    @argument('country')
+    def create_org(self, **kw: dict) -> dict:
+        """
+        Creates an organization.
+        COUNTRY has to be 2 characters as defined by
+        """
+        org = Organization(**self.schema.load(kw))
+        db.session.add(org)
+        db.session.commit()
+        return self.schema.dump(org)
+
+    def init_db(self, db: SQLAlchemy):
+        """Creates the default organization."""
+        org = Organization(**app.config.get_namespace('ORGANIZATION_'))
+        db.session.add(org)

ereuse_devicehub/resources/user/models.py

@@ -1,11 +1,11 @@
 from uuid import uuid4
 
-from flask import current_app
+from flask import current_app as app
-from sqlalchemy import Column, Unicode
+from sqlalchemy import Column, Unicode, UniqueConstraint
 from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy_utils import EmailType, PasswordType
+from sqlalchemy_utils import CountryType, EmailType, PasswordType
 
-from ereuse_devicehub.resources.models import STR_SIZE, Thing
+from ereuse_devicehub.resources.models import STR_SIZE, STR_SM_SIZE, Thing
 
 
 class User(Thing):
@@ -14,7 +14,7 @@ class User(Thing):
     email = Column(EmailType, nullable=False, unique=True)
     password = Column(PasswordType(max_length=STR_SIZE,
                                    onload=lambda **kwargs: dict(
-                                       schemes=current_app.config['PASSWORD_SCHEMES'],
+                                       schemes=app.config['PASSWORD_SCHEMES'],
                                        **kwargs
                                    )))
     """
@@ -26,4 +26,25 @@ class User(Thing):
     token = Column(UUID(as_uuid=True), default=uuid4, unique=True)
 
     def __repr__(self) -> str:
-        return '<{0.t} {0.id!r} email={0.email!r}>'.format(self)
+        return '<{0.t} {0.id} email={0.email}>'.format(self)
+
+
+class Organization(Thing):
+    id = Column(UUID(as_uuid=True), default=uuid4, primary_key=True)
+    name = Column(Unicode(length=STR_SM_SIZE), unique=True)
+    tax_id = Column(Unicode(length=STR_SM_SIZE),
+                    comment='The Tax / Fiscal ID of the organization, '
+                            'e.g. the TIN in the US or the CIF/NIF in Spain.')
+    country = Column(CountryType, comment='Country issuing the tax_id number.')
+
+    __table_args__ = (
+        UniqueConstraint(tax_id, country, name='Registration Number per country.'),
+    )
+
+    @classmethod
+    def get_default_org(cls) -> 'Organization':
+        """Retrieves the default organization."""
+        return Organization.query.filter_by(**app.config.get_namespace('ORGANIZATION_')).one()
+
+    def __repr__(self) -> str:
+        return '<Org {0.id}: {0.name}>'.format(self)

setup.py

@@ -14,11 +14,15 @@ setup(
         'marshmallow_enum',
         'ereuse-utils [Naming]',
         'psycopg2-binary',
-        'sqlalchemy-utils'
+        'sqlalchemy-utils',
+        'requests',
+        'requests-toolbelt',
+        'hashids'
     ],
     tests_requires=[
         'pytest',
-        'pytest-datadir'
+        'pytest-datadir',
+        'requests_mock'
     ],
     classifiers={
         'Development Status :: 4 - Beta',

tests/conftest.py

@@ -7,6 +7,7 @@ from ereuse_devicehub.client import Client, UserClient
 from ereuse_devicehub.config import DevicehubConfig
 from ereuse_devicehub.db import db
 from ereuse_devicehub.devicehub import Devicehub
+from ereuse_devicehub.resources.tag import Tag
 from ereuse_devicehub.resources.user.models import User
 
 
@@ -14,6 +15,8 @@ class TestConfig(DevicehubConfig):
     SQLALCHEMY_DATABASE_URI = 'postgresql://localhost/dh_test'
     SCHEMA = 'test'
     TESTING = True
+    ORGANIZATION_NAME = 'FooOrg'
+    ORGANIZATION_TAX_ID = 'FooOrgId'
 
 
 @pytest.fixture(scope='module')
@@ -28,7 +31,8 @@ def _app(config: TestConfig) -> Devicehub:
 
 @pytest.fixture()
 def app(request, _app: Devicehub) -> Devicehub:
-    db.create_all(app=_app)
+    with _app.app_context():
+        _app.init_db()
     # More robust than 'yield'
     request.addfinalizer(lambda *args, **kw: db.drop_all(app=_app))
     return _app
@@ -80,6 +84,16 @@ def auth_app_context(app: Devicehub):
 
 
 def file(name: str) -> dict:
-    """Opens and parses a JSON file from the ``files`` subdir."""
+    """Opens and parses a YAML file from the ``files`` subdir."""
     with Path(__file__).parent.joinpath('files').joinpath(name + '.yaml').open() as f:
         return yaml.load(f)
+
+
+@pytest.fixture()
+def tag_id(app: Devicehub) -> str:
+    """Creates a tag and returns its id."""
+    with app.app_context():
+        t = Tag(id='foo')
+        db.session.add(t)
+        db.session.commit()
+        return t.id

tests/test_device.py

@@ -2,6 +2,9 @@ from datetime import timedelta
 from uuid import UUID
 
 import pytest
+from ereuse_utils.naming import Naming
+from pytest import raises
+from sqlalchemy.util import OrderedSet
 
 from ereuse_devicehub.client import UserClient
 from ereuse_devicehub.db import db
@@ -10,8 +13,10 @@ from ereuse_devicehub.resources.device.exceptions import NeedsId
 from ereuse_devicehub.resources.device.models import Component, Computer, Desktop, Device, \
     GraphicCard, Laptop, Microtower, Motherboard, NetworkAdapter
 from ereuse_devicehub.resources.device.schemas import Device as DeviceS
-from ereuse_devicehub.resources.device.sync import Sync
+from ereuse_devicehub.resources.device.sync import MismatchBetweenTags, MismatchBetweenTagsAndHid, \
+    Sync
 from ereuse_devicehub.resources.event.models import Remove, Test
+from ereuse_devicehub.resources.tag.model import Tag
 from ereuse_devicehub.resources.user import User
 from teal.db import ResourceNotFound
 from tests.conftest import file
@@ -23,23 +28,23 @@ def test_device_model():
     Tests that the correctness of the device model and its relationships.
     """
     pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s')
-    pc.components = components = [
+    net = NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s')
-        NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'),
+    graphic = GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
-        GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
+    pc.components.add(net)
-    ]
+    pc.components.add(graphic)
     db.session.add(pc)
     db.session.commit()
     pc = Desktop.query.one()
     assert pc.serial_number == 'p1s'
-    assert pc.components == components
+    assert pc.components == OrderedSet([net, graphic])
     network_adapter = NetworkAdapter.query.one()
     assert network_adapter.parent == pc
 
     # Removing a component from pc doesn't delete the component
-    del pc.components[0]
+    pc.components.remove(net)
     db.session.commit()
     pc = Device.query.first()  # this is the same as querying for Desktop directly
-    assert pc.components[0].type == GraphicCard.__name__
+    assert pc.components == {graphic}
     network_adapter = NetworkAdapter.query.one()
     assert network_adapter not in pc.components
     assert network_adapter.parent is None
@@ -47,6 +52,7 @@ def test_device_model():
     # Deleting the pc deletes everything
     gcard = GraphicCard.query.one()
     db.session.delete(pc)
+    db.session.flush()
     assert pc.id == 1
     assert Desktop.query.first() is None
     db.session.commit()
@@ -74,7 +80,8 @@ def test_physical_properties():
                     manufacturer='mr',
                     width=2.0,
                     pid='abc')
-    pc = Computer(components=[c])
+    pc = Computer()
+    pc.components.add(c)
     db.session.add(pc)
     db.session.commit()
     assert c.physical_properties == {
@@ -99,9 +106,10 @@ def test_component_similar_one():
     snapshot = file('pc-components.db')
     d = snapshot['device']
     snapshot['components'][0]['serial_number'] = snapshot['components'][1]['serial_number'] = None
-    pc = Computer(**d, components=[Component(**c) for c in snapshot['components']])
+    pc = Computer(**d, components=OrderedSet(Component(**c) for c in snapshot['components']))
     component1, component2 = pc.components  # type: Component
     db.session.add(pc)
+    db.session.flush()
     # Let's create a new component named 'A' similar to 1
     componentA = Component(model=component1.model, manufacturer=component1.manufacturer)
     similar_to_a = componentA.similar_one(pc, set())
@@ -124,10 +132,10 @@ def test_add_remove():
     values = file('pc-components.db')
     pc = values['device']
     c1, c2 = [Component(**c) for c in values['components']]
-    pc = Computer(**pc, components=[c1, c2])
+    pc = Computer(**pc, components=OrderedSet([c1, c2]))
     db.session.add(pc)
     c3 = Component(serial_number='nc1')
-    pc2 = Computer(serial_number='s2', components=[c3])
+    pc2 = Computer(serial_number='s2', components=OrderedSet([c3]))
     c4 = Component(serial_number='c4s')
     db.session.add(pc2)
     db.session.add(c4)
@@ -141,45 +149,212 @@ def test_add_remove():
     assert len(events) == 1
     assert isinstance(events[0], Remove)
     assert events[0].device == pc2
-    assert events[0].components == [c3]
+    assert events[0].components == OrderedSet([c3])
 
 
 @pytest.mark.usefixtures('app_context')
-def test_execute_register_computer():
+def test_sync_run_components_empty():
+    """
+    Syncs a device that has an empty components list. The system should
+    remove all the components from the device.
+    """
+    s = file('pc-components.db')
+    pc = Computer(**s['device'], components=OrderedSet(Component(**c) for c in s['components']))
+    db.session.add(pc)
+    db.session.commit()
+
+    # Create a new transient non-db synced object
+    pc = Computer(**s['device'])
+    db_pc, _ = Sync().run(pc, components=OrderedSet())
+    assert not db_pc.components
+    assert not pc.components
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_run_components_none():
+    """
+    Syncs a device that has a None components. The system should
+    keep all the components from the device.
+    """
+    s = file('pc-components.db')
+    pc = Computer(**s['device'], components=OrderedSet(Component(**c) for c in s['components']))
+    db.session.add(pc)
+    db.session.commit()
+
+    # Create a new transient non-db synced object
+    transient_pc = Computer(**s['device'])
+    db_pc, _ = Sync().run(transient_pc, components=None)
+    assert db_pc.components
+    assert db_pc.components == pc.components
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_computer_new_computer_no_tag():
+    """
+    Syncs a new computer with HID and without a tag, creating it.
+    :return:
+    """
     # Case 1: device does not exist on DB
     pc = Computer(**file('pc-components.db')['device'])
-    db_pc, _ = Sync.execute_register(pc, set())
+    db_pc = Sync().execute_register(pc)
     assert pc.physical_properties == db_pc.physical_properties
 
 
 @pytest.mark.usefixtures('app_context')
-def test_execute_register_computer_existing():
+def test_sync_execute_register_computer_existing_no_tag():
+    """
+    Syncs an existing computer with HID and without a tag.
+    """
     pc = Computer(**file('pc-components.db')['device'])
     db.session.add(pc)
-    db.session.commit()  # We need two separate sessions
+    db.session.commit()
-    pc = Computer(**file('pc-components.db')['device'])
+
+    pc = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
     # 1: device exists on DB
-    db_pc, _ = Sync.execute_register(pc, set())
+    db_pc = Sync().execute_register(pc)
     assert pc.physical_properties == db_pc.physical_properties
 
 
 @pytest.mark.usefixtures('app_context')
-def test_execute_register_computer_no_hid():
+def test_sync_execute_register_computer_no_hid_no_tag():
+    """
+    Syncs a computer without HID and no tag.
+
+    This should fail as we don't have a way to identify it.
+    """
     pc = Computer(**file('pc-components.db')['device'])
     # 1: device has no HID
     pc.hid = pc.model = None
     with pytest.raises(NeedsId):
-        Sync.execute_register(pc, set())
+        Sync().execute_register(pc)
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_computer_tag_not_linked():
+    """
+    Syncs a new computer with HID and a non-linked tag.
+
+    It is OK if the tag was not linked, it will be linked in this process.
+    """
+    tag = Tag(id='FOO')
+    db.session.add(tag)
+    db.session.commit()
+
+    # Create a new transient non-db object
+    pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([Tag(id='FOO')]))
+    returned_pc = Sync().execute_register(pc)
+    assert returned_pc == pc
+    assert tag.device == pc, 'Tag has to be linked'
+    assert Computer.query.one() == pc, 'Computer had to be set to db'
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_no_hid_tag_not_linked(tag_id: str):
+    """
+    Validates registering a computer without HID and a non-linked tag.
+
+    In this case it is ok still, as the non-linked tag proves that
+    the computer was not existing before (otherwise the tag would
+    be linked), and thus it creates a new computer.
+    """
+    tag = Tag(id=tag_id)
+    pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([tag]))
+    returned_pc = Sync().execute_register(pc)
+    db.session.commit()
+    assert returned_pc == pc
+    db_tag = next(iter(returned_pc.tags))
+    # they are not the same tags though
+    # tag is a transient obj and db_tag the one from the db
+    # they have the same pk though
+    assert tag != db_tag, 'They are not the same tags though'
+    assert db_tag.id == tag.id
+    assert Computer.query.one() == pc, 'Computer had to be set to db'
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_tag_does_not_exist():
+    """
+    Ensures not being able to register if the tag does not exist,
+    even if the device has HID or it existed before.
+
+    Tags have to be created before trying to link them through a Snapshot.
+    """
+    pc = Computer(**file('pc-components.db')['device'], tags=OrderedSet([Tag()]))
+    with raises(ResourceNotFound):
+        Sync().execute_register(pc)
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_tag_linked_same_device():
+    """
+    If the tag is linked to the device, regardless if it has HID,
+    the system should match the device through the tag.
+    (If it has HID it validates both HID and tag point at the same
+    device, this his checked in ).
+    """
+    orig_pc = Computer(**file('pc-components.db')['device'])
+    db.session.add(Tag(id='foo', device=orig_pc))
+    db.session.commit()
+
+    pc = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
+    pc.tags.add(Tag(id='foo'))
+    db_pc = Sync().execute_register(pc)
+    assert db_pc.id == orig_pc.id
+    assert len(db_pc.tags) == 1
+    assert next(iter(db_pc.tags)).id == 'foo'
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_tag_linked_other_device_mismatch_between_tags():
+    """
+    Checks that sync raises an error if finds that at least two passed-in
+    tags are not linked to the same device.
+    """
+    pc1 = Computer(**file('pc-components.db')['device'])
+    db.session.add(Tag(id='foo-1', device=pc1))
+    pc2 = Computer(**file('pc-components.db')['device'])
+    pc2.serial_number = 'pc2-serial'
+    pc2.hid = Naming.hid(pc2.manufacturer, pc2.serial_number, pc2.model)
+    db.session.add(Tag(id='foo-2', device=pc2))
+    db.session.commit()
+
+    pc1 = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
+    pc1.tags.add(Tag(id='foo-1'))
+    pc1.tags.add(Tag(id='foo-2'))
+    with raises(MismatchBetweenTags):
+        Sync().execute_register(pc1)
+
+
+@pytest.mark.usefixtures('app_context')
+def test_sync_execute_register_mismatch_between_tags_and_hid():
+    """
+    Checks that sync raises an error if it finds that the HID does
+    not point at the same device as the tag does.
+
+    In this case we set HID -> pc1 but tag -> pc2
+    """
+    pc1 = Computer(**file('pc-components.db')['device'])
+    db.session.add(Tag(id='foo-1', device=pc1))
+    pc2 = Computer(**file('pc-components.db')['device'])
+    pc2.serial_number = 'pc2-serial'
+    pc2.hid = Naming.hid(pc2.manufacturer, pc2.serial_number, pc2.model)
+    db.session.add(Tag(id='foo-2', device=pc2))
+    db.session.commit()
+
+    pc1 = Computer(**file('pc-components.db')['device'])  # Create a new transient non-db object
+    pc1.tags.add(Tag(id='foo-2'))
+    with raises(MismatchBetweenTagsAndHid):
+        Sync().execute_register(pc1)
 
 
 def test_get_device(app: Devicehub, user: UserClient):
     """Checks GETting a Desktop with its components."""
     with app.app_context():
         pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s')
-        pc.components = [
+        pc.components = OrderedSet([
             NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'),
             GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
-        ]
+        ])
         db.session.add(pc)
         db.session.add(Test(device=pc,
                             elapsed=timedelta(seconds=4),
@@ -209,10 +384,10 @@ def test_get_devices(app: Devicehub, user: UserClient):
     """Checks GETting multiple devices."""
     with app.app_context():
         pc = Desktop(model='p1mo', manufacturer='p1ma', serial_number='p1s')
-        pc.components = [
+        pc.components = OrderedSet([
             NetworkAdapter(model='c1mo', manufacturer='c1ma', serial_number='c1s'),
             GraphicCard(model='c2mo', manufacturer='c2ma', memory=1500)
-        ]
+        ])
         pc1 = Microtower(model='p2mo', manufacturer='p2ma', serial_number='p2s')
         pc2 = Laptop(model='p3mo', manufacturer='p3ma', serial_number='p3s')
         db.session.add_all((pc, pc1, pc2))

tests/test_organization.py

@@ -0,0 +1,16 @@
+import pytest
+
+from ereuse_devicehub.config import DevicehubConfig
+from ereuse_devicehub.resources.user import Organization
+
+
+@pytest.mark.usefixtures('app_context')
+def test_default_org_exists(config: DevicehubConfig):
+    """
+    Ensures that the default organization is created on app
+    initialization and that is accessible for the method
+    :meth:`ereuse_devicehub.resources.user.Organization.get_default_org`.
+    """
+    assert Organization.query.filter_by(name=config.ORGANIZATION_NAME,
+                                        tax_id=config.ORGANIZATION_TAX_ID).one()
+    assert Organization.get_default_org().name == config.ORGANIZATION_NAME

tests/test_snapshot.py

@@ -9,8 +9,10 @@ from ereuse_devicehub.db import db
 from ereuse_devicehub.devicehub import Devicehub
 from ereuse_devicehub.resources.device.exceptions import NeedsId
 from ereuse_devicehub.resources.device.models import Device, Microtower
+from ereuse_devicehub.resources.device.sync import MismatchBetweenTagsAndHid
 from ereuse_devicehub.resources.event.models import Appearance, Bios, Event, Functionality, \
     Snapshot, SnapshotRequest, SoftwareType
+from ereuse_devicehub.resources.tag import Tag
 from ereuse_devicehub.resources.user.models import User
 from tests.conftest import file
 
@@ -249,3 +251,31 @@ def _test_snapshot_computer_no_hid(user: UserClient):
     user.post(s, res=Device)
     s['device']['id'] = 1  # Assign the ID of the placeholder
     user.post(s, res=Snapshot)
+
+
+def test_snapshot_mismatch_id():
+    """Tests uploading a device with an ID from another device."""
+    # Note that this won't happen as in this new version
+    # the ID is not used in the Snapshot process
+    pass
+
+
+def test_snapshot_tag_inner_tag(tag_id: str, user: UserClient, app: Devicehub):
+    """Tests a posting Snapshot with a local tag."""
+    b = file('basic.snapshot')
+    b['device']['tags'] = [{'type': 'Tag', 'id': tag_id}]
+    snapshot_and_check(user, b)
+    with app.app_context():
+        tag, *_ = Tag.query.all()  # type: Tag
+        assert tag.device_id == 1, 'Tag should be linked to the first device'
+
+
+def test_snapshot_tag_inner_tag_mismatch_between_tags_and_hid(user: UserClient, tag_id: str):
+    """Ensures one device cannot 'steal' the tag from another one."""
+    pc1 = file('basic.snapshot')
+    pc1['device']['tags'] = [{'type': 'Tag', 'id': tag_id}]
+    user.post(pc1, res=Snapshot)
+    pc2 = file('1-device-with-components.snapshot')
+    user.post(pc2, res=Snapshot)  # PC2 uploads well
+    pc2['device']['tags'] = [{'type': 'Tag', 'id': tag_id}]  # Set tag from pc1 to pc2
+    user.post(pc2, res=Snapshot, status=MismatchBetweenTagsAndHid)

tests/test_tag.py

@@ -0,0 +1,121 @@
+import pytest
+from pytest import raises
+from sqlalchemy.exc import IntegrityError
+
+from ereuse_devicehub.client import UserClient
+from ereuse_devicehub.db import db
+from ereuse_devicehub.devicehub import Devicehub
+from ereuse_devicehub.resources.device.models import Computer
+from ereuse_devicehub.resources.tag import Tag
+from ereuse_devicehub.resources.tag.view import CannotCreateETag, TagNotLinked
+from ereuse_devicehub.resources.user import Organization
+from teal.db import MultipleResourcesFound, ResourceNotFound
+from teal.marshmallow import ValidationError
+
+
+@pytest.mark.usefixtures('app_context')
+def test_create_tag():
+    """Creates a tag specifying a custom organization."""
+    org = Organization(name='Bar', tax_id='BarTax')
+    tag = Tag(id='bar-1', org=org)
+    db.session.add(tag)
+    db.session.commit()
+
+
+@pytest.mark.usefixtures('app_context')
+def test_create_tag_default_org():
+    """Creates a tag using the default organization."""
+    tag = Tag(id='foo-1')
+    assert not tag.org_id, 'org-id is set as default value so it should only load on flush'
+    # We don't want the organization to load, or it would make this
+    # object, from transient to new (added to session)
+    assert 'org' not in vars(tag), 'Organization should not have been loaded'
+    db.session.add(tag)
+    db.session.commit()
+    assert tag.org.name == 'FooOrg'  # as defined in the settings
+
+
+@pytest.mark.usefixtures('app_context')
+def test_create_tag_no_slash():
+    """Checks that no tags can be created that contain a slash."""
+    with raises(ValidationError):
+        Tag(id='/')
+
+
+@pytest.mark.usefixtures('app_context')
+def test_create_two_same_tags():
+    """Ensures there cannot be two tags with the same ID and organization."""
+    db.session.add(Tag(id='foo-bar'))
+    db.session.add(Tag(id='foo-bar'))
+    with raises(IntegrityError):
+        db.session.commit()
+    db.session.rollback()
+    # And it works if tags are in different organizations
+    db.session.add(Tag(id='foo-bar'))
+    org2 = Organization(name='org 2', tax_id='tax id org 2')
+    db.session.add(Tag(id='foo-bar', org=org2))
+    db.session.commit()
+
+
+def test_tag_post(app: Devicehub, user: UserClient):
+    """Checks the POST method of creating a tag."""
+    user.post(res=Tag, query=[('ids', 'foo')], data={})
+    with app.app_context():
+        assert Tag.query.filter_by(id='foo').one()
+
+
+def test_tag_post_etag(user: UserClient):
+    """
+    Ensures users cannot create eReuse.org tags through POST;
+    only terminal.
+    """
+    user.post(res=Tag, query=[('ids', 'FO-123456')], data={}, status=CannotCreateETag)
+    # Although similar, these are not eTags and should pass
+    user.post(res=Tag, query=[
+        ('ids', 'FO-0123-45'),
+        ('ids', 'FOO012345678910'),
+        ('ids', 'FO'),
+        ('ids', 'FO-'),
+        ('ids', 'FO-123'),
+        ('ids', 'FOO-123456')
+    ], data={})
+
+
+def test_tag_get_device_from_tag_endpoint(app: Devicehub, user: UserClient):
+    """Checks getting a linked device from a tag endpoint"""
+    with app.app_context():
+        # Create a pc with a tag
+        tag = Tag(id='foo-bar')
+        pc = Computer(serial_number='sn1')
+        pc.tags.add(tag)
+        db.session.add(pc)
+        db.session.commit()
+    computer, _ = user.get(res=Tag, item='foo-bar/device')
+    assert computer['serialNumber'] == 'sn1'
+
+
+def test_tag_get_device_from_tag_endpoint_no_linked(app: Devicehub, user: UserClient):
+    """As above, but when the tag is not linked."""
+    with app.app_context():
+        db.session.add(Tag(id='foo-bar'))
+        db.session.commit()
+    user.get(res=Tag, item='foo-bar/device', status=TagNotLinked)
+
+
+def test_tag_get_device_from_tag_endpoint_no_tag(user: UserClient):
+    """As above, but when there is no tag with such ID."""
+    user.get(res=Tag, item='foo-bar/device', status=ResourceNotFound)
+
+
+def test_tag_get_device_from_tag_endpoint_multiple_tags(app: Devicehub, user: UserClient):
+    """
+    As above, but when there are two tags with the same ID, the
+    system should not return any of both (to be deterministic) so
+    it should raise an exception.
+    """
+    with app.app_context():
+        db.session.add(Tag(id='foo-bar'))
+        org2 = Organization(name='org 2', tax_id='tax id org 2')
+        db.session.add(Tag(id='foo-bar', org=org2))
+        db.session.commit()
+    user.get(res=Tag, item='foo-bar/device', status=MultipleResourcesFound)

tests/test_user.py

@@ -34,10 +34,11 @@ def test_create_user_email_insensitive(app: Devicehub):
     with app.app_context():
         user = User(email='FOO@foo.com')
         db.session.add(user)
+        db.session.commit()
         # We search in case insensitive manner
         u1 = User.query.filter_by(email='foo@foo.com').one()
         assert u1 == user
-        assert u1.email == 'FOO@foo.com'
+        assert u1.email == 'foo@foo.com'
 
 
 def test_hash_password(app: Devicehub):