diff --git a/ereuse_devicehub/db.py b/ereuse_devicehub/db.py index ac06c9d6..5fc091cb 100644 --- a/ereuse_devicehub/db.py +++ b/ereuse_devicehub/db.py @@ -1,9 +1,29 @@ import citext from sqlalchemy import event from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import sessionmaker from sqlalchemy.sql import expression from sqlalchemy_utils import view -from teal.db import SchemaSQLAlchemy +from teal.db import SchemaSQLAlchemy, SchemaSession + + +class DhSession(SchemaSession): + def final_flush(self): + """A regular flush that performs expensive final operations + through Devicehub (like saving searches), so it is thought + to be used once in each request, at the very end before + a commit. + """ + # This was done before with an ``before_commit`` sqlalchemy event + # however it is too fragile –it does not detect previously-flushed + # things + # This solution makes this more aware to the user, although + # has the same problem. This is not final solution. + # todo a solution would be for this session to save, on every + # flush, all the new / dirty interesting things in a variable + # until DeviceSearch is executed + from ereuse_devicehub.resources.device.search import DeviceSearch + DeviceSearch.update_modified_devices(session=self) class SQLAlchemy(SchemaSQLAlchemy): @@ -23,6 +43,9 @@ class SQLAlchemy(SchemaSQLAlchemy): if common_schema: self.drop_schema(schema='common') + def create_session(self, options): + return sessionmaker(class_=DhSession, db=self, **options) + def create_view(name, selectable): """Creates a view. diff --git a/ereuse_devicehub/devicehub.py b/ereuse_devicehub/devicehub.py index 097a538c..c3925ec8 100644 --- a/ereuse_devicehub/devicehub.py +++ b/ereuse_devicehub/devicehub.py @@ -8,7 +8,6 @@ import ereuse_utils.cli from ereuse_utils.session import DevicehubClient from flask.globals import _app_ctx_stack, g from flask_sqlalchemy import SQLAlchemy -from sqlalchemy import event from teal.teal import Teal from ereuse_devicehub.auth import Auth @@ -47,16 +46,10 @@ class Devicehub(Teal): self.id = inventory """The Inventory ID of this instance. In Teal is the app.schema.""" self.dummy = Dummy(self) - self.before_request(self.register_db_events_listeners) self.cli.command('regenerate-search')(self.regenerate_search) self.cli.command('init-db')(self.init_db) self.before_request(self._prepare_request) - def register_db_events_listeners(self): - """Registers the SQLAlchemy event listeners.""" - # todo can I make it with a global Session only? - event.listen(db.session, 'before_commit', DeviceSearch.update_modified_devices) - # noinspection PyMethodOverriding @click.option('--name', '-n', default='Test 1', diff --git a/ereuse_devicehub/dummy/dummy.py b/ereuse_devicehub/dummy/dummy.py index 511d11a1..4495a821 100644 --- a/ereuse_devicehub/dummy/dummy.py +++ b/ereuse_devicehub/dummy/dummy.py @@ -141,9 +141,9 @@ class Dummy: assert len(inventory['items']) i, _ = user.get(res=Device, query=[('search', 'intel')]) - assert len(i['items']) == 12 + assert 12 == len(i['items']) i, _ = user.get(res=Device, query=[('search', 'pc')]) - assert len(i['items']) == 14 + assert 14 == len(i['items']) # Let's create a set of events for the pc device # Make device Ready diff --git a/ereuse_devicehub/resources/event/models.py b/ereuse_devicehub/resources/event/models.py index 7e180810..9efc9e0f 100644 --- a/ereuse_devicehub/resources/event/models.py +++ b/ereuse_devicehub/resources/event/models.py @@ -10,7 +10,7 @@ import teal.db from boltons import urlutils from citext import CIText from flask import current_app as app, g -from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, DateTime, Enum as DBEnum, \ +from sqlalchemy import BigInteger, Boolean, CheckConstraint, Column, Enum as DBEnum, \ Float, ForeignKey, Integer, Interval, JSON, Numeric, SmallInteger, Unicode, event, orm from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declared_attr @@ -378,9 +378,10 @@ class Step(db.Model): type = Column(Unicode(STR_SM_SIZE), nullable=False) num = Column(SmallInteger, primary_key=True) severity = Column(teal.db.IntEnum(Severity), default=Severity.Info, nullable=False) - start_time = Column(DateTime, nullable=False) + start_time = Column(db.TIMESTAMP(timezone=True), nullable=False) start_time.comment = Event.start_time.comment - end_time = Column(DateTime, CheckConstraint('end_time > start_time'), nullable=False) + end_time = Column(db.TIMESTAMP(timezone=True), CheckConstraint('end_time > start_time'), + nullable=False) end_time.comment = Event.end_time.comment erasure = relationship(EraseBasic, @@ -1187,7 +1188,7 @@ class Trade(JoinedTableMixin, EventWithMultipleDevices): This class and its inheritors extend `Schema's Trade `_. """ - shipping_date = Column(DateTime) + shipping_date = Column(db.TIMESTAMP(timezone=True)) shipping_date.comment = """ When are the devices going to be ready for shipping? """ diff --git a/ereuse_devicehub/resources/event/views.py b/ereuse_devicehub/resources/event/views.py index a492fb79..0d6e846c 100644 --- a/ereuse_devicehub/resources/event/views.py +++ b/ereuse_devicehub/resources/event/views.py @@ -23,9 +23,10 @@ class EventView(View): Model = db.Model._decl_class_registry.data[json['type']]() event = Model(**e) db.session.add(event) - db.session.commit() + db.session().final_flush() ret = self.schema.jsonify(event) ret.status_code = 201 + db.session.commit() return ret def one(self, id: UUID): @@ -84,7 +85,8 @@ class SnapshotView(View): snapshot.events |= rates db.session.add(snapshot) - db.session.commit() + db.session().final_flush() ret = self.schema.jsonify(snapshot) # transform it back ret.status_code = 201 + db.session.commit() return ret diff --git a/ereuse_devicehub/resources/lot/views.py b/ereuse_devicehub/resources/lot/views.py index 96c8c824..eb272c34 100644 --- a/ereuse_devicehub/resources/lot/views.py +++ b/ereuse_devicehub/resources/lot/views.py @@ -34,9 +34,10 @@ class LotView(View): l = request.get_json() lot = Lot(**l) db.session.add(lot) - db.session.commit() + db.session().final_flush() ret = self.schema.jsonify(lot) ret.status_code = 201 + db.session.commit() return ret def patch(self, id): @@ -144,17 +145,21 @@ class LotBaseChildrenView(View): def post(self, id: uuid.UUID): lot = self.get_lot(id) self._post(lot, self.get_ids()) - db.session.commit() + db.session().final_flush() ret = self.schema.jsonify(lot) ret.status_code = 201 + + db.session.commit() return ret def delete(self, id: uuid.UUID): lot = self.get_lot(id) self._delete(lot, self.get_ids()) + db.session().final_flush() + response = self.schema.jsonify(lot) db.session.commit() - return self.schema.jsonify(lot) + return response def _post(self, lot: Lot, ids: Set[uuid.UUID]): raise NotImplementedError diff --git a/ereuse_devicehub/resources/tag/view.py b/ereuse_devicehub/resources/tag/view.py index f337c8de..9cfe6d7d 100644 --- a/ereuse_devicehub/resources/tag/view.py +++ b/ereuse_devicehub/resources/tag/view.py @@ -32,8 +32,10 @@ class TagView(View): tags_id, _ = g.tag_provider.post('/', {}, query=[('num', num)]) tags = [Tag(id=tag_id, provider=g.inventory.tag_provider) for tag_id in tags_id] db.session.add_all(tags) + db.session().final_flush() + response = things_response(self.schema.dump(tags, many=True, nested=1), code=201) db.session.commit() - return things_response(self.schema.dump(tags, many=True, nested=1), code=201) + return response def _post_one(self): # todo do we use this? @@ -42,6 +44,7 @@ class TagView(View): if tag.like_etag(): raise CannotCreateETag(tag.id) db.session.add(tag) + db.session().final_flush() db.session.commit() return Response(status=201) @@ -69,6 +72,7 @@ class TagDeviceView(View): raise LinkedToAnotherDevice(tag.device_id) else: tag.device_id = device_id + db.session().final_flush() db.session.commit() return Response(status=204) diff --git a/tests/files/erase-sectors.snapshot.yaml b/tests/files/erase-sectors.snapshot.yaml index 4331352e..59610ff5 100644 --- a/tests/files/erase-sectors.snapshot.yaml +++ b/tests/files/erase-sectors.snapshot.yaml @@ -10,28 +10,28 @@ device: model: pc1ml manufacturer: pc1mr components: -- type: SolidStateDrive - serialNumber: c1s - model: c1ml - manufacturer: c1mr - events: - - type: EraseSectors - startTime: 2018-06-01T08:12:06 - endTime: 2018-06-01T09:12:06 - steps: - - type: StepZero - severity: Info - startTime: 2018-06-01T08:15:00 - endTime: 2018-06-01T09:16:00 - - type: StepRandom - severity: Info - startTime: 2018-06-01T08:16:00 - endTime: 2018-06-01T09:17:00 -- type: Processor - serialNumber: p1s - model: p1ml - manufacturer: p1mr -- type: RamModule - serialNumber: rm1s - model: rm1ml - manufacturer: rm1mr + - type: SolidStateDrive + serialNumber: c1s + model: c1ml + manufacturer: c1mr + events: + - type: EraseSectors + startTime: '2018-06-01T08:12:06+02:00' + endTime: '2018-06-01T09:12:06+02:00' + steps: + - type: StepZero + severity: Info + startTime: '2018-06-01T08:15:00+02:00' + endTime: '2018-06-01T09:16:00+02:00' + - type: StepRandom + severity: Info + startTime: '2018-06-01T08:16:00+02:00' + endTime: '2018-06-01T09:17:00+02:00' + - type: Processor + serialNumber: p1s + model: p1ml + manufacturer: p1mr + - type: RamModule + serialNumber: rm1s + model: rm1ml + manufacturer: rm1mr diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 4fda076a..2b7ac98d 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -86,7 +86,7 @@ def test_snapshot_post(user: UserClient): assert snapshot['components'] == device['components'] assert {c['type'] for c in snapshot['components']} == {m.GraphicCard.t, m.RamModule.t, - m.Processor.t} + m.Processor.t} rate = next(e for e in snapshot['events'] if e['type'] == WorkbenchRate.t) rate, _ = user.get(res=Event, item=rate['id']) assert rate['device']['id'] == snapshot['device']['id'] @@ -298,7 +298,9 @@ def test_erase_privacy_standards(user: UserClient): privacy properties. """ s = file('erase-sectors.snapshot') + assert '2018-06-01T09:12:06+02:00' == s['components'][0]['events'][0]['endTime'] snapshot = snapshot_and_check(user, s, (EraseSectors.t,), perform_second_snapshot=True) + assert '2018-06-01T07:12:06+00:00' == snapshot['events'][0]['endTime'] storage, *_ = snapshot['components'] assert storage['type'] == 'SolidStateDrive', 'Components must be ordered by input order' storage, _ = user.get(res=m.Device, item=storage['id']) # Let's get storage events too @@ -306,13 +308,15 @@ def test_erase_privacy_standards(user: UserClient): erasure1, _snapshot1, erasure2, _snapshot2 = storage['events'] assert erasure1['type'] == erasure2['type'] == 'EraseSectors' assert _snapshot1['type'] == _snapshot2['type'] == 'Snapshot' - assert snapshot == user.get(res=Event, item=_snapshot2['id'])[0] + get_snapshot, _ = user.get(res=Event, item=_snapshot2['id']) + assert get_snapshot['events'][0]['endTime'] == '2018-06-01T07:12:06+00:00' + assert snapshot == get_snapshot erasure, _ = user.get(res=Event, item=erasure1['id']) assert len(erasure['steps']) == 2 - assert erasure['steps'][0]['startTime'] == '2018-06-01T08:15:00+00:00' - assert erasure['steps'][0]['endTime'] == '2018-06-01T09:16:00+00:00' - assert erasure['steps'][1]['startTime'] == '2018-06-01T08:16:00+00:00' - assert erasure['steps'][1]['endTime'] == '2018-06-01T09:17:00+00:00' + assert erasure['steps'][0]['startTime'] == '2018-06-01T06:15:00+00:00' + assert erasure['steps'][0]['endTime'] == '2018-06-01T07:16:00+00:00' + assert erasure['steps'][1]['startTime'] == '2018-06-01T06:16:00+00:00' + assert erasure['steps'][1]['endTime'] == '2018-06-01T07:17:00+00:00' assert erasure['device']['id'] == storage['id'] step1, step2 = erasure['steps'] assert step1['type'] == 'StepZero'