docker compose: make init user configurable

docker-compose.yml

@@ -10,6 +10,8 @@ services:
       - DOMAIN=${DEVICEHUB_DOMAIN:-localhost}
       - PORT=${DEVICEHUB_PORT:-8000}
       - ALLOWED_HOSTS=${DEVICEHUB_ALLOWED_HOSTS:-$DEVICEHUB_DOMAIN}
+      - INIT_USER=${INIT_USER:-}
+      - INIT_PASSWD=${INIT_PASSWD:-}
       - DEMO=${DEMO:-false}
       - DEMO_IDHUB_DOMAIN=${DEMO_IDHUB_DOMAIN:-}
       - DEMO_IDHUB_PREDEFINED_TOKEN=${IDHUB_PREDEFINED_TOKEN:-}

docker/devicehub-django.entrypoint.sh

@@ -147,7 +147,6 @@ run_demo() {
                         'example/demo-snapshots-vc/snapshot_pre-verifiable-credential.json' \
                         > 'example/snapshots/snapshot_workbench-script_verifiable-credential.json'
         fi
-        ./manage.py create_default_states "${INIT_ORG}"
         /usr/bin/time ./manage.py up_snapshots example/snapshots/ "${INIT_USER}"
 }
 

evidence/mixin_parse.py

@@ -24,6 +24,7 @@ class BuildMix:
         if not self.uuid:
             logger.error("snapshot without UUID. Software {}".format(self.json.get("software")))
             return
+
         self.get_details()
         self.generate_chids()
 

example/migrations/README.md

@@ -0,0 +1,12 @@
+Para poder hacer una migracion de los datos de devicehub-teal necesitamos varios ficheros csv con los datos.
+Por ejemplo:
+
+--csv-lots-dhid con la relación dhid con nombre del lote.
+--csv-dhid es la relación de dhid con uuid de un snapshot.
+--lots es la relación entre el nombre de un lote y el nombre del tipo de lote
+--snapshots es el directorio donde buscar los snapshots reales. Los busca por uuid
+
+
+```
+  python example/migrations/migration-script.py --email user@example.org --csv-lots-dhid example/migrations/device-lots.csv --csv-dhid example/migrations/dhids.csv --lots example/migrations/lot.csv --snapshots example/migrations/snapshots/
+```

example/migrations/device-lots.csv

@@ -0,0 +1,2 @@
+"lot_name";"dhid"
+"Moravia202205y06";"N9EEG"

example/migrations/dhids.csv

@@ -0,0 +1,12 @@
+"dhid";"uuid"
+"JGMLO";"ae36bd1b-a0a5-4860-ad67-1644d2ce0c7b"
+"MLVKL";"8ad8022b-e1d7-4c9a-bfad-e3677beb9f8d"
+"D8BWP";"9868c6d4-a96c-4ae6-b2bd-1ec72a1891a4"
+"D8BYP";"b3a1fff7-24f9-4a72-9362-86934d03b20a"
+"MLVKL";"5d2f7902-ceb4-452c-832c-adaf3cbc3686"
+"9GRZ5";"e48b2121-7edf-4a6e-ba0c-9f2e1a87c7f9"
+"V9XRK";"e6f5a76f-4905-4553-b4d4-bc73bf1a8022"
+"V9XBK";"7aa682aa-77c5-43a5-91e4-1ef9a2736156"
+"7DVLE";"522a4403-bfb3-4fe6-b454-4db6c3040a58"
+"QPAVE";"946fd30b-68f7-425f-aca4-1ae39a86d35"
+"N9EEG";"0cf11287-5603-45e4-9dd6-075871286de9"

example/migrations/lot.csv

@@ -0,0 +1,2 @@
+"name";"type"
+"Moravia202205y06";"Incoming"

example/migrations/migration-script.py

@@ -0,0 +1,261 @@
+import os
+import json
+import django
+import logging
+import argparse
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dhub.settings')
+
+django.setup()
+
+from django.contrib.auth import get_user_model
+
+from utils.save_snapshots import move_json, save_in_disk
+from evidence.parse import Build
+from evidence.models import SystemProperty
+from lot.models import Lot, LotTag, DeviceLot
+
+
+logger = logging.getLogger(__name__)
+
+User = get_user_model()
+
+SEPARATOR = ";"
+QUOTA = '"'
+PATH_SNAPTHOPS = "examples/snapshots"
+
+
+### read csv ###
+def get_dict(row, header):
+    if not row or not header:
+        return
+    if len(row) != len(header):
+        return
+
+    return {header[i]: row[i] for i in range(len(header))}
+
+
+def open_csv(csv):
+    # return a list of dictionaries whith the header of csv as keys of the dicts
+    with open(csv) as f:
+        _file = f.read()
+
+    rows = _file.split("\n")
+    if len(rows) < 2:
+        return []
+
+    header = [x.replace(QUOTA, '') for x in rows[0].split(SEPARATOR)]
+    data = []
+    for row in rows[1:]:
+        lrow = [x.replace(QUOTA, '') for x in row.split(SEPARATOR)]
+        drow = get_dict(lrow, header)
+        if drow:
+            data.append(drow)
+    return data
+### end read csv ###
+
+
+### read snapshot ###
+def open_snapshot(snapshot_path):
+    with open(snapshot_path) as f:
+        try:
+            snap = json.loads(f.read())
+        except Exception as err:
+            logger.error("{}, error: {}".format(snapshot_path, err))
+            return None, None
+    return snap
+### end read snapshot ###
+
+
+### migration snapshots ###
+def create_custom_id(dhid, uuid, user):
+
+    if not uuid or not dhid:
+        return
+
+    tag = SystemProperty.objects.filter(
+        uuid=uuid,
+        key='CUSTOM_ID',
+        owner=user.institution
+    ).first()
+
+    if tag or not uuid or not dhid:
+        return
+
+    SystemProperty.objects.create(
+        uuid=uuid,
+        key='CUSTOM_ID',
+        value=dhid,
+        owner=user.institution,
+        user=user
+    )
+
+
+def migrate_snapshots(dhids, snapshot_path, user):
+    snapshot = open_snapshot(snapshot_path)
+    uuid = snapshot.get("uuid")
+    if not snapshot or not uuid or snapshot.get("software") == "Web":
+        return
+
+    logger.info(snapshot.get("version"))
+    if snapshot.get('version') == "2022.12.2-beta":
+        return
+
+    # insert snapshot
+    path_name = save_in_disk(snapshot, user.institution.name)
+    Build(snapshot, user)
+    move_json(path_name, user.institution.name)
+
+    # insert dhid
+    dhid = dhids.get(uuid)
+    try:
+        create_custom_id(dhid, uuid, user)
+    except Exception as err:
+        logger.error(err)
+        logger.error("DHID: %s uuid: %s", dhid, uuid)
+
+### end migration snapshots ###
+
+
+### migration lots ###
+def migrate_lots(row, user):
+    tag = row.get("type", "Temporal")
+    name = row.get("lot_name")
+    ltag = LotTag.objects.filter(name=tag, owner=user.institution).first()
+    if tag and not ltag:
+        ltag = LotTag.objects.create(
+            name=tag,
+            owner=user.institution,
+            user=user
+        )
+
+    if Lot.objects.filter(name=name, owner=user.institution).first():
+        return
+
+    Lot.objects.create(
+        name=name,
+        owner=user.institution,
+        user=user,
+        type=ltag
+    )
+
+
+def add_device_in_lot(row, user):
+    lot_name = row.get("lot_name")
+    dhid = row.get("dhid")
+
+    if not lot_name or not dhid:
+        return
+
+    dev = SystemProperty.objects.filter(
+        key='CUSTOM_ID',
+        value=dhid,
+        owner=user.institution,
+    ).first()
+
+    if not dev:
+        logger.warning("Not exist dhid %s", dhid)
+        return
+
+    lot = Lot.objects.filter(
+        name=lot_name,
+        owner=user.institution,
+        user=user,
+    ).first()
+
+    if not lot:
+        lot = Lot.objects.create(
+            name=lot_name,
+            owner=user.institution,
+            user=user,
+        )
+
+    if DeviceLot.objects.filter(lot=lot, device_id=dhid).exists():
+        return
+    DeviceLot.objects.create(lot=lot, device_id=dhid)
+
+### end migration lots ###
+
+
+### initial main ###
+def prepare_logger():
+
+    logger.setLevel(logging.INFO)
+
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(logging.INFO)
+    formatter = logging.Formatter('[%(asctime)s] migrate: %(levelname)s: %(message)s')
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+
+
+def parse_args():
+    """
+    Parse config argument, if available
+    """
+    parser = argparse.ArgumentParser(
+        usage="migration-script.py [-h] [--csv CSV]",
+        description="Csv file with datas to migratie.")
+    parser.add_argument(
+        '--csv-dhid',
+        help="path to the csv file with relation dhid and uuid."
+    )
+    parser.add_argument(
+        '--lots',
+        help="path to the csv file with relation lot_name and type of lot."
+    )
+    parser.add_argument(
+        '--csv-lots-dhid',
+        help="path to the csv file with relation lot_name and type of lot."
+    )
+    parser.add_argument(
+        '--email',
+        help="email of user.",
+    )
+    parser.add_argument(
+        '--snapshots',
+        help="dir where reside the snapshots.",
+    )
+    return parser.parse_args()
+
+
+def main():
+    prepare_logger()
+    logger.info("START")
+    args = parse_args()
+    user = User.objects.get(email=args.email)
+
+    if args.snapshots:
+        global PATH_SNAPTHOPS
+        PATH_SNAPTHOPS = args.snapshots
+
+    # migration snapthots
+    if args.csv_dhid and args.snapshots:
+        dhids = {}
+        for row in open_csv(args.csv_dhid):
+            dhid = row.get("dhid")
+            uuid = row.get("uuid")
+            if not dhid or not uuid:
+                continue
+            dhids[uuid] = dhid
+
+        for root, _, files in os.walk(PATH_SNAPTHOPS):
+            for f in files:
+                if f[-4:] != "json":
+                    continue
+                snapshot_path = os.path.join(root, f)
+                migrate_snapshots(dhids, snapshot_path, user)
+
+    # migration lots
+    if args.lots:
+        for row in open_csv(args.lots):
+            migrate_lots(row, user)
+
+    # migration dhids in lots
+    if args.csv_lots_dhid:
+        for row in open_csv(args.csv_lots_dhid):
+            add_device_in_lot(row, user)
+
+
+if __name__ == '__main__':
+    main()

example/migrations/migration.sql

@@ -0,0 +1,59 @@
+-- Defined the email of user
+\set email 'user@example.org'
+
+-- save dhids and uuids of snapshots
+copy(
+select d.devicehub_id as dhid, sp.uuid as uuid from usody.placeholder as p
+  join common.user as u on u.id=owner_id
+  join usody.computer as c on c.id=p.binding_id
+  join usody.device as d on d.id=p.device_id
+  join usody.action_with_one_device as one on p.binding_id=one.device_id
+  join usody.snapshot as sp on sp.id=one.id
+where u.email=:'email'
+  and not p.binding_id is null
+) to '/var/lib/postgresql/dhids.csv'
+with (format csv, header, delimiter ';', quote '"');
+
+
+-- save lots and types
+copy(
+select distinct l.name as lot_name, 'Incoming' as type from usody.transfer as t
+  join usody.lot as l on l.id=t.lot_id
+  join common.user as u on u.id=l.owner_id
+where u.email=:'email' and
+    t.user_to_id=u.id and
+    l.owner_id=u.id
+) to '/var/lib/postgresql/lot_incoming.csv'
+with (format csv, header, delimiter ';', quote '"');
+
+copy(
+select distinct l.name as lot_name, 'Outgoing' as type from usody.transfer as t
+  join usody.lot as l on l.id=t.lot_id
+  join common.user as u on u.id=l.owner_id
+where u.email=:'email' and
+    t.user_from_id=u.id and
+    l.owner_id=u.id
+) to '/var/lib/postgresql/lot_outgoing.csv'
+with (format csv, header, delimiter ';', quote '"');
+
+copy(
+select distinct l.name as lot_name, 'Temporary' as type from usody.lot as l
+  left join usody.transfer as t on t.lot_id=l.id
+  join common.user as u on u.id=l.owner_id
+where u.email=:'email' and
+  l.owner_id=u.id and
+  t.lot_id is null
+) to '/var/lib/postgresql/lot_temporary.csv'
+with (format csv, header, delimiter ';', quote '"');
+
+-- save devices in lots
+copy(
+select l.name as lot_name, d.devicehub_id as dhid from usody.lot_device as ld
+  join usody.lot as l on l.id=ld.lot_id
+  join usody.device as d on d.id=ld.device_id
+  join common.user as u on u.id=ld.author_id
+  join usody.placeholder as p on p.device_id=d.id
+where u.email=:'email'
+  and not p.binding_id is null
+) to '/var/lib/postgresql/devices-lots.csv'
+with (format csv, header, delimiter ';', quote '"');

lot/migrations/0008_rename_closed_lot_archived.py

@@ -1,18 +0,0 @@
-# Generated by Django 5.0.6 on 2025-02-25 12:32
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('lot', '0007_lottag_inbox'),
-    ]
-
-    operations = [
-        migrations.RenameField(
-            model_name='lot',
-            old_name='closed',
-            new_name='archived',
-        ),
-    ]

lot/models.py

@@ -32,7 +32,7 @@ class Lot(models.Model):
     name = models.CharField(max_length=STR_SIZE, blank=True, null=True)
     code = models.CharField(max_length=STR_SIZE, blank=True, null=True)
     description = models.CharField(max_length=STR_SIZE, blank=True, null=True)
-    archived = models.BooleanField(default=False)
+    closed = models.BooleanField(default=False)
     owner = models.ForeignKey(Institution, on_delete=models.CASCADE)
     user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True)
     type = models.ForeignKey(LotTag, on_delete=models.CASCADE)

lot/templates/lots.html

@@ -7,13 +7,13 @@
     <h3>{{ subtitle }}</h3>
   </div>
   <div class="col text-center">
-    {% if show_archived %}
-      <a href="?show_archived=false" class="btn btn-green-admin">
-        {% trans 'Show active lots' %}
+    {% if show_closed %}
+      <a href="?show_closed=false" class="btn btn-green-admin">
+        {% trans 'Hide closed lots' %}
       </a>
     {% else %}
-      <a href="?show_archived=true" class="btn btn-green-admin">
-        {% trans 'Show archived lots' %}
+      <a href="?show_closed=true" class="btn btn-green-admin">
+        {% trans 'Show closed lots' %}
       </a>
     {% endif %}
 

lot/views.py

@@ -25,7 +25,7 @@ class NewLotView(DashboardView, CreateView):
         "name",
         "code",
         "description",
-        "archived",
+        "closed",
     )
 
     def get_form(self):
@@ -54,7 +54,7 @@ class DeleteLotView(DashboardView, DeleteView):
         "name",
         "code",
         "description",
-        "archived",
+        "closed",
     )
 
     def form_valid(self, form):
@@ -73,7 +73,7 @@ class EditLotView(DashboardView, UpdateView):
         "name",
         "code",
         "description",
-        "archived",
+        "closed",
     )
 
     def get_form_kwargs(self):
@@ -149,15 +149,15 @@ class LotsTagsView(DashboardView, TemplateView):
         tag = get_object_or_404(LotTag, owner=self.request.user.institution, id=self.pk)
         self.title += " {}".format(tag.name)
         self.breadcrumb += " {}".format(tag.name)
-        show_archived = self.request.GET.get('show_archived', 'false') == 'true'
+        show_closed = self.request.GET.get('show_closed', 'false') == 'true'
         lots = Lot.objects.filter(owner=self.request.user.institution).filter(
-            type=tag, archived=show_archived
+            type=tag, closed=show_closed
         )
         context.update({
             'lots': lots,
             'title': self.title,
             'breadcrumb': self.breadcrumb,
-            'show_archived': show_archived
+            'show_closed': show_closed
         })
         return context
 

user/management/commands/add_institution.py

@@ -1,6 +1,6 @@
 from django.core.management.base import BaseCommand
 from user.models import Institution
-from lot.models import LotTag, Lot
+from lot.models import LotTag
 
 
 class Command(BaseCommand):
@@ -12,7 +12,6 @@ class Command(BaseCommand):
     def handle(self, *args, **kwargs):
         self.institution = Institution.objects.create(name=kwargs['name'])
         self.create_lot_tags()
-        self.create_lots()
 
     def create_lot_tags(self):
         LotTag.objects.create(
@@ -30,59 +29,3 @@ class Command(BaseCommand):
                 name=tag,
                 owner=self.institution
             )
-
-    def create_lots(self):
-        for g in LotTag.objects.all():
-            if g.name == "Entrada":
-                Lot.objects.create(
-                    name="donante-orgA",
-                    owner=self.institution,
-                    archived=True,
-                    type=g
-                )
-                Lot.objects.create(
-                    name="donante-orgB",
-                    owner=self.institution,
-                    type=g
-                )
-                Lot.objects.create(
-                    name="donante-orgC",
-                    owner=self.institution,
-                    type=g
-                )
-
-            if g.name == "Salida":
-                Lot.objects.create(
-                    name="beneficiario-org1",
-                    owner=self.institution,
-                    type=g
-                )
-                Lot.objects.create(
-                    name="beneficiario-org2",
-                    owner=self.institution,
-                    archived=True,
-                    type=g
-                )
-                Lot.objects.create(
-                    name="beneficiario-org3",
-                    owner=self.institution,
-                    type=g
-                )
-
-            if g.name == "Temporal":
-                Lot.objects.create(
-                    name="palet1",
-                    owner=self.institution,
-                    type=g
-                )
-                Lot.objects.create(
-                    name="palet2",
-                    owner=self.institution,
-                    type=g
-                )
-                Lot.objects.create(
-                    name="palet3",
-                    owner=self.institution,
-                    archived=True,
-                    type=g
-                )