add optional dpp_module and snapshots

- dpp_module becomes optional - docker compose for simple devicehub and with the dpp module - changed logic in entrypoint so different parts are configured or - not depending on the new DPP_MODULE env var - optional snapshots - new default directory for SNAPSHOTS_PATH - new env var IMPORT_SNAPSHOTS to optionally import the snapshots or not - new Make file targets: - dc_up_devicehub: docker compose for simple devicehub - dc_up_devicehub_dpp: docker compose for devicehub with DPP_module
2023-09-29 10:51:48 +02:00 · 2023-09-29 10:51:48 +02:00 · bcb4c69677
parent b594022194
commit bcb4c69677
8 changed files with 216 additions and 122 deletions
--- a/.gitignore
+++ b/.gitignore
@ -135,6 +135,7 @@ env*
 examples/create-db2.sh
 package-lock.json
 snapshots/
+!examples/snapshots
 modules/

 # emacs
--- a/14
+++ b/14
@ -32,3 +32,17 @@ docker:
 	$(MAKE) docker_build
 	$(MAKE) docker_publish
 	@printf "\ndocker images published\n"
+
+# manage 2 kinds of deployments with docker compose
+
+dc_up_devicehub:
+	docker compose -f docker-compose_devicehub.yml up || true
+
+dc_down_devicehub:
+	docker compose -f docker-compose_devicehub.yml down -v || true
+
+dc_up_devicehub_dpp:
+	docker compose -f docker-compose_devicehub-dpp.yml up || true
+
+dc_down_devicehub_dpp:
+	docker compose -f docker-compose_devicehub-dpp.yml down -v || true
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,95 +0,0 @@
-version: "3.9"
-services:
-
-  devicehub-id-server:
-    init: true
-    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp_docker__2c4b0006
-    environment:
-      - DB_USER=${DB_USER}
-      - DB_PASSWORD=${DB_PASSWORD}
-      - DB_HOST=postgres-id-server
-      - DB_DATABASE=${DB_DATABASE}
-      - HOST=${HOST}
-      - EMAIL_DEMO=user5000@dhub.com
-      - PASSWORD_DEMO=${PASSWORD_DEMO}
-      - JWT_PASS=${JWT_PASS}
-      - SECRET_KEY=${SECRET_KEY}
-      - API_DLT=${API_DLT}
-      - API_RESOLVER=${API_RESOLVER}
-      - API_DLT_TOKEN=${API_DLT_TOKEN}
-      - DEVICEHUB_HOST=${SERVER_ID_DEVICEHUB_HOST}
-      - ID_FEDERATED=${SERVER_ID_FEDERATED}
-      - URL_MANUALS=${URL_MANUALS}
-      - ID_SERVICE=${SERVER_ID_SERVICE}
-      - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST}
-    ports:
-      - 5000:5000
-    volumes:
-      - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
-      - shared:/shared:rw
-
-  postgres-id-server:
-    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp_docker__2c4b0006
-    # 4. To create the database.
-    # 5. Give permissions to the corresponding users in the database.
-    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
-    environment:
-      - POSTGRES_PASSWORD=${DB_PASSWORD}
-      - POSTGRES_USER=${DB_USER}
-      - POSTGRES_DB=${DB_DATABASE}
-    # DEBUG
-    #ports:
-    #  - 5432:5432
-    # TODO persistence
-    #volumes:
-    #  - pg_data:/var/lib/postgresql/data
-
-  devicehub-id-client:
-    init: true
-    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp_docker__2c4b0006
-    environment:
-      - DB_USER=${DB_USER}
-      - DB_PASSWORD=${DB_PASSWORD}
-      - DB_HOST=postgres-id-client
-      - DB_DATABASE=${DB_DATABASE}
-      - HOST=${HOST}
-      - EMAIL_DEMO=user5001@dhub.com
-      - PASSWORD_DEMO=${PASSWORD_DEMO}
-      - JWT_PASS=${JWT_PASS}
-      - SECRET_KEY=${SECRET_KEY}
-      - API_DLT=${API_DLT}
-      - API_RESOLVER=${API_RESOLVER}
-      - API_DLT_TOKEN=${API_DLT_TOKEN}
-      - DEVICEHUB_HOST=${CLIENT_ID_DEVICEHUB_HOST}
-      - SERVER_ID_HOST=${SERVER_ID_DEVICEHUB_HOST}
-      - ID_FEDERATED=${CLIENT_ID_FEDERATED}
-      - URL_MANUALS=${URL_MANUALS}
-      - ID_SERVICE=${CLIENT_ID_SERVICE}
-    ports:
-      - 5001:5000
-    volumes:
-      - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
-      - shared:/shared:ro
-
-  postgres-id-client:
-    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp_docker__2c4b0006
-    # 4. To create the database.
-    # 5. Give permissions to the corresponding users in the database.
-    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
-    environment:
-      - POSTGRES_PASSWORD=${DB_PASSWORD}
-      - POSTGRES_USER=${DB_USER}
-      - POSTGRES_DB=${DB_DATABASE}
-    # DEBUG
-    #ports:
-    #  - 5432:5432
-    # TODO persistence
-    #volumes:
-    #  - pg_data:/var/lib/postgresql/data
-
-
-    # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
-    #nginx
-
-volumes:
-  shared:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1 @@
+docker-compose_devicehub-dpp.yml
--- a/docker-compose_devicehub-dpp.yml
+++ b/docker-compose_devicehub-dpp.yml
@ -0,0 +1,99 @@
+version: "3.9"
+services:
+
+  devicehub-id-server:
+    init: true
+    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp__285635b7
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres-id-server
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${SERVER_ID_EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - API_DLT=${API_DLT}
+      - API_RESOLVER=${API_RESOLVER}
+      - API_DLT_TOKEN=${API_DLT_TOKEN}
+      - DEVICEHUB_HOST=${SERVER_ID_DEVICEHUB_HOST}
+      - ID_FEDERATED=${SERVER_ID_FEDERATED}
+      - URL_MANUALS=${URL_MANUALS}
+      - ID_SERVICE=${SERVER_ID_SERVICE}
+      - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST}
+      - DPP_MODULE=y
+      - IMPORT_SNAPSHOTS=${IMPORT_SNAPSHOTS}
+    ports:
+      - 5000:5000
+    volumes:
+      - ${SNAPSHOTS_PATH:-./examples/snapshots}:/mnt/snapshots:ro
+      - shared:/shared:rw
+
+  postgres-id-server:
+    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp__285635b7
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    # DEBUG
+    #ports:
+    #  - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+  devicehub-id-client:
+    init: true
+    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp__285635b7
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres-id-client
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${CLIENT_ID_EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - API_DLT=${API_DLT}
+      - API_RESOLVER=${API_RESOLVER}
+      - API_DLT_TOKEN=${API_DLT_TOKEN}
+      - DEVICEHUB_HOST=${CLIENT_ID_DEVICEHUB_HOST}
+      - SERVER_ID_HOST=${SERVER_ID_DEVICEHUB_HOST}
+      - ID_FEDERATED=${CLIENT_ID_FEDERATED}
+      - URL_MANUALS=${URL_MANUALS}
+      - ID_SERVICE=${CLIENT_ID_SERVICE}
+      - DPP_MODULE=y
+      - IMPORT_SNAPSHOTS=${IMPORT_SNAPSHOTS}
+    ports:
+      - 5001:5000
+    volumes:
+      - ${SNAPSHOTS_PATH:-./examples/snapshots}:/mnt/snapshots:ro
+      - shared:/shared:ro
+
+  postgres-id-client:
+    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp__285635b7
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    # DEBUG
+    #ports:
+    #  - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+
+    # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+    #nginx
+
+volumes:
+  shared:
--- a/docker-compose_devicehub.yml
+++ b/docker-compose_devicehub.yml
@ -0,0 +1,48 @@
+version: "3.9"
+services:
+
+  devicehub:
+    init: true
+    image: dkr-dsg.ac.upc.edu/ereuse/devicehub:dpp__285635b7
+    environment:
+      - DB_USER=${DB_USER}
+      - DB_PASSWORD=${DB_PASSWORD}
+      - DB_HOST=postgres
+      - DB_DATABASE=${DB_DATABASE}
+      - HOST=${HOST}
+      - EMAIL_DEMO=${EMAIL_DEMO}
+      - PASSWORD_DEMO=${PASSWORD_DEMO}
+      - JWT_PASS=${JWT_PASS}
+      - SECRET_KEY=${SECRET_KEY}
+      - DEVICEHUB_HOST=${DEVICEHUB_HOST}
+      - URL_MANUALS=${URL_MANUALS}
+      - AUTHORIZED_CLIENT_URL=${CLIENT_ID_DEVICEHUB_HOST}
+      - DPP_MODULE=n
+      - IMPORT_SNAPSHOTS=${IMPORT_SNAPSHOTS}
+    ports:
+      - 5000:5000
+    volumes:
+      - ${SNAPSHOTS_PATH:-./examples/snapshots}:/mnt/snapshots:ro
+      - shared:/shared:rw
+
+  postgres:
+    image: dkr-dsg.ac.upc.edu/ereuse/postgres:dpp__285635b7
+    # 4. To create the database.
+    # 5. Give permissions to the corresponding users in the database.
+    #   extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
+    environment:
+      - POSTGRES_PASSWORD=${DB_PASSWORD}
+      - POSTGRES_USER=${DB_USER}
+      - POSTGRES_DB=${DB_DATABASE}
+    # DEBUG
+    #ports:
+    #  - 5432:5432
+    # TODO persistence
+    #volumes:
+    #  - pg_data:/var/lib/postgresql/data
+
+    # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
+    #nginx
+
+volumes:
+  shared:
--- a/docker/devicehub.entrypoint.sh
+++ b/docker/devicehub.entrypoint.sh
@ -7,16 +7,24 @@ set -x

 # 3. Generate an environment .env file.
 gen_env_vars() {
-        # generate config using env vars from docker
-        cat > .env <<END
-DB_USER='${DB_USER}'
-DB_PASSWORD='${DB_PASSWORD}'
-DB_HOST='${DB_HOST}'
-DB_DATABASE='${DB_DATABASE}'
+        # specific dpp env vars
+        if [ "${DPP_MODULE}" = 'y' ]; then
+                dpp_env_vars="$(cat <<END
 API_DLT='${API_DLT}'
 API_DLT_TOKEN='${API_DLT_TOKEN}'
 API_RESOLVER='${API_RESOLVER}'
 ID_FEDERATED='${ID_FEDERATED}'
+END
+)"
+        fi
+
+        # generate config using env vars from docker
+        cat > .env <<END
+${dpp_env_vars:-}
+DB_USER='${DB_USER}'
+DB_PASSWORD='${DB_PASSWORD}'
+DB_HOST='${DB_HOST}'
+DB_DATABASE='${DB_DATABASE}'
 URL_MANUALS='${URL_MANUALS}'

 HOST='${HOST}'
@ -145,33 +153,48 @@ config_oidc() {
        fi
 }

+config_dpp_part1() {
+        # 12. Add a new server to the 'api resolver'
+        handle_federated_id
+
+        # 13. Do a rsync api resolve
+        flask dlt_rsync_members
+
+        # 14. Register a new user to the DLT
+        flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
+}
+
+config_dpp_part2() {
+        # 16.
+        flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+        # 20. config server or client ID
+        config_oidc
+}
+
 config_phase() {
        init_flagfile='/already_configured'
        if [ ! -f "${init_flagfile}" ]; then
                # 7, 8, 9, 11
                init_data

-                # 12. Add a new server to the 'api resolver'
-                handle_federated_id
-
-                # 13. Do a rsync api resolve
-                flask dlt_rsync_members
-
-                # 14. Register a new user to the DLT
-                flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
+                if [ "${DPP_MODULE}" = 'y' ]; then
+                        # 12, 13, 14
+                        config_dpp_part1
+                fi

                # non DL user (only for the inventory)
                #   flask adduser user2@dhub.com ${PASSWORD_DEMO}

                # # 15. Add inventory snapshots for user "${EMAIL_DEMO}".
-                cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
-                /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+                if [ "${IMPORT_SNAPSHOTS}" = 'y' ]; then
+                        cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
+                        /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
+                fi

-                # # 16.
-                flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
-
-                # config server or client ID
-                config_oidc
+                if [ "${DPP_MODULE}" = 'y' ]; then
+                        # 16, 20
+                        config_dpp_part2
+                fi

                # remain next command as the last operation for this if conditional
                touch "${init_flagfile}"
--- a/examples/env.example
+++ b/examples/env.example
@ -4,6 +4,9 @@ API_DLT_TOKEN=$TOKEN
 API_RESOLVER='http://$IP_API_RESOLVER'
 # you might change or register ID_FEDERATED if you change DEVICEHUB_HOST
 ID_FEDERATED='DH12'
+# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
+SERVER_ID_FEDERATED='DH12'
+CLIENT_ID_FEDERATED='DH20'

 # Database Variables
 DB_USER='dhub'
@ -13,10 +16,7 @@ DB_DATABASE='dpp'
 SCHEMA='dbtest'
 DB_SCHEMA='dbtest'

-# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
-SERVER_ID_FEDERATED='DH12'
-CLIENT_ID_FEDERATED='DH20'
-
+DEVICEHUB_HOST='http://localhost:5000'
 #SERVER_ID_DEVICEHUB_HOST='http://devicehub-server-id.example.com'
 SERVER_ID_DEVICEHUB_HOST='http://localhost:5000'
 #CLIENT_ID_DEVICEHUB_HOST='http://devicehub-client-id.example.com'
@ -26,6 +26,8 @@ CLIENT_ID_SERVICE='client_id'
 HOST='localhost'

 EMAIL_DEMO='user@example.com'
+SERVER_ID_EMAIL_DEMO='user5000@example.com'
+CLIENT_ID_EMAIL_DEMO='user5001@example.com'
 PASSWORD_DEMO='1234'

 JWT_PASS='aaaa'
@ -33,9 +35,10 @@ SECRET_KEY='aaaa'

 # important to import snapshots (step 15)
 #   rel path starts with ./
-#SNAPSHOTS_PATH='./relpath/to/snapshots'
+SNAPSHOTS_PATH='./examples/snapshots'
 #   full path starts with /
-SNAPSHOTS_PATH='/tmp/dhub_docker/snapshots'
+#SNAPSHOTS_PATH='/tmp/dhub_docker/snapshots'
+IMPORT_SNAPSHOTS='y'

 # If you have a URL_MANUALS implementation, please change this url
 URL_MANUALS='http://localhost:4000'
--- a/examples/snapshots/snapshot01.json
+++ b/examples/snapshots/snapshot01.json