add basic dockerization to devicehub dpp

This commit is contained in:
pedro 2023-09-19 18:03:23 +02:00
parent 0b70f42daa
commit 907bf2dba0
9 changed files with 308 additions and 1 deletions

3
.gitignore vendored
View File

@ -136,3 +136,6 @@ examples/create-db2.sh
package-lock.json package-lock.json
snapshots/ snapshots/
modules/ modules/
# emacs
*~

30
Makefile Normal file
View File

@ -0,0 +1,30 @@
project := dkr-dsg.ac.upc.edu/devicehub
branch := `git branch --show-current`
commit := `git log -1 --format=%h`
tag := ${branch}__${commit}
# docker images
devicehub_image := ${project}/devicehub:${tag}
postgres_image := ${project}/postgres:${tag}
# 2. Create a virtual environment.
docker_build:
docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} .
# DEBUG
#docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . --progress=plain --no-cache
docker build -f docker/postgres.Dockerfile -t ${postgres_image} .
# DEBUG
#docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache
docker_publish:
docker push ${devicehub_image}
.PHONY: docker
docker:
$(MAKE) docker_build
#$(MAKE) docker_publish
@printf "\nimage: ${devicehub_image}\n"
@printf "\nimage: ${postgres_image}\n"
@printf "\ndocker images built and published\n"

47
docker-compose.yml Normal file
View File

@ -0,0 +1,47 @@
version: "3.9"
services:
devicehub:
init: true
# TODO
image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b
#build .
environment:
- DB_USER=${DB_USER}
- DB_PASSWORD=${DB_PASSWORD}
- DB_HOST=postgres
- DB_DATABASE=${DB_DATABASE}
- HOST=${HOST}
- EMAIL_DEMO=${EMAIL_DEMO}
- PASSWORD_DEMO=${PASSWORD_DEMO}
- JWT_PASS=${JWT_PASS}
- SECRET_KEY=${SECRET_KEY}
- API_DLT=${API_DLT}
- API_RESOLVER=${API_RESOLVER}
- API_DLT_TOKEN=${API_DLT_TOKEN}
- DEVICEHUB_HOST=${DEVICEHUB_HOST}
- ID_FEDERATED=${ID_FEDERATED}
- URL_MANUALS=${URL_MANUALS}
ports:
- 5000:5000
volumes:
- ${SNAPSHOTS_PATH}:/mnt/snapshots:ro
postgres:
image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b
# 4. To create the database.
# 5. Give permissions to the corresponding users in the database.
# extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables
environment:
- POSTGRES_PASSWORD=${DB_PASSWORD}
- POSTGRES_USER=${DB_USER}
- POSTGRES_DB=${DB_DATABASE}
ports:
- 5432:5432
# TODO persistence
#volumes:
# - pg_data:/var/lib/postgresql/data
# TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
#nginx

View File

@ -0,0 +1,30 @@
FROM debian:bullseye-slim
RUN apt update && apt-get install --no-install-recommends -y \
python3-minimal \
python3-pip \
python-is-python3 \
python3-psycopg2 \
python3-dev \
libpq-dev \
build-essential \
libpangocairo-1.0-0 \
curl \
jq \
time \
netcat
WORKDIR /opt/devicehub
# this is exactly the same as examples/pip_install.sh except the last command
# to improve the docker layer builds, it has been separated
RUN pip install --upgrade pip
RUN pip install alembic==1.8.1 anytree==2.8.0 apispec==0.39.0 atomicwrites==1.4.0 blinker==1.5 boltons==23.0.0 cairocffi==1.4.0 cairosvg==2.5.2 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.0.12 click==6.7 click-spinner==0.1.8 colorama==0.3.9 colour==0.1.5 cssselect2==0.7.0 defusedxml==0.7.1 et-xmlfile==1.1.0 flask==1.0.2 flask-cors==3.0.10 flask-login==0.5.0 flask-sqlalchemy==2.5.1 flask-weasyprint==0.4 flask-wtf==1.0.0 hashids==1.2.0 html5lib==1.1 idna==3.4 inflection==0.5.1 itsdangerous==2.0.1 jinja2==3.0.3 mako==1.2.3 markupsafe==2.1.1 marshmallow==3.0.0b11 marshmallow-enum==1.4.1 more-itertools==8.12.0 numpy==1.22.0 odfpy==1.4.1 openpyxl==3.0.10 pandas==1.3.5 passlib==1.7.1 phonenumbers==8.9.11 pillow==9.2.0 pint==0.9 psycopg2-binary==2.8.3 py-dmidecode==0.1.0 pycparser==2.21 pyjwt==2.4.0 pyphen==0.13.0 python-dateutil==2.7.3 python-decouple==3.3 python-dotenv==0.14.0 python-editor==1.0.4 python-stdnum==1.9 pytz==2022.2.1 pyyaml==5.4 requests==2.27.1 requests-mock==1.5.2 requests-toolbelt==0.9.1 six==1.16.0 sortedcontainers==2.1.0 sqlalchemy==1.3.24 sqlalchemy-citext==1.3.post0 sqlalchemy-utils==0.33.11 tinycss2==1.1.1 tqdm==4.32.2 urllib3==1.26.12 weasyprint==44 webargs==5.5.3 webencodings==0.5.1 werkzeug==2.0.3 wtforms==3.0.1 xlrd==2.0.1 cryptography==39.0.1 Authlib==1.2.1 gunicorn==21.2.0
RUN pip install -i https://test.pypi.org/simple/ ereuseapitest==0.0.8
COPY . .
RUN pip install -e .
COPY docker/devicehub.entrypoint.sh .
ENTRYPOINT sh ./devicehub.entrypoint.sh

View File

@ -0,0 +1,12 @@
.git
.env
# TODO need to comment it to copy the entrypoint
#docker
Makefile
# Emacs backup files
*~
.\#*
# Vim swap files
*.swp
*.swo

154
docker/devicehub.entrypoint.sh Executable file
View File

@ -0,0 +1,154 @@
#!/bin/sh
set -e
set -u
# DEBUG
set -x
# 3. Generate an environment .env file.
gen_env_vars() {
# generate config using env vars from docker
cat > .env <<END
DB_USER=${DB_USER}
DB_PASSWORD=${DB_PASSWORD}
DB_HOST=${DB_HOST}
DB_DATABASE=${DB_DATABASE}
SCHEMA=dbtest
DB_SCHEMA=dbtest
HOST=${HOST}
EMAIL_DEMO=${EMAIL_DEMO}
PASSWORD_DEMO=${PASSWORD_DEMO}
JWT_PASS=${JWT_PASS}
SECRET_KEY=${SECRET_KEY}
API_DLT=${API_DLT}
API_RESOLVER=${API_RESOLVER}
API_DLT_TOKEN=${API_DLT_TOKEN}
ID_FEDERATED=${ID_FEDERATED}
OAUTH2_JWT_ENABLED=True
OAUTH2_JWT_ISS=https://authlib.org
OAUTH2_JWT_KEY=secret-key
OAUTH2_JWT_ALG=HS256
URL_MANUALS=${URL_MANUALS}
END
}
wait_for_postgres() {
# old one was
#sleep 4
default_postgres_port=5432
# thanks https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/
while ! nc -z ${DB_HOST} ${default_postgres_port}; do
sleep 0.5
done
}
init_data() {
# 7. Run alembic of the project.
alembic -x inventory=dbtest upgrade head
# 8. Running alembic from oidc module.y
cd ereuse_devicehub/modules/oidc
alembic -x inventory=dbtest upgrade head
cd -
# 9. Running alembic from dpp module.
cd ereuse_devicehub/modules/dpp/
alembic -x inventory=dbtest upgrade head
cd -
# 11. Generate a minimal data structure.
# TODO it has some errors (?)
flask initdata || true
}
big_error() {
local message="${@}"
echo "###############################################" >&2
echo "# ERROR: ${message}" >&2
echo "###############################################" >&2
exit 1
}
handle_federated_id() {
# devicehub host and id federated checker
EXPECTED_ID_FEDERATED="$(curl -s "${API_RESOLVER}/getAll" \
| jq -r '.url | to_entries | .[] | select(.value == "'"${DEVICEHUB_HOST}"'") | .key' \
| head -n 1)"
# if is a new DEVICEHUB_HOST, then register it
if [ -z "${EXPECTED_ID_FEDERATED}" ]; then
# TODO better docker compose run command
cmd="docker compose run --entrypoint= devicehub flask dlt_insert_members ${DEVICEHUB_HOST}"
big_error "No FEDERATED ID maybe you should run \`${cmd}\`"
fi
# if not new DEVICEHUB_HOST, then check consistency
# if there is already an ID in the DLT, it should match with my internal ID
if [ ! "${EXPECTED_ID_FEDERATED}" = "${ID_FEDERATED}" ]; then
big_error "ID_FEDERATED should be ${EXPECTED_ID_FEDERATED} instead of ${ID_FEDERATED}"
fi
# not needed, but reserved
# EXPECTED_DEVICEHUB_HOST="$(curl -s "${API_RESOLVER}/getAll" \
# | jq -r '.url | to_entries | .[] | select(.key == "'"${ID_FEDERATED}"'") | .value' \
# | head -n 1)"
# if [ ! "${EXPECTED_DEVICEHUB_HOST}" = "${DEVICEHUB_HOST}" ]; then
# big_error "ERROR: DEVICEHUB_HOST should be ${EXPECTED_DEVICEHUB_HOST} instead of ${DEVICEHUB_HOST}"
# fi
}
main() {
gen_env_vars
wait_for_postgres
init_flagfile='/container_initialized'
if [ ! -f "${init_flagfile}" ]; then
# 7, 8, 9, 11
init_data
# 12. Add a new server to the 'api resolver'
handle_federated_id
# 13. Do a rsync api resolve
flask dlt_rsync_members
# 14. Register a new user to the DLT
flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator
# non DL user (only for the inventory)
# flask adduser user2@dhub.com ${PASSWORD_DEMO}
# # 15. Add inventory snapshots for user "${EMAIL_DEMO}".
cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files
/usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO}
# # 16.
flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO}
# remain next command as the last operation for this if conditional
touch "${init_flagfile}"
fi
# 17. Use gunicorn
# thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html
# TODO meanwhile no nginx (step 19), gunicorn cannot serve static files, then we prefer development server
#gunicorn --access-logfile - --error-logfile - --workers 4 -b :5000 app:app
# alternative: run development server
flask run --host=0.0.0.0 --port 5000
# DEBUG
#sleep infinity
}
main "${@}"

View File

@ -0,0 +1,8 @@
FROM postgres:15.4-bookworm
# this is the latest in 2023-09-14_13-01-38
#FROM postgres:latest
# Add a SQL script that will be executed upon container startup
COPY docker/postgres.setupdb.sql /docker-entrypoint-initdb.d/
EXPOSE 5432

View File

@ -0,0 +1,5 @@
-- 6. Create the necessary extensions.
CREATE EXTENSION pgcrypto SCHEMA public;
CREATE EXTENSION ltree SCHEMA public;
CREATE EXTENSION citext SCHEMA public;
CREATE EXTENSION pg_trgm SCHEMA public;

View File

@ -5,6 +5,24 @@ DB_DATABASE='devicehub'
API_DLT='http://$IP_API_DLT' API_DLT='http://$IP_API_DLT'
API_DLT_TOKEN=$TOKEN API_DLT_TOKEN=$TOKEN
API_RESOLVER='http://$IP_API_RESOLVER' API_RESOLVER='http://$IP_API_RESOLVER'
ID_FEDERATED='DH12' # TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED
ID_FEDERATED='$ID'
URL_MANUALS='http://$IP_MANUALS' URL_MANUALS='http://$IP_MANUALS'
DEVICEHUB_HOST='http://devicehub.example.com'
HOST='localhost'
SCHEMA='dbtest'
DB_SCHEMA='dbtest'
EMAIL_DEMO='user@example.org'
PASSWORD_DEMO='changeme'
JWT_PASS='changeme'
SECRET_KEY='changeme'
# important to import snapshots (step 15)
# rel path starts with ./
#SNAPSHOTS_PATH='./relpath/to/snapshots'
# full path starts with /
SNAPSHOTS_PATH='/fullpath/to/snapshots'