diff --git a/.gitignore b/.gitignore index e7c3d207..39dff2b0 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,6 @@ examples/create-db2.sh package-lock.json snapshots/ modules/ + +# emacs +*~ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..abc81f99 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +project := dkr-dsg.ac.upc.edu/devicehub + +branch := `git branch --show-current` +commit := `git log -1 --format=%h` +tag := ${branch}__${commit} + +# docker images +devicehub_image := ${project}/devicehub:${tag} +postgres_image := ${project}/postgres:${tag} + +# 2. Create a virtual environment. +docker_build: + docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . + # DEBUG + #docker build -f docker/devicehub.Dockerfile -t ${devicehub_image} . --progress=plain --no-cache + + docker build -f docker/postgres.Dockerfile -t ${postgres_image} . + # DEBUG + #docker build -f docker/postgres.Dockerfile -t ${postgres_image} . --progress=plain --no-cache + +docker_publish: + docker push ${devicehub_image} + +.PHONY: docker +docker: + $(MAKE) docker_build + #$(MAKE) docker_publish + @printf "\nimage: ${devicehub_image}\n" + @printf "\nimage: ${postgres_image}\n" + @printf "\ndocker images built and published\n" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..5224073b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +version: "3.9" +services: + + devicehub: + init: true + # TODO + image: dkr-dsg.ac.upc.edu/devicehub/devicehub:dpp__0fb4fa5b + #build . + environment: + - DB_USER=${DB_USER} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=postgres + - DB_DATABASE=${DB_DATABASE} + - HOST=${HOST} + - EMAIL_DEMO=${EMAIL_DEMO} + - PASSWORD_DEMO=${PASSWORD_DEMO} + - JWT_PASS=${JWT_PASS} + - SECRET_KEY=${SECRET_KEY} + - API_DLT=${API_DLT} + - API_RESOLVER=${API_RESOLVER} + - API_DLT_TOKEN=${API_DLT_TOKEN} + - DEVICEHUB_HOST=${DEVICEHUB_HOST} + - ID_FEDERATED=${ID_FEDERATED} + - URL_MANUALS=${URL_MANUALS} + ports: + - 5000:5000 + volumes: + - ${SNAPSHOTS_PATH}:/mnt/snapshots:ro + + postgres: + image: dkr-dsg.ac.upc.edu/devicehub/postgres:dpp__0fb4fa5b + # 4. To create the database. + # 5. Give permissions to the corresponding users in the database. + # extra src https://github.com/docker-library/docs/blob/master/postgres/README.md#environment-variables + environment: + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_USER=${DB_USER} + - POSTGRES_DB=${DB_DATABASE} + ports: + - 5432:5432 + # TODO persistence + #volumes: + # - pg_data:/var/lib/postgresql/data + + + # TODO https://testdriven.io/blog/dockerizing-django-with-postgres-gunicorn-and-nginx/ + #nginx diff --git a/docker/devicehub.Dockerfile b/docker/devicehub.Dockerfile new file mode 100644 index 00000000..f14fc8a5 --- /dev/null +++ b/docker/devicehub.Dockerfile @@ -0,0 +1,30 @@ +FROM debian:bullseye-slim + +RUN apt update && apt-get install --no-install-recommends -y \ + python3-minimal \ + python3-pip \ + python-is-python3 \ + python3-psycopg2 \ + python3-dev \ + libpq-dev \ + build-essential \ + libpangocairo-1.0-0 \ + curl \ + jq \ + time \ + netcat + +WORKDIR /opt/devicehub + +# this is exactly the same as examples/pip_install.sh except the last command +# to improve the docker layer builds, it has been separated +RUN pip install --upgrade pip +RUN pip install alembic==1.8.1 anytree==2.8.0 apispec==0.39.0 atomicwrites==1.4.0 blinker==1.5 boltons==23.0.0 cairocffi==1.4.0 cairosvg==2.5.2 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.0.12 click==6.7 click-spinner==0.1.8 colorama==0.3.9 colour==0.1.5 cssselect2==0.7.0 defusedxml==0.7.1 et-xmlfile==1.1.0 flask==1.0.2 flask-cors==3.0.10 flask-login==0.5.0 flask-sqlalchemy==2.5.1 flask-weasyprint==0.4 flask-wtf==1.0.0 hashids==1.2.0 html5lib==1.1 idna==3.4 inflection==0.5.1 itsdangerous==2.0.1 jinja2==3.0.3 mako==1.2.3 markupsafe==2.1.1 marshmallow==3.0.0b11 marshmallow-enum==1.4.1 more-itertools==8.12.0 numpy==1.22.0 odfpy==1.4.1 openpyxl==3.0.10 pandas==1.3.5 passlib==1.7.1 phonenumbers==8.9.11 pillow==9.2.0 pint==0.9 psycopg2-binary==2.8.3 py-dmidecode==0.1.0 pycparser==2.21 pyjwt==2.4.0 pyphen==0.13.0 python-dateutil==2.7.3 python-decouple==3.3 python-dotenv==0.14.0 python-editor==1.0.4 python-stdnum==1.9 pytz==2022.2.1 pyyaml==5.4 requests==2.27.1 requests-mock==1.5.2 requests-toolbelt==0.9.1 six==1.16.0 sortedcontainers==2.1.0 sqlalchemy==1.3.24 sqlalchemy-citext==1.3.post0 sqlalchemy-utils==0.33.11 tinycss2==1.1.1 tqdm==4.32.2 urllib3==1.26.12 weasyprint==44 webargs==5.5.3 webencodings==0.5.1 werkzeug==2.0.3 wtforms==3.0.1 xlrd==2.0.1 cryptography==39.0.1 Authlib==1.2.1 gunicorn==21.2.0 + +RUN pip install -i https://test.pypi.org/simple/ ereuseapitest==0.0.8 + +COPY . . +RUN pip install -e . + +COPY docker/devicehub.entrypoint.sh . +ENTRYPOINT sh ./devicehub.entrypoint.sh diff --git a/docker/devicehub.Dockerfile.dockerignore b/docker/devicehub.Dockerfile.dockerignore new file mode 100644 index 00000000..57848430 --- /dev/null +++ b/docker/devicehub.Dockerfile.dockerignore @@ -0,0 +1,12 @@ +.git +.env +# TODO need to comment it to copy the entrypoint +#docker +Makefile + +# Emacs backup files +*~ +.\#* +# Vim swap files +*.swp +*.swo diff --git a/docker/devicehub.entrypoint.sh b/docker/devicehub.entrypoint.sh new file mode 100755 index 00000000..a7d25dbb --- /dev/null +++ b/docker/devicehub.entrypoint.sh @@ -0,0 +1,154 @@ +#!/bin/sh + +set -e +set -u +# DEBUG +set -x + +# 3. Generate an environment .env file. +gen_env_vars() { + # generate config using env vars from docker + cat > .env <&2 + echo "# ERROR: ${message}" >&2 + echo "###############################################" >&2 + exit 1 +} + +handle_federated_id() { + + # devicehub host and id federated checker + + EXPECTED_ID_FEDERATED="$(curl -s "${API_RESOLVER}/getAll" \ + | jq -r '.url | to_entries | .[] | select(.value == "'"${DEVICEHUB_HOST}"'") | .key' \ + | head -n 1)" + + # if is a new DEVICEHUB_HOST, then register it + if [ -z "${EXPECTED_ID_FEDERATED}" ]; then + # TODO better docker compose run command + cmd="docker compose run --entrypoint= devicehub flask dlt_insert_members ${DEVICEHUB_HOST}" + big_error "No FEDERATED ID maybe you should run \`${cmd}\`" + fi + + # if not new DEVICEHUB_HOST, then check consistency + + # if there is already an ID in the DLT, it should match with my internal ID + if [ ! "${EXPECTED_ID_FEDERATED}" = "${ID_FEDERATED}" ]; then + + big_error "ID_FEDERATED should be ${EXPECTED_ID_FEDERATED} instead of ${ID_FEDERATED}" + fi + + # not needed, but reserved + # EXPECTED_DEVICEHUB_HOST="$(curl -s "${API_RESOLVER}/getAll" \ + # | jq -r '.url | to_entries | .[] | select(.key == "'"${ID_FEDERATED}"'") | .value' \ + # | head -n 1)" + # if [ ! "${EXPECTED_DEVICEHUB_HOST}" = "${DEVICEHUB_HOST}" ]; then + # big_error "ERROR: DEVICEHUB_HOST should be ${EXPECTED_DEVICEHUB_HOST} instead of ${DEVICEHUB_HOST}" + # fi + +} + +main() { + + gen_env_vars + + wait_for_postgres + + init_flagfile='/container_initialized' + if [ ! -f "${init_flagfile}" ]; then + + # 7, 8, 9, 11 + init_data + + # 12. Add a new server to the 'api resolver' + handle_federated_id + + # 13. Do a rsync api resolve + flask dlt_rsync_members + + # 14. Register a new user to the DLT + flask dlt_register_user "${EMAIL_DEMO}" ${PASSWORD_DEMO} Operator + + # non DL user (only for the inventory) + # flask adduser user2@dhub.com ${PASSWORD_DEMO} + + # # 15. Add inventory snapshots for user "${EMAIL_DEMO}". + cp /mnt/snapshots/snapshot*.json ereuse_devicehub/commands/snapshot_files + /usr/bin/time flask snapshot "${EMAIL_DEMO}" ${PASSWORD_DEMO} + + # # 16. + flask check_install "${EMAIL_DEMO}" ${PASSWORD_DEMO} + + # remain next command as the last operation for this if conditional + touch "${init_flagfile}" + fi + + # 17. Use gunicorn + # thanks https://akira3030.github.io/formacion/articulos/python-flask-gunicorn-docker.html + # TODO meanwhile no nginx (step 19), gunicorn cannot serve static files, then we prefer development server + #gunicorn --access-logfile - --error-logfile - --workers 4 -b :5000 app:app + # alternative: run development server + flask run --host=0.0.0.0 --port 5000 + + # DEBUG + #sleep infinity +} + +main "${@}" diff --git a/docker/postgres.Dockerfile b/docker/postgres.Dockerfile new file mode 100644 index 00000000..7ed15cf8 --- /dev/null +++ b/docker/postgres.Dockerfile @@ -0,0 +1,8 @@ +FROM postgres:15.4-bookworm +# this is the latest in 2023-09-14_13-01-38 +#FROM postgres:latest + +# Add a SQL script that will be executed upon container startup +COPY docker/postgres.setupdb.sql /docker-entrypoint-initdb.d/ + +EXPOSE 5432 diff --git a/docker/postgres.setupdb.sql b/docker/postgres.setupdb.sql new file mode 100644 index 00000000..a6f104a3 --- /dev/null +++ b/docker/postgres.setupdb.sql @@ -0,0 +1,5 @@ +-- 6. Create the necessary extensions. +CREATE EXTENSION pgcrypto SCHEMA public; +CREATE EXTENSION ltree SCHEMA public; +CREATE EXTENSION citext SCHEMA public; +CREATE EXTENSION pg_trgm SCHEMA public; diff --git a/examples/env.example b/examples/env.example index bfa97453..68366898 100644 --- a/examples/env.example +++ b/examples/env.example @@ -5,6 +5,24 @@ DB_DATABASE='devicehub' API_DLT='http://$IP_API_DLT' API_DLT_TOKEN=$TOKEN API_RESOLVER='http://$IP_API_RESOLVER' -ID_FEDERATED='DH12' +# TODO this should be guessed by DEVICEHUB_HOST, and avoid hardcode of ID_FEDERATED +ID_FEDERATED='$ID' URL_MANUALS='http://$IP_MANUALS' +DEVICEHUB_HOST='http://devicehub.example.com' +HOST='localhost' + +SCHEMA='dbtest' +DB_SCHEMA='dbtest' + +EMAIL_DEMO='user@example.org' +PASSWORD_DEMO='changeme' + +JWT_PASS='changeme' +SECRET_KEY='changeme' + +# important to import snapshots (step 15) +# rel path starts with ./ +#SNAPSHOTS_PATH='./relpath/to/snapshots' +# full path starts with / +SNAPSHOTS_PATH='/fullpath/to/snapshots'