root: migrate pipenv to poetry (#1995)

.bumpversion.cfg

@@ -17,6 +17,8 @@ values =
 	beta
 	stable
 
+[bumpversion:file:pyproject.toml]
+
 [bumpversion:file:docker-compose.yml]
 
 [bumpversion:file:schema.yml]

.github/workflows/ci-main.yml

@@ -33,40 +33,36 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
-      - id: cache-pipenv
+      - id: cache-poetry
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: scripts/ci_prepare.sh
       - name: run job
-        run: pipenv run make ci-${{ matrix.job }}
+        run: poetry run make ci-${{ matrix.job }}
   test-migrations:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
+      - id: cache-poetry
-          python-version: '3.9'
-      - id: cache-pipenv
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: scripts/ci_prepare.sh
       - name: run migrations
-        run: pipenv run python -m lifecycle.migrate
+        run: poetry run python -m lifecycle.migrate
   test-migrations-from-stable:
     runs-on: ubuntu-latest
     steps:
@@ -74,75 +70,79 @@ jobs:
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
       - name: prepare variables
         id: ev
         run: |
           python ./scripts/gh_env.py
-      - id: cache-pipenv
+          sudo pip install -U pipenv
+      - id: cache-poetry
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: checkout stable
-        id: stable
         run: |
-          # Save current branch
-          current=$(git branch --show)
-          echo ##[set-output name=originalBranch]$current
           # Copy current, latest config to local
           cp authentik/lib/default.yml local.env.yml
           cp -R .github ..
           cp -R scripts ..
+          cp -R poetry.lock pyproject.toml ..
           git checkout $(git describe --abbrev=0 --match 'version/*')
           rm -rf .github/ scripts/
-          mv ../.github ../scripts .
+          mv ../.github ../scripts ../poetry.lock ../pyproject.toml .
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: |
           scripts/ci_prepare.sh
           # Sync anyways since stable will have different dependencies
-          pipenv sync --dev
+          # TODO: Remove after next stable release
+          if [[ -f "Pipfile.lock" ]]; then
+            pipenv install --dev
+          fi
+          poetry install
       - name: run migrations to stable
-        run: pipenv run python -m lifecycle.migrate
+        run: poetry run python -m lifecycle.migrate
       - name: checkout current code
         run: |
           set -x
           git fetch
           git reset --hard HEAD
-          git checkout ${{ steps.stable.outputs.originalBranch }}
+          # TODO: Remove after next stable release
-          pipenv sync --dev
+          rm -f poetry.lock
+          git checkout $GITHUB_SHA
+          # TODO: Remove after next stable release
+          if [[ -f "Pipfile.lock" ]]; then
+            pipenv install --dev
+          fi
+          poetry install
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: scripts/ci_prepare.sh
       - name: migrate to latest
-        run: pipenv run python -m lifecycle.migrate
+        run: poetry run python -m lifecycle.migrate
   test-unittest:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
+      - id: cache-poetry
-          python-version: '3.9'
-      - id: cache-pipenv
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: scripts/ci_prepare.sh
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
       - name: run unittest
         run: |
-          pipenv run make test
+          poetry run make test
-          pipenv run coverage xml
+          poetry run coverage xml
       - name: run testspace
         if: ${{ always() }}
         run: |
@@ -154,16 +154,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
+      - id: cache-poetry
-          python-version: '3.9'
-      - id: cache-pipenv
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: scripts/ci_prepare.sh
       - uses: testspace-com/setup-testspace@v1
         with:
@@ -172,8 +170,8 @@ jobs:
         uses: helm/kind-action@v1.2.0
       - name: run integration
         run: |
-          pipenv run make test-integration
+          poetry run make test-integration
-          pipenv run coverage xml
+          poetry run coverage xml
       - name: run testspace
         if: ${{ always() }}
         run: |
@@ -185,8 +183,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
@@ -195,14 +191,14 @@ jobs:
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
-      - id: cache-pipenv
+      - id: cache-poetry
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: |
           scripts/ci_prepare.sh
           docker-compose -f tests/e2e/docker-compose.yml up -d
@@ -219,8 +215,8 @@ jobs:
           npm run build
       - name: run e2e
         run: |
-          pipenv run make test-e2e-provider
+          poetry run make test-e2e-provider
-          pipenv run coverage xml
+          poetry run coverage xml
       - name: run testspace
         if: ${{ always() }}
         run: |
@@ -232,8 +228,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9'
       - uses: actions/setup-node@v2
         with:
           node-version: '16'
@@ -242,14 +236,14 @@ jobs:
       - uses: testspace-com/setup-testspace@v1
         with:
           domain: ${{github.repository_owner}}
-      - id: cache-pipenv
+      - id: cache-poetry
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: |
           scripts/ci_prepare.sh
           docker-compose -f tests/e2e/docker-compose.yml up -d
@@ -266,8 +260,8 @@ jobs:
           npm run build
       - name: run e2e
         run: |
-          pipenv run make test-e2e-rest
+          poetry run make test-e2e-rest
-          pipenv run coverage xml
+          poetry run coverage xml
       - name: run testspace
         if: ${{ always() }}
         run: |

.github/workflows/translation-compile.yml

@@ -22,22 +22,20 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-        with:
+      - id: cache-poetry
-          python-version: '3.9'
-      - id: cache-pipenv
         uses: actions/cache@v2.1.7
         with:
-          path: ~/.local/share/virtualenvs
+          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-pipenv-v2-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ runner.os }}-poetry-cache-v3-${{ hashFiles('**/poetry.lock') }}
       - name: prepare
         env:
-          INSTALL: ${{ steps.cache-pipenv.outputs.cache-hit }}
+          INSTALL: ${{ steps.cache-poetry.outputs.cache-hit }}
         run: |
           sudo apt-get update
           sudo apt-get install -y gettext
           scripts/ci_prepare.sh
       - name: run compile
-        run: pipenv run ./manage.py compilemessages
+        run: poetry run make ./manage.py compilemessages
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         id: cpr

Dockerfile

@@ -1,16 +1,4 @@
-# Stage 1: Lock python dependencies
+# Stage 1: Build website
-FROM docker.io/python:3.10.1-slim-bullseye as locker
-
-COPY ./Pipfile /app/
-COPY ./Pipfile.lock /app/
-
-WORKDIR /app/
-
-RUN pip install pipenv && \
-    pipenv lock -r > requirements.txt && \
-    pipenv lock -r --dev-only > requirements-dev.txt
-
-# Stage 2: Build website
 FROM --platform=${BUILDPLATFORM} docker.io/node:16 as website-builder
 
 COPY ./website /work/website/
@@ -18,7 +6,7 @@ COPY ./website /work/website/
 ENV NODE_ENV=production
 RUN cd /work/website && npm i && npm run build-docs-only
 
-# Stage 3: Build webui
+# Stage 2: Build webui
 FROM --platform=${BUILDPLATFORM} docker.io/node:16 as web-builder
 
 COPY ./web /work/web/
@@ -27,7 +15,7 @@ COPY ./website /work/website/
 ENV NODE_ENV=production
 RUN cd /work/web && npm i && npm run build
 
-# Stage 4: Build go proxy
+# Stage 3: Build go proxy
 FROM docker.io/golang:1.17.5-bullseye AS builder
 
 WORKDIR /work
@@ -43,22 +31,26 @@ COPY ./go.sum /work/go.sum
 
 RUN go build -o /work/authentik ./cmd/server/main.go
 
-# Stage 5: Run
+# Stage 4: Run
 FROM docker.io/python:3.10.1-slim-bullseye
 
 WORKDIR /
-COPY --from=locker /app/requirements.txt /
-COPY --from=locker /app/requirements-dev.txt /
 
 ARG GIT_BUILD_HASH
 ENV GIT_BUILD_HASH=$GIT_BUILD_HASH
 
+COPY ./pyproject.toml /
+COPY ./poetry.lock /
+
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl ca-certificates gnupg git runit libpq-dev \
         postgresql-client build-essential libxmlsec1-dev \
         pkg-config libmaxminddb0 && \
-    pip install -r /requirements.txt --no-cache-dir && \
+    pip install poetry && \
+    poetry config virtualenvs.create false && \
+    poetry install --no-dev && \
+    rm -rf ~/.cache/pypoetry && \
     apt-get remove --purge -y build-essential git && \
     apt-get autoremove --purge -y && \
     apt-get clean && \

Makefile

@@ -106,20 +106,24 @@ web-extract:
 # These targets are use by GitHub actions to allow usage of matrix
 # which makes the YAML File a lot smaller
 
-ci-pylint:
+ci--meta-debug:
+	python -V
+	node --version
+
+ci-pylint: ci--meta-debug
 	pylint authentik tests lifecycle
 
-ci-black:
+ci-black: ci--meta-debug
 	black --check authentik tests lifecycle
 
-ci-isort:
+ci-isort: ci--meta-debug
 	isort --check authentik tests lifecycle
 
-ci-bandit:
+ci-bandit: ci--meta-debug
 	bandit -r authentik tests lifecycle
 
-ci-pyright:
+ci-pyright: ci--meta-debug
 	pyright e2e lifecycle
 
-ci-pending-migrations:
+ci-pending-migrations: ci--meta-debug
 	./manage.py makemigrations --check

Pipfile

@@ -1,68 +0,0 @@
-[[source]]
-name = "pypi"
-url = "https://pypi.org/simple"
-verify_ssl = true
-
-[packages]
-boto3 = "*"
-celery = "*"
-channels = "*"
-channels-redis = "*"
-codespell = "*"
-colorama = "*"
-dacite = "*"
-deepmerge = "*"
-defusedxml = "*"
-django = "*"
-django-dbbackup = { git = 'https://github.com/django-dbbackup/django-dbbackup.git', ref = '9d1909c30a3271c8c9c8450add30d6e0b996e145' }
-django-filter = "*"
-django-guardian = "*"
-django-model-utils = "*"
-django-otp = "*"
-django-prometheus = "*"
-django-redis = "*"
-django-storages = "*"
-djangorestframework = "*"
-djangorestframework-guardian = "*"
-docker = "*"
-drf-spectacular = "*"
-duo-client = "*"
-facebook-sdk = "*"
-geoip2 = "*"
-gunicorn = "*"
-kubernetes = "==v19.15.0"
-ldap3 = "*"
-lxml = "*"
-packaging = "*"
-psycopg2-binary = "*"
-pycryptodome = "*"
-pyjwt = "*"
-pyyaml = "*"
-requests-oauthlib = "*"
-sentry-sdk = { git = 'https://github.com/beryju/sentry-python.git', ref = '379aee28b15d3b87b381317746c4efd24b3d7bc3' }
-service_identity = "*"
-structlog = "*"
-swagger-spec-validator = "*"
-twisted = "==21.7.0"
-ua-parser = "*"
-urllib3 = {extras = ["secure"],version = "*"}
-uvicorn = {extras = ["standard"],version = "*"}
-webauthn = "*"
-xmlsec = "*"
-flower = "*"
-wsproto = "*"
-
-[dev-packages]
-bandit = "*"
-black = "==21.11b1"
-bump2version = "*"
-colorama = "*"
-coverage = {extras = ["toml"],version = "*"}
-pylint = "*"
-pylint-django = "*"
-pytest = "*"
-pytest-django = "*"
-pytest-randomly = "*"
-requests-mock = "*"
-selenium = "*"
-importlib-metadata = "*"

lifecycle/ak

@@ -67,7 +67,7 @@ if [[ "$1" == "server" ]]; then
     /authentik-proxy
 elif [[ "$1" == "worker" ]]; then
     echo "worker" > $MODE_FILE
-    check_if_root "celery -A authentik.root.celery worker -Ofair --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q authentik,authentik_scheduled,authentik_events"
+    check_if_root "celery -A authentik.root.celery worker -Ofair --max-tasks-per-child=1 --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q authentik,authentik_scheduled,authentik_events"
 elif [[ "$1" == "flower" ]]; then
     echo "flower" > $MODE_FILE
     celery -A authentik.root.celery flower
@@ -80,7 +80,7 @@ elif [[ "$1" == "restore" ]]; then
 elif [[ "$1" == "bash" ]]; then
     /bin/bash
 elif [[ "$1" == "test" ]]; then
-    pip install --no-cache -r requirements-dev.txt
+    poetry install
     touch /unittest.xml
     chown authentik:authentik /unittest.xml
     check_if_root "python -m manage test authentik"

pyproject.toml

@@ -89,3 +89,78 @@ DJANGO_SETTINGS_MODULE = "authentik.root.settings"
 python_files = ["tests.py", "test_*.py", "*_tests.py"]
 junit_family = "xunit2"
 addopts = "-p no:celery --junitxml=unittest.xml"
+
+[tool.poetry]
+name = "authentik"
+version = "2021.12.4"
+description = ""
+authors = ["Jens Langhammer <jens.langhammer@beryju.org>"]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+boto3 = "*"
+celery = "*"
+channels = "*"
+channels-redis = "*"
+codespell = "*"
+colorama = "*"
+dacite = "*"
+deepmerge = "*"
+defusedxml = "*"
+django = "*"
+django-dbbackup = { git = 'https://github.com/django-dbbackup/django-dbbackup.git', rev = '9d1909c30a3271c8c9c8450add30d6e0b996e145' }
+django-filter = "*"
+django-guardian = "*"
+django-model-utils = "*"
+django-otp = "*"
+django-prometheus = "*"
+django-redis = "*"
+django-storages = "*"
+djangorestframework = "*"
+djangorestframework-guardian = "*"
+docker = "*"
+drf-spectacular = "*"
+duo-client = "*"
+facebook-sdk = "*"
+geoip2 = "*"
+gunicorn = "*"
+kubernetes = "==v19.15.0"
+ldap3 = "*"
+lxml = "*"
+packaging = "*"
+psycopg2-binary = "*"
+pycryptodome = "*"
+pyjwt = "*"
+pyyaml = "*"
+requests-oauthlib = "*"
+sentry-sdk = { git = 'https://github.com/beryju/sentry-python.git', rev = '379aee28b15d3b87b381317746c4efd24b3d7bc3' }
+service_identity = "*"
+structlog = "*"
+swagger-spec-validator = "*"
+twisted = "==21.7.0"
+ua-parser = "*"
+urllib3 = {extras = ["secure"],version = "*"}
+uvicorn = {extras = ["standard"],version = "*"}
+webauthn = "*"
+xmlsec = "*"
+flower = "*"
+wsproto = "*"
+
+[tool.poetry.dev-dependencies]
+bandit = "*"
+black = "*"
+bump2version = "*"
+colorama = "*"
+coverage = {extras = ["toml"],version = "*"}
+pylint = "*"
+pylint-django = "*"
+pytest = "*"
+pytest-django = "*"
+pytest-randomly = "*"
+requests-mock = "*"
+selenium = "*"
+importlib-metadata = "*"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

scripts/ci_prepare.sh

@@ -1,10 +1,12 @@
 #!/bin/bash -xe
 docker-compose -f scripts/ci.docker-compose.yml up -d
+
 sudo apt update
 sudo apt install -y libxmlsec1-dev pkg-config
-sudo pip install -U wheel pipenv
+python3 -m pip install -U wheel poetry
+poetry env use python3.10
 if [[ "$INSTALL" != "true" ]]; then
-    pipenv install --dev
+    poetry install
 fi
-pipenv run python -m scripts.generate_ci_config
+poetry run python -m scripts.generate_ci_config
 npm install -g pyright@1.1.136

website/developer-docs/setup/full-dev-environment.md

@@ -9,7 +9,7 @@ To create a local development setup for authentik, you need the following:
 ### Requirements
 
 - Python 3.9
-- pipenv, which is used to manage dependencies, and can be installed with `pip install pipenv`
+- poetry, which is used to manage dependencies, and can be installed with `pip install poetry`
 - Go 1.16
 - PostgreSQL (any recent version will do)
 - Redis (any recent version will do)
@@ -19,8 +19,8 @@ For PostgreSQL and Redis, you can use the docker-compose file in `scripts/`. You
 ### Setup
 
 ```shell
-pipenv shell # Creates a python virtualenv, and activates it in a new shell
+poetry shell # Creates a python virtualenv, and activates it in a new shell
-pipenv sync --dev # Install all required dependencies, including development dependencies
+poetry install # Install all required dependencies, including development dependencies
 ```
 
 To configure authentik to use the local databases, create a file in the authentik directory called `local.env.yml`, with the following contents