From f691c6896ff460b403add836f829daf1c1cf18d1 Mon Sep 17 00:00:00 2001 From: Ken Sternberg Date: Mon, 25 Sep 2023 12:50:59 -0700 Subject: [PATCH 1/8] core/allow alternative postgres credentials This commit allows the `dev-reset` command in the Makefile to pick up and use credentials from the `.env` file if they are present, or fallback to the defaults provided if they are not. This is the only place in the Makefile where the database credentials are used directly against postgresql binaries. The syntax was tested with bash, zsh, and csh, and did not fail under those. The `$${:-}` syntax is a combination of a Makefile idiom for "Pass a single `$` to the environment where this command will be executed," and the shell expresion `${VARIABLE:-default}` means "dereference the environment variable; if it is undefined, used the default value provided." --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 4750b17ef..45e46e5a1 100644 --- a/Makefile +++ b/Makefile @@ -219,9 +219,9 @@ install: web-install website-install poetry install dev-reset: - dropdb -U postgres -h localhost authentik + dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} # Also remove the test-db if it exists - dropdb -U postgres -h localhost test_authentik || true - createdb -U postgres -h localhost authentik + dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} test_authentik || true + createdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} redis-cli -n 0 flushall make migrate From 7cc74849322de637a03eb38288004037063526e3 Mon Sep 17 00:00:00 2001 From: Ken Sternberg Date: Mon, 25 Sep 2023 13:05:44 -0700 Subject: [PATCH 2/8] Re-arrange sequence to avoid recursive make. Nothing wrong with recursive make; it just wasn't essential here. `migrate` is just a build target, not a task. --- Makefile | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 45e46e5a1..9c4079134 100644 --- a/Makefile +++ b/Makefile @@ -218,10 +218,18 @@ ci-pending-migrations: ci--meta-debug install: web-install website-install poetry install -dev-reset: + +.PHONY: dev-drop-db dev-create-db dev-reset + +dev-drop-db: dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} # Also remove the test-db if it exists dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} test_authentik || true - createdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} redis-cli -n 0 flushall - make migrate + + +dev-create-db: + createdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} + + +dev-reset: dev-drop-db dev-create-db migrate ## Drop and restore the Authentik PostgreSQL instance to a "fresh install" state. From 23e0b3da5e978feb8b011b7e0beb288774dca5a2 Mon Sep 17 00:00:00 2001 From: Ken Sternberg Date: Mon, 25 Sep 2023 13:15:59 -0700 Subject: [PATCH 3/8] Cleanup according to the Usage: checkmake [options] ... checkmake -h | --help checkmake --version checkmake --list-rules Makefile linting tool. --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9c4079134..2340942fa 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +.PHONY: gen dev-reset all clean test web website + .SHELLFLAGS += -x -e PWD = $(shell pwd) UID = $(shell id -u) @@ -219,7 +221,6 @@ install: web-install website-install poetry install -.PHONY: dev-drop-db dev-create-db dev-reset dev-drop-db: dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} @@ -227,9 +228,7 @@ dev-drop-db: dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} test_authentik || true redis-cli -n 0 flushall - dev-create-db: createdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} - dev-reset: dev-drop-db dev-create-db migrate ## Drop and restore the Authentik PostgreSQL instance to a "fresh install" state. From b68e344daaec62a46682c4b3b52e251cd5e184da Mon Sep 17 00:00:00 2001 From: Ken Sternberg Date: Mon, 25 Sep 2023 13:34:59 -0700 Subject: [PATCH 4/8] core: added 'help' to the Makefile --- Makefile | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 2340942fa..b43f35925 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,13 @@ CODESPELL_ARGS = -D - -D .github/codespell-dictionary.txt \ website/integrations \ website/src -all: lint-fix lint test gen web +all: lint-fix lint test gen web ## Lint, build, and test everything + +help: ## Show this help + @echo "\nSpecify a command. The choices are:\n" + @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-24s\033[m %s\n", $$1, $$2}' + @echo "" test-go: go test -timeout 0 -v -race -cover ./... @@ -35,26 +41,26 @@ test-docker: docker-compose run -u root server test rm -f .env -test: +test: ## Run the server tests and produce a coverage report coverage run manage.py test --keepdb authentik coverage html coverage report -lint-fix: +lint-fix: ## Lint and automatically fix errors in the python source code. Reports spelling errors. isort authentik $(PY_SOURCES) black authentik $(PY_SOURCES) ruff authentik $(PY_SOURCES) codespell -w $(CODESPELL_ARGS) -lint: +lint: ## Lint the python and golang sources pylint $(PY_SOURCES) bandit -r $(PY_SOURCES) -x node_modules golangci-lint run -v -migrate: +migrate: ## Run the Authentik Django server's migrations python -m lifecycle.migrate -i18n-extract: i18n-extract-core web-i18n-extract +i18n-extract: i18n-extract-core web-i18n-extract ## Extract strings that require translation into files to send to a translation service i18n-extract-core: ak makemessages --ignore web --ignore internal --ignore web --ignore web-api --ignore website -l en @@ -63,7 +69,7 @@ i18n-extract-core: ## API Schema ######################### -gen-build: +gen-build: ## Extract the schema from the database AUTHENTIK_DEBUG=true ak make_blueprint_schema > blueprints/schema.json AUTHENTIK_DEBUG=true ak spectacular --file schema.yml @@ -86,7 +92,7 @@ gen-clean: rm -rf web/api/src/ rm -rf api/ -gen-client-ts: +gen-client-ts: ## Build and install the Authentik API for Typescript into the Authentik UI Application docker run \ --rm -v ${PWD}:/local \ --user ${UID}:${GID} \ @@ -128,24 +134,24 @@ gen: gen-build gen-clean gen-client-ts ## Web ######################### -web-build: web-install +web-build: web-install ## Build the Authentik UI cd web && npm run build web: web-lint-fix web-lint web-check-compile -web-install: +web-install: ## Install the necessary libraries to build the Authentik UI cd web && npm ci -web-watch: +web-watch: ## Build and watch the Authentik UI for changes, updating automatically rm -rf web/dist/ mkdir web/dist/ touch web/dist/.gitkeep cd web && npm run watch -web-storybook-watch: +web-storybook-watch: ## Build and run the storybook documentation server cd web && npm run storybook -web-lint-fix: +web-lint-fix: ## Automatically fix formatting issues in the Authentik UI source code cd web && npm run prettier web-lint: @@ -162,7 +168,7 @@ web-i18n-extract: ## Website ######################### -website: website-lint-fix website-build +website: website-lint-fix website-build ## Build the documentation website website-install: cd website && npm ci @@ -173,7 +179,7 @@ website-lint-fix: website-build: cd website && npm run build -website-watch: +website-watch: ## Build and watch the documentation website, updating automatically cd website && npm run watch ######################### From 759abbebf700b3941e62b8d8f09c84a256af63cc Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Mon, 25 Sep 2023 23:06:59 +0200 Subject: [PATCH 5/8] get postgres config from authentik config loader Signed-off-by: Jens Langhammer --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b43f35925..1d539c6c4 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,10 @@ GID = $(shell id -g) NPM_VERSION = $(shell python -m scripts.npm_version) PY_SOURCES = authentik tests scripts lifecycle +pg_user := $(shell python -m authentik.lib.config postgresql.user 2>/dev/null) +pg_host := $(shell python -m authentik.lib.config postgresql.host 2>/dev/null) +pg_name := $(shell python -m authentik.lib.config postgresql.name 2>/dev/null) + CODESPELL_ARGS = -D - -D .github/codespell-dictionary.txt \ -I .github/codespell-words.txt \ -S 'web/src/locales/**' \ @@ -26,7 +30,7 @@ all: lint-fix lint test gen web ## Lint, build, and test everything help: ## Show this help @echo "\nSpecify a command. The choices are:\n" @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ - awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-24s\033[m %s\n", $$1, $$2}' + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-24s\033[m %s\n", $$1, $$2}' @echo "" test-go: @@ -229,12 +233,12 @@ install: web-install website-install dev-drop-db: - dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} + echo dropdb -U ${pg_user} -h ${pg_host} ${pg_name} # Also remove the test-db if it exists - dropdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} test_authentik || true - redis-cli -n 0 flushall + dropdb -U ${pg_user} -h ${pg_host} test_${pg_name} || true + echo redis-cli -n 0 flushall dev-create-db: - createdb -U $${PG_USER:-postgres} -h $${PG_HOST:-localhost} $${PG_DB:-authentik} + createdb -U ${pg_user} -h ${pg_host} ${pg_name} dev-reset: dev-drop-db dev-create-db migrate ## Drop and restore the Authentik PostgreSQL instance to a "fresh install" state. From fe2202fde0555996a728487aa99fac801ef215ea Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Mon, 25 Sep 2023 23:10:34 +0200 Subject: [PATCH 6/8] don't set -x by default Signed-off-by: Jens Langhammer --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1d539c6c4..b1154f873 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: gen dev-reset all clean test web website -.SHELLFLAGS += -x -e +.SHELLFLAGS += ${SHELLFLAGS} -e PWD = $(shell pwd) UID = $(shell id -u) GID = $(shell id -g) From 978480008a33636ed83bdf10174a8faca3ea9c83 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Mon, 25 Sep 2023 23:16:42 +0200 Subject: [PATCH 7/8] sort help Signed-off-by: Jens Langhammer --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b1154f873..61aa5256f 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,8 @@ all: lint-fix lint test gen web ## Lint, build, and test everything help: ## Show this help @echo "\nSpecify a command. The choices are:\n" @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ - awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-24s\033[m %s\n", $$1, $$2}' + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-24s\033[m %s\n", $$1, $$2}' | \ + sort @echo "" test-go: From 0f4fd80a26dd429e2ee5a688158f400fc311c603 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Mon, 25 Sep 2023 23:17:03 +0200 Subject: [PATCH 8/8] update help strings Signed-off-by: Jens Langhammer --- Makefile | 57 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 61aa5256f..d4e33fec9 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ UID = $(shell id -u) GID = $(shell id -g) NPM_VERSION = $(shell python -m scripts.npm_version) PY_SOURCES = authentik tests scripts lifecycle +DOCKER_IMAGE ?= "authentik:test" pg_user := $(shell python -m authentik.lib.config postgresql.user 2>/dev/null) pg_host := $(shell python -m authentik.lib.config postgresql.host 2>/dev/null) @@ -37,16 +38,16 @@ help: ## Show this help test-go: go test -timeout 0 -v -race -cover ./... -test-docker: +test-docker: ## Run all tests in a docker-compose echo "PG_PASS=$(openssl rand -base64 32)" >> .env echo "AUTHENTIK_SECRET_KEY=$(openssl rand -base64 32)" >> .env docker-compose pull -q docker-compose up --no-start docker-compose start postgresql redis - docker-compose run -u root server test + docker-compose run -u root server test-all rm -f .env -test: ## Run the server tests and produce a coverage report +test: ## Run the server tests and produce a coverage report (locally) coverage run manage.py test --keepdb authentik coverage html coverage report @@ -70,6 +71,20 @@ i18n-extract: i18n-extract-core web-i18n-extract ## Extract strings that requir i18n-extract-core: ak makemessages --ignore web --ignore internal --ignore web --ignore web-api --ignore website -l en +install: web-install website-install ## Install all requires dependencies for `web`, `website` and `core` + poetry install + +dev-drop-db: + echo dropdb -U ${pg_user} -h ${pg_host} ${pg_name} + # Also remove the test-db if it exists + dropdb -U ${pg_user} -h ${pg_host} test_${pg_name} || true + echo redis-cli -n 0 flushall + +dev-create-db: + createdb -U ${pg_user} -h ${pg_host} ${pg_name} + +dev-reset: dev-drop-db dev-create-db migrate ## Drop and restore the Authentik PostgreSQL instance to a "fresh install" state. + ######################### ## API Schema ######################### @@ -78,11 +93,11 @@ gen-build: ## Extract the schema from the database AUTHENTIK_DEBUG=true ak make_blueprint_schema > blueprints/schema.json AUTHENTIK_DEBUG=true ak spectacular --file schema.yml -gen-changelog: +gen-changelog: ## (Release) generate the changelog based from the commits since the last tag git log --pretty=format:" - %s" $(shell git describe --tags $(shell git rev-list --tags --max-count=1))...$(shell git branch --show-current) | sort > changelog.md npx prettier --write changelog.md -gen-diff: +gen-diff: ## (Release) generate the changelog diff between the current schema and the last tag git show $(shell git describe --tags $(shell git rev-list --tags --max-count=1)):schema.yml > old_schema.yml docker run \ --rm -v ${PWD}:/local \ @@ -97,7 +112,7 @@ gen-clean: rm -rf web/api/src/ rm -rf api/ -gen-client-ts: ## Build and install the Authentik API for Typescript into the Authentik UI Application +gen-client-ts: ## Build and install the authentik API for Typescript into the authentik UI Application docker run \ --rm -v ${PWD}:/local \ --user ${UID}:${GID} \ @@ -113,7 +128,7 @@ gen-client-ts: ## Build and install the Authentik API for Typescript into the A cd gen-ts-api && npm i \cp -rfv gen-ts-api/* web/node_modules/@goauthentik/api -gen-client-go: +gen-client-go: ## Build and install the authentik API for Golang mkdir -p ./gen-go-api ./gen-go-api/templates wget https://raw.githubusercontent.com/goauthentik/client-go/main/config.yaml -O ./gen-go-api/config.yaml wget https://raw.githubusercontent.com/goauthentik/client-go/main/templates/README.mustache -O ./gen-go-api/templates/README.mustache @@ -130,7 +145,7 @@ gen-client-go: go mod edit -replace goauthentik.io/api/v3=./gen-go-api rm -rf ./gen-go-api/config.yaml ./gen-go-api/templates/ -gen-dev-config: +gen-dev-config: ## Generate a local development config file python -m scripts.generate_config gen: gen-build gen-clean gen-client-ts @@ -142,7 +157,7 @@ gen: gen-build gen-clean gen-client-ts web-build: web-install ## Build the Authentik UI cd web && npm run build -web: web-lint-fix web-lint web-check-compile +web: web-lint-fix web-lint web-check-compile ## Automatically fix formatting issues in the Authentik UI source code, lint the code, and compile it web-install: ## Install the necessary libraries to build the Authentik UI cd web && npm ci @@ -156,7 +171,7 @@ web-watch: ## Build and watch the Authentik UI for changes, updating automatica web-storybook-watch: ## Build and run the storybook documentation server cd web && npm run storybook -web-lint-fix: ## Automatically fix formatting issues in the Authentik UI source code +web-lint-fix: cd web && npm run prettier web-lint: @@ -173,7 +188,7 @@ web-i18n-extract: ## Website ######################### -website: website-lint-fix website-build ## Build the documentation website +website: website-lint-fix website-build ## Automatically fix formatting issues in the Authentik website/docs source code, lint the code, and compile it website-install: cd website && npm ci @@ -191,8 +206,8 @@ website-watch: ## Build and watch the documentation website, updating automatic ## Docker ######################### -docker: - DOCKER_BUILDKIT=1 docker build . --progress plain --tag authentik:test +docker: ## Build a docker image of the current source tree + DOCKER_BUILDKIT=1 docker build . --progress plain --tag ${DOCKER_IMAGE} ######################### ## CI @@ -227,19 +242,3 @@ ci-pyright: ci--meta-debug ci-pending-migrations: ci--meta-debug ak makemigrations --check - -install: web-install website-install - poetry install - - - -dev-drop-db: - echo dropdb -U ${pg_user} -h ${pg_host} ${pg_name} - # Also remove the test-db if it exists - dropdb -U ${pg_user} -h ${pg_host} test_${pg_name} || true - echo redis-cli -n 0 flushall - -dev-create-db: - createdb -U ${pg_user} -h ${pg_host} ${pg_name} - -dev-reset: dev-drop-db dev-create-db migrate ## Drop and restore the Authentik PostgreSQL instance to a "fresh install" state.