root: automate system migrations, move docker to lifecycle folder

This commit is contained in:
Jens Langhammer 2020-09-10 00:14:59 +02:00
parent 1356a8108b
commit 430905295d
16 changed files with 158 additions and 48 deletions

View File

@ -11,27 +11,22 @@ RUN pip install pipenv && \
FROM python:3.8-slim-buster
COPY --from=locker /app/requirements.txt /app/
COPY --from=locker /app/requirements-dev.txt /app/
WORKDIR /app/
WORKDIR /
COPY --from=locker /app/requirements.txt /
COPY --from=locker /app/requirements-dev.txt /
RUN apt-get update && \
apt-get install -y --no-install-recommends postgresql-client-11 build-essential && \
rm -rf /var/lib/apt/ && \
pip install -r requirements.txt --no-cache-dir && \
pip install -r /requirements.txt --no-cache-dir && \
apt-get remove --purge -y build-essential && \
apt-get autoremove --purge && \
adduser --system --no-create-home --uid 1000 --group --home /app passbook
adduser --system --no-create-home --uid 1000 --group --home /passbook passbook
COPY ./passbook/ /app/passbook
COPY ./manage.py /app/
COPY ./docker/gunicorn.conf.py /app/
COPY ./docker/bootstrap.sh /bootstrap.sh
COPY ./docker/wait_for_db.py /app/wait_for_db.py
WORKDIR /app/
COPY ./passbook/ /passbook
COPY ./manage.py /
COPY ./lifecycle/ /lifecycle
USER passbook
ENTRYPOINT [ "/bootstrap.sh" ]
ENTRYPOINT [ "/lifecycle/bootstrap.sh" ]

View File

@ -25,7 +25,7 @@ wget https://raw.githubusercontent.com/BeryJu/passbook/master/docker-compose.yml
# export PG_PASS=$(pwgen 40 1)
docker-compose pull
docker-compose up -d
docker-compose exec server ./manage.py migrate
docker-compose run --rm server migrate
```
For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org//installation/kubernetes/)

View File

@ -1,10 +0,0 @@
#!/bin/bash -e
/app/wait_for_db.py
printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@"
if [[ "$1" == "server" ]]; then
gunicorn -c gunicorn.conf.py passbook.root.asgi:application
elif [[ "$1" == "worker" ]]; then
celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule
else
./manage.py "$@"
fi

View File

@ -2,13 +2,13 @@
from time import sleep
from django.test import override_settings
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import USER, SeleniumTestCase
from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
from passbook.policies.expression.models import ExpressionPolicy

View File

@ -1,13 +1,13 @@
"""test OAuth Provider flow"""
from time import sleep
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase
from passbook.core.models import Application
from passbook.flows.models import Flow

View File

@ -1,14 +1,14 @@
"""test OAuth2 OpenID Provider flow"""
from time import sleep
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import USER, SeleniumTestCase
from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair

View File

@ -1,13 +1,13 @@
"""test SAML Provider flow"""
from time import sleep
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase
from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair

View File

@ -1,14 +1,14 @@
"""test SAML Source"""
from time import sleep
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import SeleniumTestCase
from passbook.crypto.models import CertificateKeyPair
from passbook.flows.models import Flow

View File

@ -2,15 +2,15 @@
from os.path import abspath
from time import sleep
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger
from yaml import safe_dump
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import SeleniumTestCase
from passbook.flows.models import Flow
from passbook.providers.oauth2.generators import generate_client_secret

0
lifecycle/__init__.py Normal file
View File

14
lifecycle/bootstrap.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash -e
python -m lifecycle.wait_for_db
printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@"
if [[ "$1" == "server" ]]; then
gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
elif [[ "$1" == "worker" ]]; then
celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule
elif [[ "$1" == "migrate" ]]; then
# Run system migrations first, run normal migrations after
python -m lifecycle.migrate
python -m manage migrate
else
python -m manage "$@"
fi

55
lifecycle/migrate.py Executable file
View File

@ -0,0 +1,55 @@
#!/usr/bin/env python
"""System Migration handler"""
from importlib.util import module_from_spec, spec_from_file_location
from inspect import getmembers, isclass
from pathlib import Path
from typing import Any
from psycopg2 import connect
from structlog import get_logger
from passbook.lib.config import CONFIG
LOGGER = get_logger()
class BaseMigration:
"""Base System Migration"""
cur: Any
con: Any
def __init__(self, cur: Any, con: Any):
self.cur = cur
self.con = con
def needs_migration(self) -> bool:
"""Return true if Migration needs to be run"""
return False
def run(self):
"""Run the actual migration"""
if __name__ == "__main__":
conn = connect(
dbname=CONFIG.y("postgresql.name"),
user=CONFIG.y("postgresql.user"),
password=CONFIG.y("postgresql.password"),
host=CONFIG.y("postgresql.host"),
)
curr = conn.cursor()
for migration in Path(__file__).parent.absolute().glob("system_migrations/*.py"):
spec = spec_from_file_location("lifecycle.system_migrations", migration)
mod = module_from_spec(spec)
# pyright: reportGeneralTypeIssues=false
spec.loader.exec_module(mod)
for _, sub in getmembers(mod, isclass):
migration = sub(curr, conn)
if migration.needs_migration():
LOGGER.info("Migration needs to be applied", migration=sub)
migration.run()
LOGGER.info("Migration finished applying", migration=sub)

View File

@ -0,0 +1,50 @@
from os import system
from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """delete from django_migrations where app = 'passbook_stages_prompt';
drop table passbook_stages_prompt_prompt cascade;
drop table passbook_stages_prompt_promptstage cascade;
drop table passbook_stages_prompt_promptstage_fields;
drop table corsheaders_corsmodel cascade;
drop table oauth2_provider_accesstoken cascade;
drop table oauth2_provider_grant cascade;
drop table oauth2_provider_refreshtoken cascade;
drop table oidc_provider_client cascade;
drop table oidc_provider_client_response_types cascade;
drop table oidc_provider_code cascade;
drop table oidc_provider_responsetype cascade;
drop table oidc_provider_rsakey cascade;
drop table oidc_provider_token cascade;
drop table oidc_provider_userconsent cascade;
drop table passbook_providers_app_gw_applicationgatewayprovider cascade;
delete from django_migrations where app = 'passbook_flows' and name = '0008_default_flows';
delete from django_migrations where app = 'passbook_flows' and name = '0009_source_flows';
delete from django_migrations where app = 'passbook_flows' and name = '0010_provider_flows';
delete from django_migrations where app = 'passbook_stages_password' and
name = '0002_passwordstage_change_flow';"""
class To010Migration(BaseMigration):
def needs_migration(self) -> bool:
self.cur.execute(
"select * from information_schema.tables where table_name='oidc_provider_client'"
)
return bool(self.cur.rowcount)
def system_crit(self, command):
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def run(self):
self.cur.execute(SQL_STATEMENT)
self.con.commit()
self.system_crit("./manage.py migrate passbook_stages_prompt")
self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake")
self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake")
self.system_crit(
"./manage.py migrate passbook_flows 0010_provider_flows --fake"
)
self.system_crit("./manage.py migrate passbook_flows")
self.system_crit("./manage.py migrate passbook_stages_password --fake")

View File

@ -5249,7 +5249,7 @@ paths:
tags:
- stages
parameters: []
/stages/prompt/stages/{pbm_uuid}/:
/stages/prompt/stages/{stage_uuid}/:
get:
operationId: stages_prompt_stages_read
description: PromptStage Viewset
@ -5303,7 +5303,7 @@ paths:
tags:
- stages
parameters:
- name: pbm_uuid
- name: stage_uuid
in: path
description: A UUID string identifying this Prompt Stage.
required: true
@ -7463,7 +7463,7 @@ definitions:
type: object
properties:
pk:
title: Pbm uuid
title: Stage uuid
type: string
format: uuid
readOnly: true
@ -7477,6 +7477,12 @@ definitions:
type: string
format: uuid
uniqueItems: true
validation_policies:
type: array
items:
type: string
format: uuid
uniqueItems: true
UserDeleteStage:
required:
- name