root: automate system migrations, move docker to lifecycle folder

This commit is contained in:
Jens Langhammer 2020-09-10 00:14:59 +02:00
parent 1356a8108b
commit 430905295d
16 changed files with 158 additions and 48 deletions

View File

@ -11,27 +11,22 @@ RUN pip install pipenv && \
FROM python:3.8-slim-buster FROM python:3.8-slim-buster
COPY --from=locker /app/requirements.txt /app/ WORKDIR /
COPY --from=locker /app/requirements-dev.txt /app/ COPY --from=locker /app/requirements.txt /
COPY --from=locker /app/requirements-dev.txt /
WORKDIR /app/
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y --no-install-recommends postgresql-client-11 build-essential && \ apt-get install -y --no-install-recommends postgresql-client-11 build-essential && \
rm -rf /var/lib/apt/ && \ rm -rf /var/lib/apt/ && \
pip install -r requirements.txt --no-cache-dir && \ pip install -r /requirements.txt --no-cache-dir && \
apt-get remove --purge -y build-essential && \ apt-get remove --purge -y build-essential && \
apt-get autoremove --purge && \ apt-get autoremove --purge && \
adduser --system --no-create-home --uid 1000 --group --home /app passbook adduser --system --no-create-home --uid 1000 --group --home /passbook passbook
COPY ./passbook/ /app/passbook COPY ./passbook/ /passbook
COPY ./manage.py /app/ COPY ./manage.py /
COPY ./docker/gunicorn.conf.py /app/ COPY ./lifecycle/ /lifecycle
COPY ./docker/bootstrap.sh /bootstrap.sh
COPY ./docker/wait_for_db.py /app/wait_for_db.py
WORKDIR /app/
USER passbook USER passbook
ENTRYPOINT [ "/bootstrap.sh" ] ENTRYPOINT [ "/lifecycle/bootstrap.sh" ]

View File

@ -25,7 +25,7 @@ wget https://raw.githubusercontent.com/BeryJu/passbook/master/docker-compose.yml
# export PG_PASS=$(pwgen 40 1) # export PG_PASS=$(pwgen 40 1)
docker-compose pull docker-compose pull
docker-compose up -d docker-compose up -d
docker-compose exec server ./manage.py migrate docker-compose run --rm server migrate
``` ```
For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org//installation/kubernetes/) For bigger setups, there is a Helm Chart in the `helm/` directory. This is documented [here](https://passbook.beryju.org//installation/kubernetes/)

View File

@ -1,10 +0,0 @@
#!/bin/bash -e
/app/wait_for_db.py
printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@"
if [[ "$1" == "server" ]]; then
gunicorn -c gunicorn.conf.py passbook.root.asgi:application
elif [[ "$1" == "worker" ]]; then
celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule
else
./manage.py "$@"
fi

View File

@ -2,13 +2,13 @@
from time import sleep from time import sleep
from django.test import override_settings from django.test import override_settings
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import USER, SeleniumTestCase from e2e.utils import USER, SeleniumTestCase
from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding from passbook.flows.models import Flow, FlowDesignation, FlowStageBinding
from passbook.policies.expression.models import ExpressionPolicy from passbook.policies.expression.models import ExpressionPolicy

View File

@ -1,13 +1,13 @@
"""test OAuth Provider flow""" """test OAuth Provider flow"""
from time import sleep from time import sleep
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from docker import DockerClient, from_env from docker import DockerClient, from_env
from docker.models.containers import Container from docker.models.containers import Container
from docker.types import Healthcheck from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase from e2e.utils import USER, SeleniumTestCase
from passbook.core.models import Application from passbook.core.models import Application
from passbook.flows.models import Flow from passbook.flows.models import Flow

View File

@ -1,14 +1,14 @@
"""test OAuth2 OpenID Provider flow""" """test OAuth2 OpenID Provider flow"""
from time import sleep from time import sleep
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import USER, SeleniumTestCase from e2e.utils import USER, SeleniumTestCase
from passbook.core.models import Application from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair from passbook.crypto.models import CertificateKeyPair

View File

@ -1,13 +1,13 @@
"""test SAML Provider flow""" """test SAML Provider flow"""
from time import sleep from time import sleep
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from docker import DockerClient, from_env from docker import DockerClient, from_env
from docker.models.containers import Container from docker.models.containers import Container
from docker.types import Healthcheck from docker.types import Healthcheck
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from structlog import get_logger
from e2e.utils import USER, SeleniumTestCase from e2e.utils import USER, SeleniumTestCase
from passbook.core.models import Application from passbook.core.models import Application
from passbook.crypto.models import CertificateKeyPair from passbook.crypto.models import CertificateKeyPair

View File

@ -1,14 +1,14 @@
"""test SAML Source""" """test SAML Source"""
from time import sleep from time import sleep
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger from structlog import get_logger
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import SeleniumTestCase from e2e.utils import SeleniumTestCase
from passbook.crypto.models import CertificateKeyPair from passbook.crypto.models import CertificateKeyPair
from passbook.flows.models import Flow from passbook.flows.models import Flow

View File

@ -2,15 +2,15 @@
from os.path import abspath from os.path import abspath
from time import sleep from time import sleep
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ec from selenium.webdriver.support import expected_conditions as ec
from structlog import get_logger from structlog import get_logger
from yaml import safe_dump from yaml import safe_dump
from docker import DockerClient, from_env
from docker.models.containers import Container
from docker.types import Healthcheck
from e2e.utils import SeleniumTestCase from e2e.utils import SeleniumTestCase
from passbook.flows.models import Flow from passbook.flows.models import Flow
from passbook.providers.oauth2.generators import generate_client_secret from passbook.providers.oauth2.generators import generate_client_secret

0
lifecycle/__init__.py Normal file
View File

14
lifecycle/bootstrap.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash -e
python -m lifecycle.wait_for_db
printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap", "command": "%s"}\n' "$@"
if [[ "$1" == "server" ]]; then
gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
elif [[ "$1" == "worker" ]]; then
celery worker --autoscale=10,3 -E -B -A=passbook.root.celery -s=/tmp/celerybeat-schedule
elif [[ "$1" == "migrate" ]]; then
# Run system migrations first, run normal migrations after
python -m lifecycle.migrate
python -m manage migrate
else
python -m manage "$@"
fi

55
lifecycle/migrate.py Executable file
View File

@ -0,0 +1,55 @@
#!/usr/bin/env python
"""System Migration handler"""
from importlib.util import module_from_spec, spec_from_file_location
from inspect import getmembers, isclass
from pathlib import Path
from typing import Any
from psycopg2 import connect
from structlog import get_logger
from passbook.lib.config import CONFIG
LOGGER = get_logger()
class BaseMigration:
"""Base System Migration"""
cur: Any
con: Any
def __init__(self, cur: Any, con: Any):
self.cur = cur
self.con = con
def needs_migration(self) -> bool:
"""Return true if Migration needs to be run"""
return False
def run(self):
"""Run the actual migration"""
if __name__ == "__main__":
conn = connect(
dbname=CONFIG.y("postgresql.name"),
user=CONFIG.y("postgresql.user"),
password=CONFIG.y("postgresql.password"),
host=CONFIG.y("postgresql.host"),
)
curr = conn.cursor()
for migration in Path(__file__).parent.absolute().glob("system_migrations/*.py"):
spec = spec_from_file_location("lifecycle.system_migrations", migration)
mod = module_from_spec(spec)
# pyright: reportGeneralTypeIssues=false
spec.loader.exec_module(mod)
for _, sub in getmembers(mod, isclass):
migration = sub(curr, conn)
if migration.needs_migration():
LOGGER.info("Migration needs to be applied", migration=sub)
migration.run()
LOGGER.info("Migration finished applying", migration=sub)

View File

@ -0,0 +1,50 @@
from os import system
from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """delete from django_migrations where app = 'passbook_stages_prompt';
drop table passbook_stages_prompt_prompt cascade;
drop table passbook_stages_prompt_promptstage cascade;
drop table passbook_stages_prompt_promptstage_fields;
drop table corsheaders_corsmodel cascade;
drop table oauth2_provider_accesstoken cascade;
drop table oauth2_provider_grant cascade;
drop table oauth2_provider_refreshtoken cascade;
drop table oidc_provider_client cascade;
drop table oidc_provider_client_response_types cascade;
drop table oidc_provider_code cascade;
drop table oidc_provider_responsetype cascade;
drop table oidc_provider_rsakey cascade;
drop table oidc_provider_token cascade;
drop table oidc_provider_userconsent cascade;
drop table passbook_providers_app_gw_applicationgatewayprovider cascade;
delete from django_migrations where app = 'passbook_flows' and name = '0008_default_flows';
delete from django_migrations where app = 'passbook_flows' and name = '0009_source_flows';
delete from django_migrations where app = 'passbook_flows' and name = '0010_provider_flows';
delete from django_migrations where app = 'passbook_stages_password' and
name = '0002_passwordstage_change_flow';"""
class To010Migration(BaseMigration):
def needs_migration(self) -> bool:
self.cur.execute(
"select * from information_schema.tables where table_name='oidc_provider_client'"
)
return bool(self.cur.rowcount)
def system_crit(self, command):
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def run(self):
self.cur.execute(SQL_STATEMENT)
self.con.commit()
self.system_crit("./manage.py migrate passbook_stages_prompt")
self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake")
self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake")
self.system_crit(
"./manage.py migrate passbook_flows 0010_provider_flows --fake"
)
self.system_crit("./manage.py migrate passbook_flows")
self.system_crit("./manage.py migrate passbook_stages_password --fake")

View File

@ -5249,7 +5249,7 @@ paths:
tags: tags:
- stages - stages
parameters: [] parameters: []
/stages/prompt/stages/{pbm_uuid}/: /stages/prompt/stages/{stage_uuid}/:
get: get:
operationId: stages_prompt_stages_read operationId: stages_prompt_stages_read
description: PromptStage Viewset description: PromptStage Viewset
@ -5303,7 +5303,7 @@ paths:
tags: tags:
- stages - stages
parameters: parameters:
- name: pbm_uuid - name: stage_uuid
in: path in: path
description: A UUID string identifying this Prompt Stage. description: A UUID string identifying this Prompt Stage.
required: true required: true
@ -7463,7 +7463,7 @@ definitions:
type: object type: object
properties: properties:
pk: pk:
title: Pbm uuid title: Stage uuid
type: string type: string
format: uuid format: uuid
readOnly: true readOnly: true
@ -7477,6 +7477,12 @@ definitions:
type: string type: string
format: uuid format: uuid
uniqueItems: true uniqueItems: true
validation_policies:
type: array
items:
type: string
format: uuid
uniqueItems: true
UserDeleteStage: UserDeleteStage:
required: required:
- name - name