trustchain-oc1-orchestral
/
authentik
Archived
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lifecycle: improve reliability of system migrations (#7089) 

* lifecycle: improve reliability of system migrations

better transaction handling which allows for re-trying migrations without needing manual intervention

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix lint

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* remove explicit commit

---------

Signed-off-by: Jens Langhammer <jens@goauthentik.io>
This commit is contained in:
Jens L 2023-10-06 18:56:10 +02:00 committed by GitHub
parent 25ee6f8116
commit b90ed6bab3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 97 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lifecycle/migrate.py
View File

@ -1,12 +1,12 @@
#!/usr/bin/env python #!/usr/bin/env python
"""System Migration handler""" """System Migration handler"""
import os
from importlib.util import module_from_spec, spec_from_file_location from importlib.util import module_from_spec, spec_from_file_location
from inspect import getmembers, isclass from inspect import getmembers, isclass
from os import environ, system
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from psycopg import connect from psycopg import Connection, Cursor, connect
from structlog.stdlib import get_logger from structlog.stdlib import get_logger
from authentik.lib.config import CONFIG from authentik.lib.config import CONFIG
@ -16,16 +16,33 @@ ADV_LOCK_UID = 1000
LOCKED = False LOCKED = False
class CommandError(Exception):
"""Error raised when a system_crit command fails"""
class BaseMigration: class BaseMigration:
"""Base System Migration""" """Base System Migration"""
cur: Any cur: Cursor
con: Any con: Connection
def __init__(self, cur: Any, con: Any): def __init__(self, cur: Any, con: Any):
self.cur = cur self.cur = cur
self.con = con self.con = con
def system_crit(self, command: str):
"""Run system command"""
LOGGER.debug("Running system_crit command", command=command)
retval = system(command) # nosec
if retval != 0:
raise CommandError("Migration error")
def fake_migration(self, *app_migration: tuple[str, str]):
"""Fake apply a list of migrations, arguments are
expected to be tuples of (app_label, migration_name)"""
for app, _migration in app_migration:
self.system_crit(f"./manage.py migrate {app} {_migration} --fake")
def needs_migration(self) -> bool: def needs_migration(self) -> bool:
"""Return true if Migration needs to be run""" """Return true if Migration needs to be run"""
return False return False
@ -82,7 +99,7 @@ if __name__ == "__main__":
LOGGER.info("Migration finished applying", migration=sub) LOGGER.info("Migration finished applying", migration=sub)
release_lock() release_lock()
LOGGER.info("applying django migrations") LOGGER.info("applying django migrations")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "authentik.root.settings") environ.setdefault("DJANGO_SETTINGS_MODULE", "authentik.root.settings")
wait_for_lock() wait_for_lock()
try: try:
from django.core.management import execute_from_command_line from django.core.management import execute_from_command_line

31
lifecycle/system_migrations/install_id.py
View File

@ -4,11 +4,9 @@ from uuid import uuid4
from authentik.lib.config import CONFIG from authentik.lib.config import CONFIG
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """BEGIN TRANSACTION; SQL_STATEMENT = """CREATE TABLE IF NOT EXISTS authentik_install_id (
CREATE TABLE IF NOT EXISTS authentik_install_id (
id TEXT NOT NULL id TEXT NOT NULL
); );"""
COMMIT;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -19,19 +17,18 @@ class Migration(BaseMigration):
return not bool(self.cur.rowcount) return not bool(self.cur.rowcount)
def upgrade(self, migrate=False): def upgrade(self, migrate=False):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)
if migrate: if migrate:
# If we already have migrations in the database, assume we're upgrading an existing install # If we already have migrations in the database, assume we're upgrading an existing install
# and set the install id to the secret key # and set the install id to the secret key
self.cur.execute( self.cur.execute(
"INSERT INTO authentik_install_id (id) VALUES (%s)", (CONFIG.get("secret_key"),) "INSERT INTO authentik_install_id (id) VALUES (%s)", (CONFIG.get("secret_key"),)
) )
else: else:
# Otherwise assume a new install, generate an install ID based on a UUID # Otherwise assume a new install, generate an install ID based on a UUID
install_id = str(uuid4()) install_id = str(uuid4())
self.cur.execute("INSERT INTO authentik_install_id (id) VALUES (%s)", (install_id,)) self.cur.execute("INSERT INTO authentik_install_id (id) VALUES (%s)", (install_id,))
self.con.commit()
def run(self): def run(self):
self.cur.execute( self.cur.execute(

44
lifecycle/system_migrations/otp_merge.py
View File

@ -1,10 +1,7 @@
# flake8: noqa # flake8: noqa
from os import system
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ SQL_STATEMENT = """
BEGIN TRANSACTION;
DELETE FROM django_migrations WHERE app = 'otp_static'; DELETE FROM django_migrations WHERE app = 'otp_static';
DELETE FROM django_migrations WHERE app = 'otp_totp'; DELETE FROM django_migrations WHERE app = 'otp_totp';
-- Rename tables (static) -- Rename tables (static)
@ -15,7 +12,7 @@ ALTER SEQUENCE otp_static_staticdevice_id_seq RENAME TO authentik_stages_authent
-- Rename tables (totp) -- Rename tables (totp)
ALTER TABLE otp_totp_totpdevice RENAME TO authentik_stages_authenticator_totp_totpdevice; ALTER TABLE otp_totp_totpdevice RENAME TO authentik_stages_authenticator_totp_totpdevice;
ALTER SEQUENCE otp_totp_totpdevice_id_seq RENAME TO authentik_stages_authenticator_totp_totpdevice_id_seq; ALTER SEQUENCE otp_totp_totpdevice_id_seq RENAME TO authentik_stages_authenticator_totp_totpdevice_id_seq;
COMMIT;""" """
class Migration(BaseMigration): class Migration(BaseMigration):
@ -25,23 +22,24 @@ class Migration(BaseMigration):
) )
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def system_crit(self, command):
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def run(self): def run(self):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)
self.system_crit( self.fake_migration(
"./manage.py migrate authentik_stages_authenticator_static 0008_initial --fake" (
) "authentik_stages_authenticator_static",
self.system_crit( "0008_initial",
"./manage.py migrate authentik_stages_authenticator_static 0009_throttling --fake" ),
) (
self.system_crit( "authentik_stages_authenticator_static",
"./manage.py migrate authentik_stages_authenticator_totp 0008_initial --fake" "0009_throttling",
) ),
self.system_crit( (
"./manage.py migrate authentik_stages_authenticator_totp 0009_auto_20190420_0723 --fake" "authentik_stages_authenticator_totp",
) "0008_initial",
),
(
"authentik_stages_authenticator_totp",
"0009_auto_20190420_0723",
),
)

28
lifecycle/system_migrations/to_0_10.py
View File

@ -1,10 +1,7 @@
# flake8: noqa # flake8: noqa
from os import system
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ SQL_STATEMENT = """
BEGIN TRANSACTION;
DELETE FROM django_migrations WHERE app = 'passbook_stages_prompt'; DELETE FROM django_migrations WHERE app = 'passbook_stages_prompt';
DROP TABLE passbook_stages_prompt_prompt cascade; DROP TABLE passbook_stages_prompt_prompt cascade;
DROP TABLE passbook_stages_prompt_promptstage cascade; DROP TABLE passbook_stages_prompt_promptstage cascade;
@ -25,7 +22,7 @@ DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0008_defa
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0009_source_flows'; DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0009_source_flows';
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0010_provider_flows'; DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0010_provider_flows';
DELETE FROM django_migrations WHERE app = 'passbook_stages_password' AND name = '0002_passwordstage_change_flow'; DELETE FROM django_migrations WHERE app = 'passbook_stages_password' AND name = '0002_passwordstage_change_flow';
COMMIT;""" """
class Migration(BaseMigration): class Migration(BaseMigration):
@ -35,17 +32,14 @@ class Migration(BaseMigration):
) )
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def system_crit(self, command):
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def run(self): def run(self):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)
self.system_crit("./manage.py migrate passbook_stages_prompt") self.system_crit("./manage.py migrate passbook_stages_prompt")
self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake") self.fake_migration(
self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake") ("passbook_flows", "0008_default_flows"),
self.system_crit("./manage.py migrate passbook_flows 0010_provider_flows --fake") ("passbook_flows", "0009_source_flows"),
self.system_crit("./manage.py migrate passbook_flows") ("passbook_flows", "0010_provider_flows"),
self.system_crit("./manage.py migrate passbook_stages_password --fake") )
self.system_crit("./manage.py migrate passbook_flows")
self.fake_migration(("passbook_stages_password", ""))

35
lifecycle/system_migrations/to_0_13_authentik.py
View File

@ -4,7 +4,7 @@ from redis import Redis
from authentik.lib.config import CONFIG from authentik.lib.config import CONFIG
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """BEGIN TRANSACTION; SQL_STATEMENT = """
ALTER TABLE passbook_audit_event RENAME TO authentik_audit_event; ALTER TABLE passbook_audit_event RENAME TO authentik_audit_event;
ALTER TABLE passbook_core_application RENAME TO authentik_core_application; ALTER TABLE passbook_core_application RENAME TO authentik_core_application;
ALTER TABLE passbook_core_group RENAME TO authentik_core_group; ALTER TABLE passbook_core_group RENAME TO authentik_core_group;
@ -92,8 +92,7 @@ ALTER SEQUENCE passbook_stages_prompt_promptstage_validation_policies_id_seq REN
UPDATE django_migrations SET app = replace(app, 'passbook', 'authentik'); UPDATE django_migrations SET app = replace(app, 'passbook', 'authentik');
UPDATE django_content_type SET app_label = replace(app_label, 'passbook', 'authentik'); UPDATE django_content_type SET app_label = replace(app_label, 'passbook', 'authentik');
"""
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -104,18 +103,18 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)
# We also need to clean the cache to make sure no pickeled objects still exist # We also need to clean the cache to make sure no pickeled objects still exist
for db in [ for db in [
CONFIG.get("redis.message_queue_db"), CONFIG.get("redis.message_queue_db"),
CONFIG.get("redis.cache_db"), CONFIG.get("redis.cache_db"),
CONFIG.get("redis.ws_db"), CONFIG.get("redis.ws_db"),
]: ]:
redis = Redis( redis = Redis(
host=CONFIG.get("redis.host"), host=CONFIG.get("redis.host"),
port=6379, port=6379,
db=db, db=db,
password=CONFIG.get("redis.password"), password=CONFIG.get("redis.password"),
) )
redis.flushall() redis.flushall()

11
lifecycle/system_migrations/to_0_14_events..py
View File

@ -1,12 +1,9 @@
# flake8: noqa # flake8: noqa
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """BEGIN TRANSACTION; SQL_STATEMENT = """ALTER TABLE authentik_audit_event RENAME TO authentik_events_event;
ALTER TABLE authentik_audit_event RENAME TO authentik_events_event;
UPDATE django_migrations SET app = replace(app, 'authentik_audit', 'authentik_events'); UPDATE django_migrations SET app = replace(app, 'authentik_audit', 'authentik_events');
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_audit', 'authentik_events'); UPDATE django_content_type SET app_label = replace(app_label, 'authentik_audit', 'authentik_events');"""
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -17,5 +14,5 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)

9
lifecycle/system_migrations/to_2021_3_authenticator.py
View File

@ -1,7 +1,7 @@
# flake8: noqa # flake8: noqa
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """BEGIN TRANSACTION; SQL_STATEMENT = """
ALTER TABLE authentik_stages_otp_static_otpstaticstage RENAME TO authentik_stages_authenticator_static_otpstaticstage; ALTER TABLE authentik_stages_otp_static_otpstaticstage RENAME TO authentik_stages_authenticator_static_otpstaticstage;
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static'); UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static'); UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
@ -13,8 +13,7 @@ UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_
ALTER TABLE authentik_stages_otp_validate_otpvalidatestage RENAME TO authentik_stages_authenticator_validate_otpvalidatestage; ALTER TABLE authentik_stages_otp_validate_otpvalidatestage RENAME TO authentik_stages_authenticator_validate_otpvalidatestage;
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate'); UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate'); UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
"""
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -26,5 +25,5 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)

10
lifecycle/system_migrations/to_2023_1_hibp_remove.py
View File

@ -1,10 +1,8 @@
# flake8: noqa # flake8: noqa
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """BEGIN TRANSACTION; SQL_STATEMENT = """DROP TABLE "authentik_policies_hibp_haveibeenpwendpolicy";
DROP TABLE "authentik_policies_hibp_haveibeenpwendpolicy"; DELETE FROM django_migrations WHERE app = 'authentik_policies_hibp';"""
DELETE FROM django_migrations WHERE app = 'authentik_policies_hibp';
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -16,5 +14,5 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
self.cur.execute(SQL_STATEMENT) with self.con.transaction():
self.con.commit() self.cur.execute(SQL_STATEMENT)