trustchain-oc1-orchestral/authentik
Archived
10
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Revert "lifecycle: improve reliability of system migrations"

Browse source 
This reverts commit 3b8b307c4d.
This commit is contained in:
Jens Langhammer 2023-10-06 15:46:45 +02:00
parent 3b8b307c4d
commit 090d2d8362
No known key found for this signature in database
8 changed files with 98 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
lifecycle/migrate.py
View file

@ -1,12 +1,12 @@
#!/usr/bin/env python #!/usr/bin/env python
"""System Migration handler""" """System Migration handler"""
import os
from importlib.util import module_from_spec, spec_from_file_location from importlib.util import module_from_spec, spec_from_file_location
from inspect import getmembers, isclass from inspect import getmembers, isclass
from os import environ, system
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from psycopg import Connection, Cursor, connect from psycopg import connect
from structlog.stdlib import get_logger from structlog.stdlib import get_logger
from authentik.lib.config import CONFIG from authentik.lib.config import CONFIG
@ -19,24 +19,13 @@ LOCKED = False
class BaseMigration: class BaseMigration:
"""Base System Migration""" """Base System Migration"""
cur: Cursor cur: Any
con: Connection con: Any
def __init__(self, cur: Any, con: Any): def __init__(self, cur: Any, con: Any):
self.cur = cur self.cur = cur
self.con = con self.con = con
def system_crit(self, command: str):
"""Run system command"""
LOGGER.debug("Running system_crit command", command=command)
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def fake_migration(self, *app_migration: tuple[str, str]):
for app, migration in app_migration:
self.system_crit(f"./manage.py migrate {app} {migration} --fake")
def needs_migration(self) -> bool: def needs_migration(self) -> bool:
"""Return true if Migration needs to be run""" """Return true if Migration needs to be run"""
return False return False
@ -93,7 +82,7 @@ if __name__ == "__main__":
LOGGER.info("Migration finished applying", migration=sub) LOGGER.info("Migration finished applying", migration=sub)
release_lock() release_lock()
LOGGER.info("applying django migrations") LOGGER.info("applying django migrations")
environ.setdefault("DJANGO_SETTINGS_MODULE", "authentik.root.settings") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "authentik.root.settings")
wait_for_lock() wait_for_lock()
try: try:
from django.core.management import execute_from_command_line from django.core.management import execute_from_command_line

33
lifecycle/system_migrations/install_id.py
View file

@ -4,9 +4,11 @@ from uuid import uuid4
from authentik.lib.config import CONFIG from authentik.lib.config import CONFIG
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """CREATE TABLE IF NOT EXISTS authentik_install_id ( SQL_STATEMENT = """BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS authentik_install_id (
id TEXT NOT NULL id TEXT NOT NULL
);""" );
COMMIT;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -17,20 +19,19 @@ class Migration(BaseMigration):
return not bool(self.cur.rowcount) return not bool(self.cur.rowcount)
def upgrade(self, migrate=False): def upgrade(self, migrate=False):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()
self.con.commit() if migrate:
if migrate: # If we already have migrations in the database, assume we're upgrading an existing install
# If we already have migrations in the database, assume we're upgrading an existing install # and set the install id to the secret key
# and set the install id to the secret key self.cur.execute(
self.cur.execute( "INSERT INTO authentik_install_id (id) VALUES (%s)", (CONFIG.get("secret_key"),)
"INSERT INTO authentik_install_id (id) VALUES (%s)", (CONFIG.get("secret_key"),) )
) else:
else: # Otherwise assume a new install, generate an install ID based on a UUID
# Otherwise assume a new install, generate an install ID based on a UUID install_id = str(uuid4())
install_id = str(uuid4()) self.cur.execute("INSERT INTO authentik_install_id (id) VALUES (%s)", (install_id,))
self.cur.execute("INSERT INTO authentik_install_id (id) VALUES (%s)", (install_id,)) self.con.commit()
self.con.commit()
def run(self): def run(self):
self.cur.execute( self.cur.execute(

44
lifecycle/system_migrations/otp_merge.py
View file

@ -1,7 +1,10 @@
# flake8: noqa # flake8: noqa
from os import system
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ SQL_STATEMENT = """
BEGIN TRANSACTION;
DELETE FROM django_migrations WHERE app = 'otp_static'; DELETE FROM django_migrations WHERE app = 'otp_static';
DELETE FROM django_migrations WHERE app = 'otp_totp'; DELETE FROM django_migrations WHERE app = 'otp_totp';
-- Rename tables (static) -- Rename tables (static)
@ -12,7 +15,7 @@ ALTER SEQUENCE otp_static_staticdevice_id_seq RENAME TO authentik_stages_authent
-- Rename tables (totp) -- Rename tables (totp)
ALTER TABLE otp_totp_totpdevice RENAME TO authentik_stages_authenticator_totp_totpdevice; ALTER TABLE otp_totp_totpdevice RENAME TO authentik_stages_authenticator_totp_totpdevice;
ALTER SEQUENCE otp_totp_totpdevice_id_seq RENAME TO authentik_stages_authenticator_totp_totpdevice_id_seq; ALTER SEQUENCE otp_totp_totpdevice_id_seq RENAME TO authentik_stages_authenticator_totp_totpdevice_id_seq;
""" COMMIT;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -22,24 +25,23 @@ class Migration(BaseMigration):
) )
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def system_crit(self, command):
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def run(self): def run(self):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()
self.fake_migration( self.system_crit(
( "./manage.py migrate authentik_stages_authenticator_static 0008_initial --fake"
"authentik_stages_authenticator_static", )
"0008_initial", self.system_crit(
), "./manage.py migrate authentik_stages_authenticator_static 0009_throttling --fake"
( )
"authentik_stages_authenticator_static", self.system_crit(
"0009_throttling", "./manage.py migrate authentik_stages_authenticator_totp 0008_initial --fake"
), )
( self.system_crit(
"authentik_stages_authenticator_totp", "./manage.py migrate authentik_stages_authenticator_totp 0009_auto_20190420_0723 --fake"
"0008_initial", )
),
(
"authentik_stages_authenticator_totp",
"0009_auto_20190420_0723",
),
)

28
lifecycle/system_migrations/to_0_10.py
View file

@ -1,7 +1,10 @@
# flake8: noqa # flake8: noqa
from os import system
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ SQL_STATEMENT = """
BEGIN TRANSACTION;
DELETE FROM django_migrations WHERE app = 'passbook_stages_prompt'; DELETE FROM django_migrations WHERE app = 'passbook_stages_prompt';
DROP TABLE passbook_stages_prompt_prompt cascade; DROP TABLE passbook_stages_prompt_prompt cascade;
DROP TABLE passbook_stages_prompt_promptstage cascade; DROP TABLE passbook_stages_prompt_promptstage cascade;
@ -22,7 +25,7 @@ DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0008_defa
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0009_source_flows'; DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0009_source_flows';
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0010_provider_flows'; DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0010_provider_flows';
DELETE FROM django_migrations WHERE app = 'passbook_stages_password' AND name = '0002_passwordstage_change_flow'; DELETE FROM django_migrations WHERE app = 'passbook_stages_password' AND name = '0002_passwordstage_change_flow';
""" COMMIT;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -32,14 +35,17 @@ class Migration(BaseMigration):
) )
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def system_crit(self, command):
retval = system(command) # nosec
if retval != 0:
raise Exception("Migration error")
def run(self): def run(self):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()
self.system_crit("./manage.py migrate passbook_stages_prompt") self.system_crit("./manage.py migrate passbook_stages_prompt")
self.fake_migration( self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake")
("passbook_flows", "0008_default_flows"), self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake")
("passbook_flows", "0009_source_flows"), self.system_crit("./manage.py migrate passbook_flows 0010_provider_flows --fake")
("passbook_flows", "0010_provider_flows"), self.system_crit("./manage.py migrate passbook_flows")
) self.system_crit("./manage.py migrate passbook_stages_password --fake")
self.system_crit("./manage.py migrate passbook_flows")
self.fake_migration(("passbook_stages_password", ""))

35
lifecycle/system_migrations/to_0_13_authentik.py
View file

@ -4,7 +4,7 @@ from redis import Redis
from authentik.lib.config import CONFIG from authentik.lib.config import CONFIG
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ SQL_STATEMENT = """BEGIN TRANSACTION;
ALTER TABLE passbook_audit_event RENAME TO authentik_audit_event; ALTER TABLE passbook_audit_event RENAME TO authentik_audit_event;
ALTER TABLE passbook_core_application RENAME TO authentik_core_application; ALTER TABLE passbook_core_application RENAME TO authentik_core_application;
ALTER TABLE passbook_core_group RENAME TO authentik_core_group; ALTER TABLE passbook_core_group RENAME TO authentik_core_group;
@ -92,7 +92,8 @@ ALTER SEQUENCE passbook_stages_prompt_promptstage_validation_policies_id_seq REN
UPDATE django_migrations SET app = replace(app, 'passbook', 'authentik'); UPDATE django_migrations SET app = replace(app, 'passbook', 'authentik');
UPDATE django_content_type SET app_label = replace(app_label, 'passbook', 'authentik'); UPDATE django_content_type SET app_label = replace(app_label, 'passbook', 'authentik');
"""
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -103,18 +104,18 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()
# We also need to clean the cache to make sure no pickeled objects still exist # We also need to clean the cache to make sure no pickeled objects still exist
for db in [ for db in [
CONFIG.get("redis.message_queue_db"), CONFIG.get("redis.message_queue_db"),
CONFIG.get("redis.cache_db"), CONFIG.get("redis.cache_db"),
CONFIG.get("redis.ws_db"), CONFIG.get("redis.ws_db"),
]: ]:
redis = Redis( redis = Redis(
host=CONFIG.get("redis.host"), host=CONFIG.get("redis.host"),
port=6379, port=6379,
db=db, db=db,
password=CONFIG.get("redis.password"), password=CONFIG.get("redis.password"),
) )
redis.flushall() redis.flushall()

11
lifecycle/system_migrations/to_0_14_events..py
View file

@ -1,9 +1,12 @@
# flake8: noqa # flake8: noqa
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ALTER TABLE authentik_audit_event RENAME TO authentik_events_event; SQL_STATEMENT = """BEGIN TRANSACTION;
ALTER TABLE authentik_audit_event RENAME TO authentik_events_event;
UPDATE django_migrations SET app = replace(app, 'authentik_audit', 'authentik_events'); UPDATE django_migrations SET app = replace(app, 'authentik_audit', 'authentik_events');
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_audit', 'authentik_events');""" UPDATE django_content_type SET app_label = replace(app_label, 'authentik_audit', 'authentik_events');
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -14,5 +17,5 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()

9
lifecycle/system_migrations/to_2021_3_authenticator.py
View file

@ -1,7 +1,7 @@
# flake8: noqa # flake8: noqa
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """ SQL_STATEMENT = """BEGIN TRANSACTION;
ALTER TABLE authentik_stages_otp_static_otpstaticstage RENAME TO authentik_stages_authenticator_static_otpstaticstage; ALTER TABLE authentik_stages_otp_static_otpstaticstage RENAME TO authentik_stages_authenticator_static_otpstaticstage;
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static'); UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static'); UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
@ -13,7 +13,8 @@ UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_
ALTER TABLE authentik_stages_otp_validate_otpvalidatestage RENAME TO authentik_stages_authenticator_validate_otpvalidatestage; ALTER TABLE authentik_stages_otp_validate_otpvalidatestage RENAME TO authentik_stages_authenticator_validate_otpvalidatestage;
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate'); UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate'); UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
"""
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -25,5 +26,5 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()

10
lifecycle/system_migrations/to_2023_1_hibp_remove.py
View file

@ -1,8 +1,10 @@
# flake8: noqa # flake8: noqa
from lifecycle.migrate import BaseMigration from lifecycle.migrate import BaseMigration
SQL_STATEMENT = """DROP TABLE "authentik_policies_hibp_haveibeenpwendpolicy"; SQL_STATEMENT = """BEGIN TRANSACTION;
DELETE FROM django_migrations WHERE app = 'authentik_policies_hibp';""" DROP TABLE "authentik_policies_hibp_haveibeenpwendpolicy";
DELETE FROM django_migrations WHERE app = 'authentik_policies_hibp';
END TRANSACTION;"""
class Migration(BaseMigration): class Migration(BaseMigration):
@ -14,5 +16,5 @@ class Migration(BaseMigration):
return bool(self.cur.rowcount) return bool(self.cur.rowcount)
def run(self): def run(self):
with self.con.transaction(): self.cur.execute(SQL_STATEMENT)
self.cur.execute(SQL_STATEMENT) self.con.commit()