lifecycle: improve reliability of system migrations
better transaction handling which allows for re-trying migrations without needing manual intervention Signed-off-by: Jens Langhammer <jens@goauthentik.io>
This commit is contained in:
parent
9050759f6b
commit
3b8b307c4d
|
@ -1,12 +1,12 @@
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
"""System Migration handler"""
|
"""System Migration handler"""
|
||||||
import os
|
|
||||||
from importlib.util import module_from_spec, spec_from_file_location
|
from importlib.util import module_from_spec, spec_from_file_location
|
||||||
from inspect import getmembers, isclass
|
from inspect import getmembers, isclass
|
||||||
|
from os import environ, system
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from psycopg import connect
|
from psycopg import Connection, Cursor, connect
|
||||||
from structlog.stdlib import get_logger
|
from structlog.stdlib import get_logger
|
||||||
|
|
||||||
from authentik.lib.config import CONFIG
|
from authentik.lib.config import CONFIG
|
||||||
|
@ -19,13 +19,24 @@ LOCKED = False
|
||||||
class BaseMigration:
|
class BaseMigration:
|
||||||
"""Base System Migration"""
|
"""Base System Migration"""
|
||||||
|
|
||||||
cur: Any
|
cur: Cursor
|
||||||
con: Any
|
con: Connection
|
||||||
|
|
||||||
def __init__(self, cur: Any, con: Any):
|
def __init__(self, cur: Any, con: Any):
|
||||||
self.cur = cur
|
self.cur = cur
|
||||||
self.con = con
|
self.con = con
|
||||||
|
|
||||||
|
def system_crit(self, command: str):
|
||||||
|
"""Run system command"""
|
||||||
|
LOGGER.debug("Running system_crit command", command=command)
|
||||||
|
retval = system(command) # nosec
|
||||||
|
if retval != 0:
|
||||||
|
raise Exception("Migration error")
|
||||||
|
|
||||||
|
def fake_migration(self, *app_migration: tuple[str, str]):
|
||||||
|
for app, migration in app_migration:
|
||||||
|
self.system_crit(f"./manage.py migrate {app} {migration} --fake")
|
||||||
|
|
||||||
def needs_migration(self) -> bool:
|
def needs_migration(self) -> bool:
|
||||||
"""Return true if Migration needs to be run"""
|
"""Return true if Migration needs to be run"""
|
||||||
return False
|
return False
|
||||||
|
@ -82,7 +93,7 @@ if __name__ == "__main__":
|
||||||
LOGGER.info("Migration finished applying", migration=sub)
|
LOGGER.info("Migration finished applying", migration=sub)
|
||||||
release_lock()
|
release_lock()
|
||||||
LOGGER.info("applying django migrations")
|
LOGGER.info("applying django migrations")
|
||||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "authentik.root.settings")
|
environ.setdefault("DJANGO_SETTINGS_MODULE", "authentik.root.settings")
|
||||||
wait_for_lock()
|
wait_for_lock()
|
||||||
try:
|
try:
|
||||||
from django.core.management import execute_from_command_line
|
from django.core.management import execute_from_command_line
|
||||||
|
|
|
@ -4,11 +4,9 @@ from uuid import uuid4
|
||||||
from authentik.lib.config import CONFIG
|
from authentik.lib.config import CONFIG
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """BEGIN TRANSACTION;
|
SQL_STATEMENT = """CREATE TABLE IF NOT EXISTS authentik_install_id (
|
||||||
CREATE TABLE IF NOT EXISTS authentik_install_id (
|
|
||||||
id TEXT NOT NULL
|
id TEXT NOT NULL
|
||||||
);
|
);"""
|
||||||
COMMIT;"""
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -19,19 +17,20 @@ class Migration(BaseMigration):
|
||||||
return not bool(self.cur.rowcount)
|
return not bool(self.cur.rowcount)
|
||||||
|
|
||||||
def upgrade(self, migrate=False):
|
def upgrade(self, migrate=False):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
if migrate:
|
self.con.commit()
|
||||||
# If we already have migrations in the database, assume we're upgrading an existing install
|
if migrate:
|
||||||
# and set the install id to the secret key
|
# If we already have migrations in the database, assume we're upgrading an existing install
|
||||||
self.cur.execute(
|
# and set the install id to the secret key
|
||||||
"INSERT INTO authentik_install_id (id) VALUES (%s)", (CONFIG.get("secret_key"),)
|
self.cur.execute(
|
||||||
)
|
"INSERT INTO authentik_install_id (id) VALUES (%s)", (CONFIG.get("secret_key"),)
|
||||||
else:
|
)
|
||||||
# Otherwise assume a new install, generate an install ID based on a UUID
|
else:
|
||||||
install_id = str(uuid4())
|
# Otherwise assume a new install, generate an install ID based on a UUID
|
||||||
self.cur.execute("INSERT INTO authentik_install_id (id) VALUES (%s)", (install_id,))
|
install_id = str(uuid4())
|
||||||
self.con.commit()
|
self.cur.execute("INSERT INTO authentik_install_id (id) VALUES (%s)", (install_id,))
|
||||||
|
self.con.commit()
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(
|
self.cur.execute(
|
||||||
|
|
|
@ -1,10 +1,7 @@
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
from os import system
|
|
||||||
|
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """
|
SQL_STATEMENT = """
|
||||||
BEGIN TRANSACTION;
|
|
||||||
DELETE FROM django_migrations WHERE app = 'otp_static';
|
DELETE FROM django_migrations WHERE app = 'otp_static';
|
||||||
DELETE FROM django_migrations WHERE app = 'otp_totp';
|
DELETE FROM django_migrations WHERE app = 'otp_totp';
|
||||||
-- Rename tables (static)
|
-- Rename tables (static)
|
||||||
|
@ -15,7 +12,7 @@ ALTER SEQUENCE otp_static_staticdevice_id_seq RENAME TO authentik_stages_authent
|
||||||
-- Rename tables (totp)
|
-- Rename tables (totp)
|
||||||
ALTER TABLE otp_totp_totpdevice RENAME TO authentik_stages_authenticator_totp_totpdevice;
|
ALTER TABLE otp_totp_totpdevice RENAME TO authentik_stages_authenticator_totp_totpdevice;
|
||||||
ALTER SEQUENCE otp_totp_totpdevice_id_seq RENAME TO authentik_stages_authenticator_totp_totpdevice_id_seq;
|
ALTER SEQUENCE otp_totp_totpdevice_id_seq RENAME TO authentik_stages_authenticator_totp_totpdevice_id_seq;
|
||||||
COMMIT;"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -25,23 +22,24 @@ class Migration(BaseMigration):
|
||||||
)
|
)
|
||||||
return bool(self.cur.rowcount)
|
return bool(self.cur.rowcount)
|
||||||
|
|
||||||
def system_crit(self, command):
|
|
||||||
retval = system(command) # nosec
|
|
||||||
if retval != 0:
|
|
||||||
raise Exception("Migration error")
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
self.system_crit(
|
self.fake_migration(
|
||||||
"./manage.py migrate authentik_stages_authenticator_static 0008_initial --fake"
|
(
|
||||||
)
|
"authentik_stages_authenticator_static",
|
||||||
self.system_crit(
|
"0008_initial",
|
||||||
"./manage.py migrate authentik_stages_authenticator_static 0009_throttling --fake"
|
),
|
||||||
)
|
(
|
||||||
self.system_crit(
|
"authentik_stages_authenticator_static",
|
||||||
"./manage.py migrate authentik_stages_authenticator_totp 0008_initial --fake"
|
"0009_throttling",
|
||||||
)
|
),
|
||||||
self.system_crit(
|
(
|
||||||
"./manage.py migrate authentik_stages_authenticator_totp 0009_auto_20190420_0723 --fake"
|
"authentik_stages_authenticator_totp",
|
||||||
)
|
"0008_initial",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"authentik_stages_authenticator_totp",
|
||||||
|
"0009_auto_20190420_0723",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
|
@ -1,10 +1,7 @@
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
from os import system
|
|
||||||
|
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """
|
SQL_STATEMENT = """
|
||||||
BEGIN TRANSACTION;
|
|
||||||
DELETE FROM django_migrations WHERE app = 'passbook_stages_prompt';
|
DELETE FROM django_migrations WHERE app = 'passbook_stages_prompt';
|
||||||
DROP TABLE passbook_stages_prompt_prompt cascade;
|
DROP TABLE passbook_stages_prompt_prompt cascade;
|
||||||
DROP TABLE passbook_stages_prompt_promptstage cascade;
|
DROP TABLE passbook_stages_prompt_promptstage cascade;
|
||||||
|
@ -25,7 +22,7 @@ DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0008_defa
|
||||||
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0009_source_flows';
|
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0009_source_flows';
|
||||||
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0010_provider_flows';
|
DELETE FROM django_migrations WHERE app = 'passbook_flows' AND name = '0010_provider_flows';
|
||||||
DELETE FROM django_migrations WHERE app = 'passbook_stages_password' AND name = '0002_passwordstage_change_flow';
|
DELETE FROM django_migrations WHERE app = 'passbook_stages_password' AND name = '0002_passwordstage_change_flow';
|
||||||
COMMIT;"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -35,17 +32,14 @@ class Migration(BaseMigration):
|
||||||
)
|
)
|
||||||
return bool(self.cur.rowcount)
|
return bool(self.cur.rowcount)
|
||||||
|
|
||||||
def system_crit(self, command):
|
|
||||||
retval = system(command) # nosec
|
|
||||||
if retval != 0:
|
|
||||||
raise Exception("Migration error")
|
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
self.system_crit("./manage.py migrate passbook_stages_prompt")
|
self.system_crit("./manage.py migrate passbook_stages_prompt")
|
||||||
self.system_crit("./manage.py migrate passbook_flows 0008_default_flows --fake")
|
self.fake_migration(
|
||||||
self.system_crit("./manage.py migrate passbook_flows 0009_source_flows --fake")
|
("passbook_flows", "0008_default_flows"),
|
||||||
self.system_crit("./manage.py migrate passbook_flows 0010_provider_flows --fake")
|
("passbook_flows", "0009_source_flows"),
|
||||||
self.system_crit("./manage.py migrate passbook_flows")
|
("passbook_flows", "0010_provider_flows"),
|
||||||
self.system_crit("./manage.py migrate passbook_stages_password --fake")
|
)
|
||||||
|
self.system_crit("./manage.py migrate passbook_flows")
|
||||||
|
self.fake_migration(("passbook_stages_password", ""))
|
||||||
|
|
|
@ -4,7 +4,7 @@ from redis import Redis
|
||||||
from authentik.lib.config import CONFIG
|
from authentik.lib.config import CONFIG
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """BEGIN TRANSACTION;
|
SQL_STATEMENT = """
|
||||||
ALTER TABLE passbook_audit_event RENAME TO authentik_audit_event;
|
ALTER TABLE passbook_audit_event RENAME TO authentik_audit_event;
|
||||||
ALTER TABLE passbook_core_application RENAME TO authentik_core_application;
|
ALTER TABLE passbook_core_application RENAME TO authentik_core_application;
|
||||||
ALTER TABLE passbook_core_group RENAME TO authentik_core_group;
|
ALTER TABLE passbook_core_group RENAME TO authentik_core_group;
|
||||||
|
@ -92,8 +92,7 @@ ALTER SEQUENCE passbook_stages_prompt_promptstage_validation_policies_id_seq REN
|
||||||
|
|
||||||
UPDATE django_migrations SET app = replace(app, 'passbook', 'authentik');
|
UPDATE django_migrations SET app = replace(app, 'passbook', 'authentik');
|
||||||
UPDATE django_content_type SET app_label = replace(app_label, 'passbook', 'authentik');
|
UPDATE django_content_type SET app_label = replace(app_label, 'passbook', 'authentik');
|
||||||
|
"""
|
||||||
END TRANSACTION;"""
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -104,18 +103,18 @@ class Migration(BaseMigration):
|
||||||
return bool(self.cur.rowcount)
|
return bool(self.cur.rowcount)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
# We also need to clean the cache to make sure no pickeled objects still exist
|
# We also need to clean the cache to make sure no pickeled objects still exist
|
||||||
for db in [
|
for db in [
|
||||||
CONFIG.get("redis.message_queue_db"),
|
CONFIG.get("redis.message_queue_db"),
|
||||||
CONFIG.get("redis.cache_db"),
|
CONFIG.get("redis.cache_db"),
|
||||||
CONFIG.get("redis.ws_db"),
|
CONFIG.get("redis.ws_db"),
|
||||||
]:
|
]:
|
||||||
redis = Redis(
|
redis = Redis(
|
||||||
host=CONFIG.get("redis.host"),
|
host=CONFIG.get("redis.host"),
|
||||||
port=6379,
|
port=6379,
|
||||||
db=db,
|
db=db,
|
||||||
password=CONFIG.get("redis.password"),
|
password=CONFIG.get("redis.password"),
|
||||||
)
|
)
|
||||||
redis.flushall()
|
redis.flushall()
|
||||||
|
|
|
@ -1,12 +1,9 @@
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """BEGIN TRANSACTION;
|
SQL_STATEMENT = """ALTER TABLE authentik_audit_event RENAME TO authentik_events_event;
|
||||||
ALTER TABLE authentik_audit_event RENAME TO authentik_events_event;
|
|
||||||
UPDATE django_migrations SET app = replace(app, 'authentik_audit', 'authentik_events');
|
UPDATE django_migrations SET app = replace(app, 'authentik_audit', 'authentik_events');
|
||||||
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_audit', 'authentik_events');
|
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_audit', 'authentik_events');"""
|
||||||
|
|
||||||
END TRANSACTION;"""
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -17,5 +14,5 @@ class Migration(BaseMigration):
|
||||||
return bool(self.cur.rowcount)
|
return bool(self.cur.rowcount)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """BEGIN TRANSACTION;
|
SQL_STATEMENT = """
|
||||||
ALTER TABLE authentik_stages_otp_static_otpstaticstage RENAME TO authentik_stages_authenticator_static_otpstaticstage;
|
ALTER TABLE authentik_stages_otp_static_otpstaticstage RENAME TO authentik_stages_authenticator_static_otpstaticstage;
|
||||||
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
|
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
|
||||||
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
|
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_static', 'authentik_stages_authenticator_static');
|
||||||
|
@ -13,8 +13,7 @@ UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_
|
||||||
ALTER TABLE authentik_stages_otp_validate_otpvalidatestage RENAME TO authentik_stages_authenticator_validate_otpvalidatestage;
|
ALTER TABLE authentik_stages_otp_validate_otpvalidatestage RENAME TO authentik_stages_authenticator_validate_otpvalidatestage;
|
||||||
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
|
UPDATE django_migrations SET app = replace(app, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
|
||||||
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
|
UPDATE django_content_type SET app_label = replace(app_label, 'authentik_stages_otp_validate', 'authentik_stages_authenticator_validate');
|
||||||
|
"""
|
||||||
END TRANSACTION;"""
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -26,5 +25,5 @@ class Migration(BaseMigration):
|
||||||
return bool(self.cur.rowcount)
|
return bool(self.cur.rowcount)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
|
|
|
@ -1,10 +1,8 @@
|
||||||
# flake8: noqa
|
# flake8: noqa
|
||||||
from lifecycle.migrate import BaseMigration
|
from lifecycle.migrate import BaseMigration
|
||||||
|
|
||||||
SQL_STATEMENT = """BEGIN TRANSACTION;
|
SQL_STATEMENT = """DROP TABLE "authentik_policies_hibp_haveibeenpwendpolicy";
|
||||||
DROP TABLE "authentik_policies_hibp_haveibeenpwendpolicy";
|
DELETE FROM django_migrations WHERE app = 'authentik_policies_hibp';"""
|
||||||
DELETE FROM django_migrations WHERE app = 'authentik_policies_hibp';
|
|
||||||
END TRANSACTION;"""
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(BaseMigration):
|
class Migration(BaseMigration):
|
||||||
|
@ -16,5 +14,5 @@ class Migration(BaseMigration):
|
||||||
return bool(self.cur.rowcount)
|
return bool(self.cur.rowcount)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
self.cur.execute(SQL_STATEMENT)
|
with self.con.transaction():
|
||||||
self.con.commit()
|
self.cur.execute(SQL_STATEMENT)
|
||||||
|
|
Reference in New Issue