From ab3d47c437077eb384611887da8a8ff5de0f0d99 Mon Sep 17 00:00:00 2001 From: Jens L Date: Tue, 22 Nov 2022 14:27:20 +0100 Subject: [PATCH] blueprints: add desired state attribute to objects (#4061) * add state attribute to delete objects Signed-off-by: Jens Langhammer * add tests, move yaml from block to files Signed-off-by: Jens Langhammer * add state to docs Signed-off-by: Jens Langhammer * only try to format Signed-off-by: Jens Langhammer Signed-off-by: Jens Langhammer --- .vscode/settings.json | 7 +- .../management/commands/schema_template.json | 9 ++ authentik/blueprints/tests/__init__.py | 11 ++- .../tests/fixtures/state_absent.yaml | 7 ++ .../tests/fixtures/state_created.yaml | 10 +++ .../tests/fixtures/state_present.yaml | 10 +++ .../tests/fixtures/static_prompt_export.yaml | 12 +++ authentik/blueprints/tests/fixtures/tags.yaml | 10 +++ authentik/blueprints/tests/test_v1.py | 33 +------- authentik/blueprints/tests/test_v1_state.py | 82 +++++++++++++++++++ authentik/blueprints/v1/common.py | 16 ++++ authentik/blueprints/v1/importer.py | 38 +++++++-- blueprints/schema.json | 9 ++ website/developer-docs/blueprints/export.md | 2 +- .../developer-docs/blueprints/v1/structure.md | 5 ++ 15 files changed, 220 insertions(+), 41 deletions(-) create mode 100644 authentik/blueprints/tests/fixtures/state_absent.yaml create mode 100644 authentik/blueprints/tests/fixtures/state_created.yaml create mode 100644 authentik/blueprints/tests/fixtures/state_present.yaml create mode 100644 authentik/blueprints/tests/fixtures/static_prompt_export.yaml create mode 100644 authentik/blueprints/tests/fixtures/tags.yaml create mode 100644 authentik/blueprints/tests/test_v1_state.py diff --git a/.vscode/settings.json b/.vscode/settings.json index aaf7de88b..6858a685d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -20,7 +20,12 @@ "todo-tree.tree.showCountsInTree": true, "todo-tree.tree.showBadges": true, "python.formatting.provider": "black", - "yaml.customTags": ["!Find sequence", "!KeyOf scalar"], + "yaml.customTags": [ + "!Find sequence", + "!KeyOf scalar", + "!Context scalar", + "!Format sequence" + ], "typescript.preferences.importModuleSpecifier": "non-relative", "typescript.preferences.importModuleSpecifierEnding": "index", "typescript.tsdk": "./web/node_modules/typescript/lib", diff --git a/authentik/blueprints/management/commands/schema_template.json b/authentik/blueprints/management/commands/schema_template.json index e264e0a8e..c00c275d8 100644 --- a/authentik/blueprints/management/commands/schema_template.json +++ b/authentik/blueprints/management/commands/schema_template.json @@ -53,6 +53,15 @@ "id": { "type": "string" }, + "state": { + "type": "string", + "enum": [ + "absent", + "present", + "created" + ], + "default": "present" + }, "attrs": { "type": "object", "properties": { diff --git a/authentik/blueprints/tests/__init__.py b/authentik/blueprints/tests/__init__.py index 6ceb7c631..2bfef0718 100644 --- a/authentik/blueprints/tests/__init__.py +++ b/authentik/blueprints/tests/__init__.py @@ -7,7 +7,6 @@ from django.apps import apps from authentik.blueprints.apps import ManagedAppConfig from authentik.blueprints.models import BlueprintInstance -from authentik.lib.config import CONFIG def apply_blueprint(*files: str): @@ -46,3 +45,13 @@ def reconcile_app(app_name: str): return wrapper return wrapper_outer + + +def load_yaml_fixture(path: str, **kwargs) -> str: + """Load yaml fixture, optionally formatting it with kwargs""" + with open(Path(__file__).resolve().parent / Path(path), "r+", encoding="utf-8") as _fixture: + fixture = _fixture.read() + try: + return fixture % kwargs + except TypeError: + return fixture diff --git a/authentik/blueprints/tests/fixtures/state_absent.yaml b/authentik/blueprints/tests/fixtures/state_absent.yaml new file mode 100644 index 000000000..e1588c940 --- /dev/null +++ b/authentik/blueprints/tests/fixtures/state_absent.yaml @@ -0,0 +1,7 @@ +version: 1 +entries: +- identifiers: + name: "%(id)s" + slug: "%(id)s" + model: authentik_flows.flow + state: absent diff --git a/authentik/blueprints/tests/fixtures/state_created.yaml b/authentik/blueprints/tests/fixtures/state_created.yaml new file mode 100644 index 000000000..8091fd600 --- /dev/null +++ b/authentik/blueprints/tests/fixtures/state_created.yaml @@ -0,0 +1,10 @@ +version: 1 +entries: +- identifiers: + name: "%(id)s" + slug: "%(id)s" + model: authentik_flows.flow + state: created + attrs: + designation: stage_configuration + title: foo diff --git a/authentik/blueprints/tests/fixtures/state_present.yaml b/authentik/blueprints/tests/fixtures/state_present.yaml new file mode 100644 index 000000000..86286112b --- /dev/null +++ b/authentik/blueprints/tests/fixtures/state_present.yaml @@ -0,0 +1,10 @@ +version: 1 +entries: +- identifiers: + name: "%(id)s" + slug: "%(id)s" + model: authentik_flows.flow + state: present + attrs: + designation: stage_configuration + title: foo diff --git a/authentik/blueprints/tests/fixtures/static_prompt_export.yaml b/authentik/blueprints/tests/fixtures/static_prompt_export.yaml new file mode 100644 index 000000000..8d7dd7f58 --- /dev/null +++ b/authentik/blueprints/tests/fixtures/static_prompt_export.yaml @@ -0,0 +1,12 @@ +version: 1 +entries: +- identifiers: + pk: cb954fd4-65a5-4ad9-b1ee-180ee9559cf4 + model: authentik_stages_prompt.prompt + attrs: + field_key: username + label: Username + type: username + required: true + placeholder: Username + order: 0 diff --git a/authentik/blueprints/tests/fixtures/tags.yaml b/authentik/blueprints/tests/fixtures/tags.yaml new file mode 100644 index 000000000..6cb823690 --- /dev/null +++ b/authentik/blueprints/tests/fixtures/tags.yaml @@ -0,0 +1,10 @@ +version: 1 +context: + foo: bar +entries: +- attrs: + expression: return True + identifiers: + name: !Format [foo-%s-%s, !Context foo, !Context bar] + id: default-source-enrollment-if-username + model: authentik_policies_expression.expressionpolicy diff --git a/authentik/blueprints/tests/test_v1.py b/authentik/blueprints/tests/test_v1.py index a5a04f664..cdd543f01 100644 --- a/authentik/blueprints/tests/test_v1.py +++ b/authentik/blueprints/tests/test_v1.py @@ -1,6 +1,7 @@ """Test blueprints v1""" from django.test import TransactionTestCase +from authentik.blueprints.tests import load_yaml_fixture from authentik.blueprints.v1.exporter import FlowExporter from authentik.blueprints.v1.importer import Importer, transaction_rollback from authentik.flows.models import Flow, FlowDesignation, FlowStageBinding @@ -10,32 +11,6 @@ from authentik.policies.models import PolicyBinding from authentik.stages.prompt.models import FieldTypes, Prompt, PromptStage from authentik.stages.user_login.models import UserLoginStage -STATIC_PROMPT_EXPORT = """version: 1 -entries: -- identifiers: - pk: cb954fd4-65a5-4ad9-b1ee-180ee9559cf4 - model: authentik_stages_prompt.prompt - attrs: - field_key: username - label: Username - type: username - required: true - placeholder: Username - order: 0 -""" - -YAML_TAG_TESTS = """version: 1 -context: - foo: bar -entries: -- attrs: - expression: return True - identifiers: - name: !Format [foo-%s-%s, !Context foo, !Context bar] - id: default-source-enrollment-if-username - model: authentik_policies_expression.expressionpolicy -""" - class TestBlueprintsV1(TransactionTestCase): """Test Blueprints""" @@ -85,14 +60,14 @@ class TestBlueprintsV1(TransactionTestCase): """Test export and import it twice""" count_initial = Prompt.objects.filter(field_key="username").count() - importer = Importer(STATIC_PROMPT_EXPORT) + importer = Importer(load_yaml_fixture("fixtures/static_prompt_export.yaml")) self.assertTrue(importer.validate()[0]) self.assertTrue(importer.apply()) count_before = Prompt.objects.filter(field_key="username").count() self.assertEqual(count_initial + 1, count_before) - importer = Importer(STATIC_PROMPT_EXPORT) + importer = Importer(load_yaml_fixture("fixtures/static_prompt_export.yaml")) self.assertTrue(importer.apply()) self.assertEqual(Prompt.objects.filter(field_key="username").count(), count_before) @@ -100,7 +75,7 @@ class TestBlueprintsV1(TransactionTestCase): def test_import_yaml_tags(self): """Test some yaml tags""" ExpressionPolicy.objects.filter(name="foo-foo-bar").delete() - importer = Importer(YAML_TAG_TESTS, {"bar": "baz"}) + importer = Importer(load_yaml_fixture("fixtures/tags.yaml"), {"bar": "baz"}) self.assertTrue(importer.validate()[0]) self.assertTrue(importer.apply()) self.assertTrue(ExpressionPolicy.objects.filter(name="foo-foo-bar")) diff --git a/authentik/blueprints/tests/test_v1_state.py b/authentik/blueprints/tests/test_v1_state.py new file mode 100644 index 000000000..02ddfcb50 --- /dev/null +++ b/authentik/blueprints/tests/test_v1_state.py @@ -0,0 +1,82 @@ +"""Test blueprints v1""" +from django.test import TransactionTestCase + +from authentik.blueprints.tests import load_yaml_fixture +from authentik.blueprints.v1.importer import Importer +from authentik.flows.models import Flow +from authentik.lib.generators import generate_id + + +class TestBlueprintsV1State(TransactionTestCase): + """Test Blueprints state attribute""" + + def test_state_present(self): + """Test state present""" + flow_slug = generate_id() + import_yaml = load_yaml_fixture("fixtures/state_present.yaml", id=flow_slug) + + importer = Importer(import_yaml) + self.assertTrue(importer.validate()[0]) + self.assertTrue(importer.apply()) + # Ensure object exists + flow: Flow = Flow.objects.filter(slug=flow_slug).first() + self.assertEqual(flow.slug, flow_slug) + + # Update object + flow.title = "bar" + flow.save() + + flow.refresh_from_db() + self.assertEqual(flow.title, "bar") + + # Ensure importer updates it + importer = Importer(import_yaml) + self.assertTrue(importer.validate()[0]) + self.assertTrue(importer.apply()) + flow: Flow = Flow.objects.filter(slug=flow_slug).first() + self.assertEqual(flow.title, "foo") + + def test_state_created(self): + """Test state created""" + flow_slug = generate_id() + import_yaml = load_yaml_fixture("fixtures/state_created.yaml", id=flow_slug) + + importer = Importer(import_yaml) + self.assertTrue(importer.validate()[0]) + self.assertTrue(importer.apply()) + # Ensure object exists + flow: Flow = Flow.objects.filter(slug=flow_slug).first() + self.assertEqual(flow.slug, flow_slug) + + # Update object + flow.title = "bar" + flow.save() + + flow.refresh_from_db() + self.assertEqual(flow.title, "bar") + + # Ensure importer doesn't update it + importer = Importer(import_yaml) + self.assertTrue(importer.validate()[0]) + self.assertTrue(importer.apply()) + flow: Flow = Flow.objects.filter(slug=flow_slug).first() + self.assertEqual(flow.title, "bar") + + def test_state_absent(self): + """Test state absent""" + flow_slug = generate_id() + import_yaml = load_yaml_fixture("fixtures/state_created.yaml", id=flow_slug) + + importer = Importer(import_yaml) + self.assertTrue(importer.validate()[0]) + self.assertTrue(importer.apply()) + # Ensure object exists + flow: Flow = Flow.objects.filter(slug=flow_slug).first() + self.assertEqual(flow.slug, flow_slug) + + import_yaml = load_yaml_fixture("fixtures/state_absent.yaml", id=flow_slug) + importer = Importer(import_yaml) + self.assertTrue(importer.validate()[0]) + self.assertTrue(importer.apply()) + flow: Flow = Flow.objects.filter(slug=flow_slug).first() + self.assertIsNone(flow) diff --git a/authentik/blueprints/v1/common.py b/authentik/blueprints/v1/common.py index 7ce8251e3..52454486b 100644 --- a/authentik/blueprints/v1/common.py +++ b/authentik/blueprints/v1/common.py @@ -41,11 +41,20 @@ class BlueprintEntryState: instance: Optional[Model] = None +class BlueprintEntryDesiredState(Enum): + """State an entry should be reconciled to""" + + ABSENT = "absent" + PRESENT = "present" + CREATED = "created" + + @dataclass class BlueprintEntry: """Single entry of a blueprint""" model: str + state: BlueprintEntryDesiredState = field(default=BlueprintEntryDesiredState.PRESENT) identifiers: dict[str, Any] = field(default_factory=dict) attrs: Optional[dict[str, Any]] = field(default_factory=dict) @@ -227,8 +236,15 @@ class BlueprintDumper(SafeDumper): self.add_representer(UUID, lambda self, data: self.represent_str(str(data))) self.add_representer(OrderedDict, lambda self, data: self.represent_dict(dict(data))) self.add_representer(Enum, lambda self, data: self.represent_str(data.value)) + self.add_representer( + BlueprintEntryDesiredState, lambda self, data: self.represent_str(data.value) + ) self.add_representer(None, lambda self, data: self.represent_str(str(data))) + def ignore_aliases(self, data): + """Don't use any YAML anchors""" + return True + def represent(self, data) -> None: if is_dataclass(data): diff --git a/authentik/blueprints/v1/importer.py b/authentik/blueprints/v1/importer.py index 37ec25707..df44f1289 100644 --- a/authentik/blueprints/v1/importer.py +++ b/authentik/blueprints/v1/importer.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from copy import deepcopy from typing import Any, Optional +from dacite.config import Config from dacite.core import from_dict from dacite.exceptions import DaciteError from deepmerge import always_merger @@ -20,6 +21,7 @@ from yaml import load from authentik.blueprints.v1.common import ( Blueprint, BlueprintEntry, + BlueprintEntryDesiredState, BlueprintEntryState, BlueprintLoader, EntryInvalidError, @@ -82,7 +84,9 @@ class Importer: self.logger = get_logger() import_dict = load(yaml_input, BlueprintLoader) try: - self.__import = from_dict(Blueprint, import_dict) + self.__import = from_dict( + Blueprint, import_dict, config=Config(cast=[BlueprintEntryDesiredState]) + ) except DaciteError as exc: raise EntryInvalidError from exc ctx = {} @@ -135,7 +139,7 @@ class Importer: sub_query &= Q(**{identifier: value}) return main_query | sub_query - def _validate_single(self, entry: BlueprintEntry) -> BaseSerializer: + def _validate_single(self, entry: BlueprintEntry) -> Optional[BaseSerializer]: """Validate a single entry""" model_app_label, model_name = entry.model.split(".") model: type[SerializerModel] = registry.get_model(model_app_label, model_name) @@ -168,8 +172,11 @@ class Importer: existing_models = model.objects.filter(self.__query_from_identifier(updated_identifiers)) serializer_kwargs = {} - if not isinstance(model(), BaseMetaModel) and existing_models.exists(): - model_instance = existing_models.first() + model_instance = existing_models.first() + if not isinstance(model(), BaseMetaModel) and model_instance: + if entry.state == BlueprintEntryDesiredState.CREATED: + self.logger.debug("instance exists, skipping") + return None self.logger.debug( "initialise serializer with instance", model=model, @@ -234,12 +241,25 @@ class Importer: except EntryInvalidError as exc: self.logger.warning(f"entry invalid: {exc}", entry=entry, error=exc) return False + if not serializer: + continue - model = serializer.save() - if "pk" in entry.identifiers: - self.__pk_map[entry.identifiers["pk"]] = model.pk - entry._state = BlueprintEntryState(model) - self.logger.debug("updated model", model=model) + if entry.state in [ + BlueprintEntryDesiredState.PRESENT, + BlueprintEntryDesiredState.CREATED, + ]: + model = serializer.save() + if "pk" in entry.identifiers: + self.__pk_map[entry.identifiers["pk"]] = model.pk + entry._state = BlueprintEntryState(model) + self.logger.debug("updated model", model=model) + elif entry.state == BlueprintEntryDesiredState.ABSENT: + instance: Optional[Model] = serializer.instance + if instance: + instance.delete() + self.logger.debug("deleted model", mode=instance) + continue + self.logger.debug("entry to delete with no instance, skipping") return True def validate(self) -> tuple[bool, list[EventDict]]: diff --git a/blueprints/schema.json b/blueprints/schema.json index 45119dffb..9251ab1c6 100644 --- a/blueprints/schema.json +++ b/blueprints/schema.json @@ -121,6 +121,15 @@ "id": { "type": "string" }, + "state": { + "type": "string", + "enum": [ + "absent", + "present", + "created" + ], + "default": "present" + }, "attrs": { "type": "object", "properties": { diff --git a/website/developer-docs/blueprints/export.md b/website/developer-docs/blueprints/export.md index aa4f4ea07..fa0203197 100644 --- a/website/developer-docs/blueprints/export.md +++ b/website/developer-docs/blueprints/export.md @@ -24,4 +24,4 @@ This export can be triggered via the API or the Web UI by clicking the download ## Cleaning up -Exports from either method will contain a (potentially) long list of objects, all with hardcoded primary keys and now ability for templating/instantiation. This is because currently, authentik does not check which primary keys are used where. It is assumed that for most exports, there'll be some manual changes done regardless, to filter out unwanted objects, adjust properties, etc. +Exports from either method will contain a (potentially) long list of objects, all with hardcoded primary keys and no ability for templating/instantiation. This is because currently, authentik does not check which primary keys are used where. It is assumed that for most exports, there'll be some manual changes done regardless, to filter out unwanted objects, adjust properties, etc. diff --git a/website/developer-docs/blueprints/v1/structure.md b/website/developer-docs/blueprints/v1/structure.md index 3085c2a4e..1bcfda383 100644 --- a/website/developer-docs/blueprints/v1/structure.md +++ b/website/developer-docs/blueprints/v1/structure.md @@ -20,6 +20,11 @@ context: entries: - # Model in app.model notation, possibilities are listed in the schema (required) model: authentik_flows.flow + # The state this object should be in (optional, can be "present", "created" or "absent") + # Present will keep the object in sync with its definition here, created will only ensure + # the object is created (and create it with the values given here), and "absent" will + # delete the object + state: present # Key:value filters to uniquely identify this object (required) identifiers: slug: initial-setup