blueprints: Add !Enumerate, !Value and !Index tags (#4338)

* Added For and Item tags

* Removed Sequence node support from ForItem tag

* Added ForItemIndex tag

* Added support for iterating over mappings

* Added support for mapping output body

* Renamed tags: For to Enumerate, ForItem to Value, ForItemIndex to Index

* Refactored tests

* Formatting

* Improved exception info

* Improved error handing

* Added docs

* lint

* Small doc improvements

* Replaced deepcopy() call with call to copy()

* Fix mistake in docs example

* Fix missed "!" in example

authentik/blueprints/tests/fixtures/tags.yaml

@@ -3,6 +3,12 @@ context:
     foo: bar
     policy_property: name
     policy_property_value: foo-bar-baz-qux
+    sequence:
+    - foo
+    - bar
+    mapping:
+      key1: value
+      key2: 2
 entries:
     - model: !Format ["%s", authentik_sources_oauth.oauthsource]
       state: !Format ["%s", present]
@@ -92,6 +98,49 @@ entries:
                   ]
               if_true_simple: !If [!Context foo, true, text]
               if_false_simple: !If [null, false, 2]
+              enumerate_mapping_to_mapping: !Enumerate [
+                  !Context mapping,
+                  MAP,
+                  [!Format ["prefix-%s", !Index 0], !Format ["other-prefix-%s", !Value 0]]
+              ]
+              enumerate_mapping_to_sequence: !Enumerate [
+                  !Context mapping,
+                  SEQ,
+                  !Format ["prefixed-pair-%s-%s", !Index 0, !Value 0]
+              ]
+              enumerate_sequence_to_sequence: !Enumerate [
+                  !Context sequence,
+                  SEQ,
+                  !Format ["prefixed-items-%s-%s", !Index 0, !Value 0]
+              ]
+              enumerate_sequence_to_mapping: !Enumerate [
+                  !Context sequence,
+                  MAP,
+                  [!Format ["index: %d", !Index 0], !Value 0]
+              ]
+              nested_complex_enumeration: !Enumerate [
+                  !Context sequence,
+                  MAP,
+                  [
+                      !Index 0,
+                      !Enumerate [
+                          !Context mapping,
+                          MAP,
+                          [
+                              !Format ["%s", !Index 0],
+                              [
+                                  !Enumerate [!Value 2, SEQ, !Format ["prefixed-%s", !Value 0]],
+                                  {
+                                    outer_value: !Value 1,
+                                    outer_index: !Index 1,
+                                    middle_value: !Value 0,
+                                    middle_index: !Index 0
+                                  }
+                              ]
+                          ]
+                      ]
+                  ]
+              ]
       identifiers:
           name: test
       conditions:

authentik/blueprints/tests/test_v1.py

@@ -162,6 +162,61 @@ class TestBlueprintsV1(TransactionTestCase):
                     "if_false_complex": ["list", "with", "items", "foo-bar"],
                     "if_true_simple": True,
                     "if_false_simple": 2,
+                    "enumerate_mapping_to_mapping": {
+                        "prefix-key1": "other-prefix-value",
+                        "prefix-key2": "other-prefix-2",
+                    },
+                    "enumerate_mapping_to_sequence": [
+                        "prefixed-pair-key1-value",
+                        "prefixed-pair-key2-2",
+                    ],
+                    "enumerate_sequence_to_sequence": [
+                        "prefixed-items-0-foo",
+                        "prefixed-items-1-bar",
+                    ],
+                    "enumerate_sequence_to_mapping": {"index: 0": "foo", "index: 1": "bar"},
+                    "nested_complex_enumeration": {
+                        "0": {
+                            "key1": [
+                                ["prefixed-f", "prefixed-o", "prefixed-o"],
+                                {
+                                    "outer_value": "foo",
+                                    "outer_index": 0,
+                                    "middle_value": "value",
+                                    "middle_index": "key1",
+                                },
+                            ],
+                            "key2": [
+                                ["prefixed-f", "prefixed-o", "prefixed-o"],
+                                {
+                                    "outer_value": "foo",
+                                    "outer_index": 0,
+                                    "middle_value": 2,
+                                    "middle_index": "key2",
+                                },
+                            ],
+                        },
+                        "1": {
+                            "key1": [
+                                ["prefixed-b", "prefixed-a", "prefixed-r"],
+                                {
+                                    "outer_value": "bar",
+                                    "outer_index": 1,
+                                    "middle_value": "value",
+                                    "middle_index": "key1",
+                                },
+                            ],
+                            "key2": [
+                                ["prefixed-b", "prefixed-a", "prefixed-r"],
+                                {
+                                    "outer_value": "bar",
+                                    "outer_index": 1,
+                                    "middle_value": 2,
+                                    "middle_index": "key2",
+                                },
+                            ],
+                        },
+                    },
                 }
             )
         )

authentik/blueprints/v1/common.py

@@ -1,13 +1,15 @@
 """transfer common classes"""
 from collections import OrderedDict
+from copy import copy
 from dataclasses import asdict, dataclass, field, is_dataclass
 from enum import Enum
 from functools import reduce
 from operator import ixor
 from os import getenv
-from typing import Any, Literal, Optional, Union
+from typing import Any, Iterable, Literal, Mapping, Optional, Union
 from uuid import UUID
 
+from deepmerge import always_merger
 from django.apps import apps
 from django.db.models import Model, Q
 from rest_framework.fields import Field
@@ -69,6 +71,9 @@ class BlueprintEntry:
 
     _state: BlueprintEntryState = field(default_factory=BlueprintEntryState)
 
+    def __post_init__(self, *args, **kwargs) -> None:
+        self.__tag_contexts: list["YAMLTagContext"] = []
+
     @staticmethod
     def from_model(model: SerializerModel, *extra_identifier_names: str) -> "BlueprintEntry":
         """Convert a SerializerModel instance to a blueprint Entry"""
@@ -85,17 +90,46 @@ class BlueprintEntry:
             attrs=all_attrs,
         )
 
+    def _get_tag_context(
+        self,
+        depth: int = 0,
+        context_tag_type: Optional[type["YAMLTagContext"] | tuple["YAMLTagContext", ...]] = None,
+    ) -> "YAMLTagContext":
+        """Get a YAMLTagContex object located at a certain depth in the tag tree"""
+        if depth < 0:
+            raise ValueError("depth must be a positive number or zero")
+
+        if context_tag_type:
+            contexts = [x for x in self.__tag_contexts if isinstance(x, context_tag_type)]
+        else:
+            contexts = self.__tag_contexts
+
+        try:
+            return contexts[-(depth + 1)]
+        except IndexError:
+            raise ValueError(f"invalid depth: {depth}. Max depth: {len(contexts) - 1}")
+
     def tag_resolver(self, value: Any, blueprint: "Blueprint") -> Any:
         """Check if we have any special tags that need handling"""
+        val = copy(value)
+
+        if isinstance(value, YAMLTagContext):
+            self.__tag_contexts.append(value)
+
         if isinstance(value, YAMLTag):
-            return value.resolve(self, blueprint)
+            val = value.resolve(self, blueprint)
+
         if isinstance(value, dict):
             for key, inner_value in value.items():
-                value[key] = self.tag_resolver(inner_value, blueprint)
+                val[key] = self.tag_resolver(inner_value, blueprint)
         if isinstance(value, list):
             for idx, inner_value in enumerate(value):
-                value[idx] = self.tag_resolver(inner_value, blueprint)
+                val[idx] = self.tag_resolver(inner_value, blueprint)
-        return value
+
+        if isinstance(value, YAMLTagContext):
+            self.__tag_contexts.pop()
+
+        return val
 
     def get_attrs(self, blueprint: "Blueprint") -> dict[str, Any]:
         """Get attributes of this entry, with all yaml tags resolved"""
@@ -145,6 +179,14 @@ class YAMLTag:
         raise NotImplementedError
 
 
+class YAMLTagContext:
+    """Base class for all YAML Tag Contexts"""
+
+    def get_context(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
+        """Implement yaml tag context logic"""
+        raise NotImplementedError
+
+
 class KeyOf(YAMLTag):
     """Reference another object by their ID"""
 
@@ -351,6 +393,136 @@ class If(YAMLTag):
             raise EntryInvalidError(exc)
 
 
+class Enumerate(YAMLTag, YAMLTagContext):
+    """Iterate over an iterable."""
+
+    iterable: YAMLTag | Iterable
+    item_body: Any
+    output_body: Literal["SEQ", "MAP"]
+
+    _OUTPUT_BODIES = {
+        "SEQ": (list, lambda a, b: [*a, b]),
+        "MAP": (
+            dict,
+            lambda a, b: always_merger.merge(
+                a, {b[0]: b[1]} if isinstance(b, (tuple, list)) else b
+            ),
+        ),
+    }
+
+    # pylint: disable=unused-argument
+    def __init__(self, loader: "BlueprintLoader", node: SequenceNode) -> None:
+        super().__init__()
+        self.iterable = loader.construct_object(node.value[0])
+        self.output_body = node.value[1].value
+        self.item_body = loader.construct_object(node.value[2])
+        self.__current_context: tuple[Any, Any] = tuple()
+
+    # pylint: disable=unused-argument
+    def get_context(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
+        return self.__current_context
+
+    def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
+        if isinstance(self.iterable, EnumeratedItem) and self.iterable.depth == 0:
+            raise EntryInvalidError(
+                f"{self.__class__.__name__} tag's iterable references this tag's context. "
+                "This is a noop. Check you are setting depth bigger than 0."
+            )
+
+        if isinstance(self.iterable, YAMLTag):
+            iterable = self.iterable.resolve(entry, blueprint)
+        else:
+            iterable = self.iterable
+
+        if not isinstance(iterable, Iterable):
+            raise EntryInvalidError(
+                f"{self.__class__.__name__}'s iterable must be an iterable "
+                "such as a sequence or a mapping"
+            )
+
+        if isinstance(iterable, Mapping):
+            iterable = tuple(iterable.items())
+        else:
+            iterable = tuple(enumerate(iterable))
+
+        try:
+            output_class, add_fn = self._OUTPUT_BODIES[self.output_body.upper()]
+        except KeyError as exc:
+            raise EntryInvalidError(exc)
+
+        result = output_class()
+
+        self.__current_context = tuple()
+
+        try:
+            for item in iterable:
+                self.__current_context = item
+                resolved_body = entry.tag_resolver(self.item_body, blueprint)
+                result = add_fn(result, resolved_body)
+                if not isinstance(result, output_class):
+                    raise EntryInvalidError(
+                        f"Invalid {self.__class__.__name__} item found: {resolved_body}"
+                    )
+        finally:
+            self.__current_context = tuple()
+
+        return result
+
+
+class EnumeratedItem(YAMLTag):
+    """Get the current item value and index provided by an Enumerate tag context"""
+
+    depth: int
+
+    _SUPPORTED_CONTEXT_TAGS = (Enumerate,)
+
+    # pylint: disable=unused-argument
+    def __init__(self, loader: "BlueprintLoader", node: ScalarNode) -> None:
+        super().__init__()
+        self.depth = int(node.value)
+
+    def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
+        try:
+            context_tag: Enumerate = entry._get_tag_context(
+                depth=self.depth,
+                context_tag_type=EnumeratedItem._SUPPORTED_CONTEXT_TAGS,
+            )
+        except ValueError as exc:
+            if self.depth == 0:
+                raise EntryInvalidError(
+                    f"{self.__class__.__name__} tags are only usable "
+                    f"inside an {Enumerate.__name__} tag"
+                )
+
+            raise EntryInvalidError(f"{self.__class__.__name__} tag: {exc}")
+
+        return context_tag.get_context(entry, blueprint)
+
+
+class Index(EnumeratedItem):
+    """Get the current item index provided by an Enumerate tag context"""
+
+    def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
+        context = super().resolve(entry, blueprint)
+
+        try:
+            return context[0]
+        except IndexError:  # pragma: no cover
+            raise EntryInvalidError(f"Empty/invalid context: {context}")
+
+
+class Value(EnumeratedItem):
+    """Get the current item value provided by an Enumerate tag context"""
+
+    def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
+        context = super().resolve(entry, blueprint)
+
+        try:
+            return context[1]
+        except IndexError:  # pragma: no cover
+            raise EntryInvalidError(f"Empty/invalid context: {context}")
+
+
 class BlueprintDumper(SafeDumper):
     """Dump dataclasses to yaml"""
 
@@ -394,6 +566,9 @@ class BlueprintLoader(SafeLoader):
         self.add_constructor("!Condition", Condition)
         self.add_constructor("!If", If)
         self.add_constructor("!Env", Env)
+        self.add_constructor("!Enumerate", Enumerate)
+        self.add_constructor("!Value", Value)
+        self.add_constructor("!Index", Index)
 
 
 class EntryInvalidError(SentryIgnoredException):

website/developer-docs/blueprints/v1/tags.md

@@ -105,3 +105,130 @@ Requires at least one argument after the mode selection.
 If only a single argument is provided, its boolean representation will be returned for all normal modes and its negated boolean representation will be returned for all negated modes.
 
 Normally, it should be used to define complex conditions for use with an `!If` tag or for the `conditions` attribute of a blueprint entry (see [the blueprint file structure](./structure.md)). However, this is essentially just a boolean evaluator so it can be used everywhere a boolean representation is required.
+
+#### `!Enumerate`, `!Index` and `!Value`
+
+These tags collectively make it possible to iterate over objects which support iteration. Any iterable Python object is supported. Such objects are sequences (`[]`), mappings (`{}`) and even strings.
+
+1. `!Enumerate` tag:
+
+This tag takes 3 arguments:
+
+```
+!Enumerate [<iterable>, <output_object_type>, <single_item_yaml>]
+```
+
+-   **iterable**: Any Python iterable or custom tag that resolves to such iterable
+-   **output_object_type**: `SEQ` or `MAP`. Controls whether the returned YAML will be a mapping or a sequence.
+-   **single_item_yaml**: The YAML to use to create a single entry in the output object
+
+2. `!Index` tag:
+
+:::info
+This tag is only valid inside an `!Enumerate` tag
+:::
+
+This tag takes 1 argument:
+
+```
+!Index <depth>
+```
+
+-   **depth**: Must be >= 0. A depth of 0 refers to the `!Enumerate` tag this tag is located in. A depth of 1 refers to one `!Enumerate` tag above that (to be used when multiple `!Enumerate` tags are nested inside each other).
+
+Accesses the `!Enumerate` tag's iterable and resolves to the index of the item currently being iterated (in case `!Enumerate` is iterating over a sequence), or the mapping key (in case `!Enumerate` is iterating over a mapping).
+
+For example, given a sequence like this - `["a", "b", "c"]`, this tag will resolve to `0` on the first `!Enumerate` tag iteration, `1` on the second and so on. However, if given a mapping like this - `{"key1": "value1", "key2": "value2", "key3": "value3"}`, it will first resolve to `key1`, then to `key2` and so on.
+
+3. `!Value` tag:
+
+:::info
+This tag is only valid inside an `!Enumerate` tag
+:::
+
+This tag takes 1 argument:
+
+```
+!Value <depth>
+```
+
+-   **depth**: Must be >= 0. A depth of 0 refers to the `!Enumerate` tag this tag is located in. A depth of 1 refers to one `!Enumerate` tag above that (to be used when multiple `!Enumerate` tags are nested inside each other).
+
+Accesses the `!Enumerate` tag's iterable and resolves to the value of the item currently being iterated.
+
+For example, given a sequence like this - `["a", "b", "c"]`, this tag will resolve to `a` on the first `!Enumerate` tag iteration, `b` on the second and so on. If given a mapping like this - `{"key1": "value1", "key2": "value2", "key3": "value3"}`, it will first resolve to `value1`, then to `value2` and so on.
+
+Minimal examples:
+
+```
+configuration_stages: !Enumerate [
+    !Context map_of_totp_stage_names_and_types,
+    SEQ, # Output a sequence
+    !Find [!Format ["authentik_stages_authenticator_%s.authenticator%sstage", !Index 0, !Index 0], [name, !Value 0]] # The value of each item in the sequence
+]
+```
+
+The above example will resolve to something like this:
+
+```
+configuration_stages:
+- !Find [authentik_stages_authenticator_<stage_type_1>.authenticator<stage_type_1>stage, [name, <stage_name_1>]]
+- !Find [authentik_stages_authenticator_<stage_type_2>.authenticator<stage_type_2>stage, [name, <stage_name_2>]]
+```
+
+Similarly, a mapping can be generated like so:
+
+```
+example: !Enumerate [
+    !Context list_of_totp_stage_names,
+    MAP, # Output a map
+    [
+        !Index 0, # The key to assign to each entry
+        !Value 0, # The value to assign to each entry
+    ]
+]
+```
+
+The above example will resolve to something like this:
+
+```
+example:
+  0: <stage_name_1>
+  1: <stage_name_2>
+```
+
+Full example:
+
+:::warning
+Note that an `!Enumeration` tag's iterable can never be an `!Item` or `!Value` tag with a depth of `0`. Minimum depth allowed is `1`. This is because a depth of `0` refers to the `!Enumeration` tag the `!Item` or `!Value` tag is in, and an `!Enumeration` tag cannot iterate over itself.
+:::
+
+```
+example: !Enumerate [
+    !Context sequence, # ["foo", "bar"]
+    MAP, # Output a map
+    [
+        !Index 0, # Use the indexes of the items in the sequence as keys
+        !Enumerate [ # Nested enumeration
+            # Iterate over each item of the parent enumerate tag.
+            # Notice depth is 1, not 0, since we are inside the nested enumeration tag!
+            !Value 1,
+            SEQ, # Output a sequence
+            !Format ["%s: (index: %d, letter: %s)", !Value 1, !Index 0, !Value 0]
+        ]
+    ]
+]
+```
+
+The above example will resolve to something like this:
+
+```
+'0':
+- 'foo: (index: 0, letter: f)'
+- 'foo: (index: 1, letter: o)'
+- 'foo: (index: 2, letter: o)'
+'1':
+- 'bar: (index: 0, letter: b)'
+- 'bar: (index: 1, letter: a)'
+- 'bar: (index: 2, letter: r)'
+```