blueprints: Add `!Enumerate`, `!Value` and `!Index` tags (#4338)
* Added For and Item tags * Removed Sequence node support from ForItem tag * Added ForItemIndex tag * Added support for iterating over mappings * Added support for mapping output body * Renamed tags: For to Enumerate, ForItem to Value, ForItemIndex to Index * Refactored tests * Formatting * Improved exception info * Improved error handing * Added docs * lint * Small doc improvements * Replaced deepcopy() call with call to copy() * Fix mistake in docs example * Fix missed "!" in example
This commit is contained in:
sdimovv
GitHub
parent
2604dc14fe
commit
53cab07a48
|
|
@ -3,6 +3,12 @@ context:
|
|||
foo: bar
|
||||
policy_property: name
|
||||
policy_property_value: foo-bar-baz-qux
|
||||
sequence:
|
||||
- foo
|
||||
- bar
|
||||
mapping:
|
||||
key1: value
|
||||
key2: 2
|
||||
entries:
|
||||
- model: !Format ["%s", authentik_sources_oauth.oauthsource]
|
||||
state: !Format ["%s", present]
|
||||
|
|
@ -92,6 +98,49 @@ entries:
|
|||
]
|
||||
if_true_simple: !If [!Context foo, true, text]
|
||||
if_false_simple: !If [null, false, 2]
|
||||
enumerate_mapping_to_mapping: !Enumerate [
|
||||
!Context mapping,
|
||||
MAP,
|
||||
[!Format ["prefix-%s", !Index 0], !Format ["other-prefix-%s", !Value 0]]
|
||||
]
|
||||
enumerate_mapping_to_sequence: !Enumerate [
|
||||
!Context mapping,
|
||||
SEQ,
|
||||
!Format ["prefixed-pair-%s-%s", !Index 0, !Value 0]
|
||||
]
|
||||
enumerate_sequence_to_sequence: !Enumerate [
|
||||
!Context sequence,
|
||||
SEQ,
|
||||
!Format ["prefixed-items-%s-%s", !Index 0, !Value 0]
|
||||
]
|
||||
enumerate_sequence_to_mapping: !Enumerate [
|
||||
!Context sequence,
|
||||
MAP,
|
||||
[!Format ["index: %d", !Index 0], !Value 0]
|
||||
]
|
||||
nested_complex_enumeration: !Enumerate [
|
||||
!Context sequence,
|
||||
MAP,
|
||||
[
|
||||
!Index 0,
|
||||
!Enumerate [
|
||||
!Context mapping,
|
||||
MAP,
|
||||
[
|
||||
!Format ["%s", !Index 0],
|
||||
[
|
||||
!Enumerate [!Value 2, SEQ, !Format ["prefixed-%s", !Value 0]],
|
||||
{
|
||||
outer_value: !Value 1,
|
||||
outer_index: !Index 1,
|
||||
middle_value: !Value 0,
|
||||
middle_index: !Index 0
|
||||
}
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
]
|
||||
identifiers:
|
||||
name: test
|
||||
conditions:
|
||||
|
|
|
|||
|
|
@ -162,6 +162,61 @@ class TestBlueprintsV1(TransactionTestCase):
|
|||
"if_false_complex": ["list", "with", "items", "foo-bar"],
|
||||
"if_true_simple": True,
|
||||
"if_false_simple": 2,
|
||||
"enumerate_mapping_to_mapping": {
|
||||
"prefix-key1": "other-prefix-value",
|
||||
"prefix-key2": "other-prefix-2",
|
||||
},
|
||||
"enumerate_mapping_to_sequence": [
|
||||
"prefixed-pair-key1-value",
|
||||
"prefixed-pair-key2-2",
|
||||
],
|
||||
"enumerate_sequence_to_sequence": [
|
||||
"prefixed-items-0-foo",
|
||||
"prefixed-items-1-bar",
|
||||
],
|
||||
"enumerate_sequence_to_mapping": {"index: 0": "foo", "index: 1": "bar"},
|
||||
"nested_complex_enumeration": {
|
||||
"0": {
|
||||
"key1": [
|
||||
["prefixed-f", "prefixed-o", "prefixed-o"],
|
||||
{
|
||||
"outer_value": "foo",
|
||||
"outer_index": 0,
|
||||
"middle_value": "value",
|
||||
"middle_index": "key1",
|
||||
},
|
||||
],
|
||||
"key2": [
|
||||
["prefixed-f", "prefixed-o", "prefixed-o"],
|
||||
{
|
||||
"outer_value": "foo",
|
||||
"outer_index": 0,
|
||||
"middle_value": 2,
|
||||
"middle_index": "key2",
|
||||
},
|
||||
],
|
||||
},
|
||||
"1": {
|
||||
"key1": [
|
||||
["prefixed-b", "prefixed-a", "prefixed-r"],
|
||||
{
|
||||
"outer_value": "bar",
|
||||
"outer_index": 1,
|
||||
"middle_value": "value",
|
||||
"middle_index": "key1",
|
||||
},
|
||||
],
|
||||
"key2": [
|
||||
["prefixed-b", "prefixed-a", "prefixed-r"],
|
||||
{
|
||||
"outer_value": "bar",
|
||||
"outer_index": 1,
|
||||
"middle_value": 2,
|
||||
"middle_index": "key2",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
"""transfer common classes"""
|
||||
from collections import OrderedDict
|
||||
from copy import copy
|
||||
from dataclasses import asdict, dataclass, field, is_dataclass
|
||||
from enum import Enum
|
||||
from functools import reduce
|
||||
from operator import ixor
|
||||
from os import getenv
|
||||
from typing import Any, Literal, Optional, Union
|
||||
from typing import Any, Iterable, Literal, Mapping, Optional, Union
|
||||
from uuid import UUID
|
||||
|
||||
from deepmerge import always_merger
|
||||
from django.apps import apps
|
||||
from django.db.models import Model, Q
|
||||
from rest_framework.fields import Field
|
||||
|
|
@ -69,6 +71,9 @@ class BlueprintEntry:
|
|||
|
||||
_state: BlueprintEntryState = field(default_factory=BlueprintEntryState)
|
||||
|
||||
def __post_init__(self, *args, **kwargs) -> None:
|
||||
self.__tag_contexts: list["YAMLTagContext"] = []
|
||||
|
||||
@staticmethod
|
||||
def from_model(model: SerializerModel, *extra_identifier_names: str) -> "BlueprintEntry":
|
||||
"""Convert a SerializerModel instance to a blueprint Entry"""
|
||||
|
|
@ -85,17 +90,46 @@ class BlueprintEntry:
|
|||
attrs=all_attrs,
|
||||
)
|
||||
|
||||
def _get_tag_context(
|
||||
self,
|
||||
depth: int = 0,
|
||||
context_tag_type: Optional[type["YAMLTagContext"] | tuple["YAMLTagContext", ...]] = None,
|
||||
) -> "YAMLTagContext":
|
||||
"""Get a YAMLTagContex object located at a certain depth in the tag tree"""
|
||||
if depth < 0:
|
||||
raise ValueError("depth must be a positive number or zero")
|
||||
|
||||
if context_tag_type:
|
||||
contexts = [x for x in self.__tag_contexts if isinstance(x, context_tag_type)]
|
||||
else:
|
||||
contexts = self.__tag_contexts
|
||||
|
||||
try:
|
||||
return contexts[-(depth + 1)]
|
||||
except IndexError:
|
||||
raise ValueError(f"invalid depth: {depth}. Max depth: {len(contexts) - 1}")
|
||||
|
||||
def tag_resolver(self, value: Any, blueprint: "Blueprint") -> Any:
|
||||
"""Check if we have any special tags that need handling"""
|
||||
val = copy(value)
|
||||
|
||||
if isinstance(value, YAMLTagContext):
|
||||
self.__tag_contexts.append(value)
|
||||
|
||||
if isinstance(value, YAMLTag):
|
||||
return value.resolve(self, blueprint)
|
||||
val = value.resolve(self, blueprint)
|
||||
|
||||
if isinstance(value, dict):
|
||||
for key, inner_value in value.items():
|
||||
value[key] = self.tag_resolver(inner_value, blueprint)
|
||||
val[key] = self.tag_resolver(inner_value, blueprint)
|
||||
if isinstance(value, list):
|
||||
for idx, inner_value in enumerate(value):
|
||||
value[idx] = self.tag_resolver(inner_value, blueprint)
|
||||
return value
|
||||
val[idx] = self.tag_resolver(inner_value, blueprint)
|
||||
|
||||
if isinstance(value, YAMLTagContext):
|
||||
self.__tag_contexts.pop()
|
||||
|
||||
return val
|
||||
|
||||
def get_attrs(self, blueprint: "Blueprint") -> dict[str, Any]:
|
||||
"""Get attributes of this entry, with all yaml tags resolved"""
|
||||
|
|
@ -145,6 +179,14 @@ class YAMLTag:
|
|||
raise NotImplementedError
|
||||
|
||||
|
||||
class YAMLTagContext:
|
||||
"""Base class for all YAML Tag Contexts"""
|
||||
|
||||
def get_context(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
|
||||
"""Implement yaml tag context logic"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class KeyOf(YAMLTag):
|
||||
"""Reference another object by their ID"""
|
||||
|
||||
|
|
@ -351,6 +393,136 @@ class If(YAMLTag):
|
|||
raise EntryInvalidError(exc)
|
||||
|
||||
|
||||
class Enumerate(YAMLTag, YAMLTagContext):
|
||||
"""Iterate over an iterable."""
|
||||
|
||||
iterable: YAMLTag | Iterable
|
||||
item_body: Any
|
||||
output_body: Literal["SEQ", "MAP"]
|
||||
|
||||
_OUTPUT_BODIES = {
|
||||
"SEQ": (list, lambda a, b: [*a, b]),
|
||||
"MAP": (
|
||||
dict,
|
||||
lambda a, b: always_merger.merge(
|
||||
a, {b[0]: b[1]} if isinstance(b, (tuple, list)) else b
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def __init__(self, loader: "BlueprintLoader", node: SequenceNode) -> None:
|
||||
super().__init__()
|
||||
self.iterable = loader.construct_object(node.value[0])
|
||||
self.output_body = node.value[1].value
|
||||
self.item_body = loader.construct_object(node.value[2])
|
||||
self.__current_context: tuple[Any, Any] = tuple()
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def get_context(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
|
||||
return self.__current_context
|
||||
|
||||
def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
|
||||
if isinstance(self.iterable, EnumeratedItem) and self.iterable.depth == 0:
|
||||
raise EntryInvalidError(
|
||||
f"{self.__class__.__name__} tag's iterable references this tag's context. "
|
||||
"This is a noop. Check you are setting depth bigger than 0."
|
||||
)
|
||||
|
||||
if isinstance(self.iterable, YAMLTag):
|
||||
iterable = self.iterable.resolve(entry, blueprint)
|
||||
else:
|
||||
iterable = self.iterable
|
||||
|
||||
if not isinstance(iterable, Iterable):
|
||||
raise EntryInvalidError(
|
||||
f"{self.__class__.__name__}'s iterable must be an iterable "
|
||||
"such as a sequence or a mapping"
|
||||
)
|
||||
|
||||
if isinstance(iterable, Mapping):
|
||||
iterable = tuple(iterable.items())
|
||||
else:
|
||||
iterable = tuple(enumerate(iterable))
|
||||
|
||||
try:
|
||||
output_class, add_fn = self._OUTPUT_BODIES[self.output_body.upper()]
|
||||
except KeyError as exc:
|
||||
raise EntryInvalidError(exc)
|
||||
|
||||
result = output_class()
|
||||
|
||||
self.__current_context = tuple()
|
||||
|
||||
try:
|
||||
for item in iterable:
|
||||
self.__current_context = item
|
||||
resolved_body = entry.tag_resolver(self.item_body, blueprint)
|
||||
result = add_fn(result, resolved_body)
|
||||
if not isinstance(result, output_class):
|
||||
raise EntryInvalidError(
|
||||
f"Invalid {self.__class__.__name__} item found: {resolved_body}"
|
||||
)
|
||||
finally:
|
||||
self.__current_context = tuple()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class EnumeratedItem(YAMLTag):
|
||||
"""Get the current item value and index provided by an Enumerate tag context"""
|
||||
|
||||
depth: int
|
||||
|
||||
_SUPPORTED_CONTEXT_TAGS = (Enumerate,)
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def __init__(self, loader: "BlueprintLoader", node: ScalarNode) -> None:
|
||||
super().__init__()
|
||||
self.depth = int(node.value)
|
||||
|
||||
def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
|
||||
try:
|
||||
context_tag: Enumerate = entry._get_tag_context(
|
||||
depth=self.depth,
|
||||
context_tag_type=EnumeratedItem._SUPPORTED_CONTEXT_TAGS,
|
||||
)
|
||||
except ValueError as exc:
|
||||
if self.depth == 0:
|
||||
raise EntryInvalidError(
|
||||
f"{self.__class__.__name__} tags are only usable "
|
||||
f"inside an {Enumerate.__name__} tag"
|
||||
)
|
||||
|
||||
raise EntryInvalidError(f"{self.__class__.__name__} tag: {exc}")
|
||||
|
||||
return context_tag.get_context(entry, blueprint)
|
||||
|
||||
|
||||
class Index(EnumeratedItem):
|
||||
"""Get the current item index provided by an Enumerate tag context"""
|
||||
|
||||
def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
|
||||
context = super().resolve(entry, blueprint)
|
||||
|
||||
try:
|
||||
return context[0]
|
||||
except IndexError: # pragma: no cover
|
||||
raise EntryInvalidError(f"Empty/invalid context: {context}")
|
||||
|
||||
|
||||
class Value(EnumeratedItem):
|
||||
"""Get the current item value provided by an Enumerate tag context"""
|
||||
|
||||
def resolve(self, entry: BlueprintEntry, blueprint: Blueprint) -> Any:
|
||||
context = super().resolve(entry, blueprint)
|
||||
|
||||
try:
|
||||
return context[1]
|
||||
except IndexError: # pragma: no cover
|
||||
raise EntryInvalidError(f"Empty/invalid context: {context}")
|
||||
|
||||
|
||||
class BlueprintDumper(SafeDumper):
|
||||
"""Dump dataclasses to yaml"""
|
||||
|
||||
|
|
@ -394,6 +566,9 @@ class BlueprintLoader(SafeLoader):
|
|||
self.add_constructor("!Condition", Condition)
|
||||
self.add_constructor("!If", If)
|
||||
self.add_constructor("!Env", Env)
|
||||
self.add_constructor("!Enumerate", Enumerate)
|
||||
self.add_constructor("!Value", Value)
|
||||
self.add_constructor("!Index", Index)
|
||||
|
||||
|
||||
class EntryInvalidError(SentryIgnoredException):
|
||||
|
|
|
|||
|
|
@ -105,3 +105,130 @@ Requires at least one argument after the mode selection.
|
|||
If only a single argument is provided, its boolean representation will be returned for all normal modes and its negated boolean representation will be returned for all negated modes.
|
||||
|
||||
Normally, it should be used to define complex conditions for use with an `!If` tag or for the `conditions` attribute of a blueprint entry (see [the blueprint file structure](./structure.md)). However, this is essentially just a boolean evaluator so it can be used everywhere a boolean representation is required.
|
||||
|
||||
#### `!Enumerate`, `!Index` and `!Value`
|
||||
|
||||
These tags collectively make it possible to iterate over objects which support iteration. Any iterable Python object is supported. Such objects are sequences (`[]`), mappings (`{}`) and even strings.
|
||||
|
||||
1. `!Enumerate` tag:
|
||||
|
||||
This tag takes 3 arguments:
|
||||
|
||||
```
|
||||
!Enumerate [<iterable>, <output_object_type>, <single_item_yaml>]
|
||||
```
|
||||
|
||||
- **iterable**: Any Python iterable or custom tag that resolves to such iterable
|
||||
- **output_object_type**: `SEQ` or `MAP`. Controls whether the returned YAML will be a mapping or a sequence.
|
||||
- **single_item_yaml**: The YAML to use to create a single entry in the output object
|
||||
|
||||
2. `!Index` tag:
|
||||
|
||||
:::info
|
||||
This tag is only valid inside an `!Enumerate` tag
|
||||
:::
|
||||
|
||||
This tag takes 1 argument:
|
||||
|
||||
```
|
||||
!Index <depth>
|
||||
```
|
||||
|
||||
- **depth**: Must be >= 0. A depth of 0 refers to the `!Enumerate` tag this tag is located in. A depth of 1 refers to one `!Enumerate` tag above that (to be used when multiple `!Enumerate` tags are nested inside each other).
|
||||
|
||||
Accesses the `!Enumerate` tag's iterable and resolves to the index of the item currently being iterated (in case `!Enumerate` is iterating over a sequence), or the mapping key (in case `!Enumerate` is iterating over a mapping).
|
||||
|
||||
For example, given a sequence like this - `["a", "b", "c"]`, this tag will resolve to `0` on the first `!Enumerate` tag iteration, `1` on the second and so on. However, if given a mapping like this - `{"key1": "value1", "key2": "value2", "key3": "value3"}`, it will first resolve to `key1`, then to `key2` and so on.
|
||||
|
||||
3. `!Value` tag:
|
||||
|
||||
:::info
|
||||
This tag is only valid inside an `!Enumerate` tag
|
||||
:::
|
||||
|
||||
This tag takes 1 argument:
|
||||
|
||||
```
|
||||
!Value <depth>
|
||||
```
|
||||
|
||||
- **depth**: Must be >= 0. A depth of 0 refers to the `!Enumerate` tag this tag is located in. A depth of 1 refers to one `!Enumerate` tag above that (to be used when multiple `!Enumerate` tags are nested inside each other).
|
||||
|
||||
Accesses the `!Enumerate` tag's iterable and resolves to the value of the item currently being iterated.
|
||||
|
||||
For example, given a sequence like this - `["a", "b", "c"]`, this tag will resolve to `a` on the first `!Enumerate` tag iteration, `b` on the second and so on. If given a mapping like this - `{"key1": "value1", "key2": "value2", "key3": "value3"}`, it will first resolve to `value1`, then to `value2` and so on.
|
||||
|
||||
Minimal examples:
|
||||
|
||||
```
|
||||
configuration_stages: !Enumerate [
|
||||
!Context map_of_totp_stage_names_and_types,
|
||||
SEQ, # Output a sequence
|
||||
!Find [!Format ["authentik_stages_authenticator_%s.authenticator%sstage", !Index 0, !Index 0], [name, !Value 0]] # The value of each item in the sequence
|
||||
]
|
||||
```
|
||||
|
||||
The above example will resolve to something like this:
|
||||
|
||||
```
|
||||
configuration_stages:
|
||||
- !Find [authentik_stages_authenticator_<stage_type_1>.authenticator<stage_type_1>stage, [name, <stage_name_1>]]
|
||||
- !Find [authentik_stages_authenticator_<stage_type_2>.authenticator<stage_type_2>stage, [name, <stage_name_2>]]
|
||||
```
|
||||
|
||||
Similarly, a mapping can be generated like so:
|
||||
|
||||
```
|
||||
example: !Enumerate [
|
||||
!Context list_of_totp_stage_names,
|
||||
MAP, # Output a map
|
||||
[
|
||||
!Index 0, # The key to assign to each entry
|
||||
!Value 0, # The value to assign to each entry
|
||||
]
|
||||
]
|
||||
```
|
||||
|
||||
The above example will resolve to something like this:
|
||||
|
||||
```
|
||||
example:
|
||||
0: <stage_name_1>
|
||||
1: <stage_name_2>
|
||||
```
|
||||
|
||||
Full example:
|
||||
|
||||
:::warning
|
||||
Note that an `!Enumeration` tag's iterable can never be an `!Item` or `!Value` tag with a depth of `0`. Minimum depth allowed is `1`. This is because a depth of `0` refers to the `!Enumeration` tag the `!Item` or `!Value` tag is in, and an `!Enumeration` tag cannot iterate over itself.
|
||||
:::
|
||||
|
||||
```
|
||||
example: !Enumerate [
|
||||
!Context sequence, # ["foo", "bar"]
|
||||
MAP, # Output a map
|
||||
[
|
||||
!Index 0, # Use the indexes of the items in the sequence as keys
|
||||
!Enumerate [ # Nested enumeration
|
||||
# Iterate over each item of the parent enumerate tag.
|
||||
# Notice depth is 1, not 0, since we are inside the nested enumeration tag!
|
||||
!Value 1,
|
||||
SEQ, # Output a sequence
|
||||
!Format ["%s: (index: %d, letter: %s)", !Value 1, !Index 0, !Value 0]
|
||||
]
|
||||
]
|
||||
]
|
||||
```
|
||||
|
||||
The above example will resolve to something like this:
|
||||
|
||||
```
|
||||
'0':
|
||||
- 'foo: (index: 0, letter: f)'
|
||||
- 'foo: (index: 1, letter: o)'
|
||||
- 'foo: (index: 2, letter: o)'
|
||||
'1':
|
||||
- 'bar: (index: 0, letter: b)'
|
||||
- 'bar: (index: 1, letter: a)'
|
||||
- 'bar: (index: 2, letter: r)'
|
||||
```
|
||||
|
|
|
|||
Reference in New Issue