blueprints: internal storage (#4397)

* rework oci client

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

* add blueprint content

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add UI

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* make path optional

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add validation

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add tests

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>
Signed-off-by: Jens Langhammer <jens@goauthentik.io>

authentik/blueprints/api.py

@@ -1,4 +1,5 @@
 """Serializer mixin for managed models"""
+from django.utils.translation import gettext_lazy as _
 from drf_spectacular.utils import extend_schema, inline_serializer
 from rest_framework.decorators import action
 from rest_framework.exceptions import ValidationError
@@ -11,6 +12,7 @@ from rest_framework.viewsets import ModelViewSet
 
 from authentik.api.decorators import permission_required
 from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed
+from authentik.blueprints.v1.importer import Importer
 from authentik.blueprints.v1.tasks import apply_blueprint, blueprints_find_dict
 from authentik.core.api.used_by import UsedByMixin
 from authentik.core.api.utils import PassiveSerializer
@@ -40,6 +42,21 @@ class BlueprintInstanceSerializer(ModelSerializer):
             raise ValidationError(exc) from exc
         return path
 
+    def validate_content(self, content: str) -> str:
+        """Ensure content (if set) is a valid blueprint"""
+        if content == "":
+            return content
+        context = self.instance.context if self.instance else {}
+        valid, logs = Importer(content, context).validate()
+        if not valid:
+            raise ValidationError(_("Failed to validate blueprint"), *[x["msg"] for x in logs])
+        return content
+
+    def validate(self, attrs: dict) -> dict:
+        if attrs.get("path", "") == "" and attrs.get("content", "") == "":
+            raise ValidationError(_("Either path or content must be set."))
+        return super().validate(attrs)
+
     class Meta:
 
         model = BlueprintInstance
@@ -54,6 +71,7 @@ class BlueprintInstanceSerializer(ModelSerializer):
             "enabled",
             "managed_models",
             "metadata",
+            "content",
         ]
         extra_kwargs = {
             "status": {"read_only": True},

authentik/blueprints/migrations/0002_blueprintinstance_content.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-01-10 19:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("authentik_blueprints", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="blueprintinstance",
+            name="content",
+            field=models.TextField(blank=True, default=""),
+        ),
+        migrations.AlterField(
+            model_name="blueprintinstance",
+            name="path",
+            field=models.TextField(blank=True, default=""),
+        ),
+    ]

authentik/blueprints/models.py

@@ -1,30 +1,18 @@
 """blueprint models"""
 from pathlib import Path
-from urllib.parse import urlparse
 from uuid import uuid4
 
 from django.contrib.postgres.fields import ArrayField
 from django.db import models
 from django.utils.translation import gettext_lazy as _
-from opencontainers.distribution.reggie import (
-    NewClient,
-    WithDebug,
-    WithDefaultName,
-    WithDigest,
-    WithReference,
-    WithUserAgent,
-    WithUsernamePassword,
-)
-from requests.exceptions import RequestException
 from rest_framework.serializers import Serializer
 from structlog import get_logger
 
+from authentik.blueprints.v1.oci import BlueprintOCIClient, OCIException
 from authentik.lib.config import CONFIG
 from authentik.lib.models import CreatedUpdatedModel, SerializerModel
 from authentik.lib.sentry import SentryIgnoredException
-from authentik.lib.utils.http import authentik_user_agent
 
-OCI_MEDIA_TYPE = "application/vnd.goauthentik.blueprint.v1+yaml"
 LOGGER = get_logger()
 
 
@@ -74,7 +62,8 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel):
 
     name = models.TextField()
     metadata = models.JSONField(default=dict)
-    path = models.TextField()
+    path = models.TextField(default="", blank=True)
+    content = models.TextField(default="", blank=True)
     context = models.JSONField(default=dict)
     last_applied = models.DateTimeField(auto_now=True)
     last_applied_hash = models.TextField()
@@ -86,60 +75,29 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel):
 
     def retrieve_oci(self) -> str:
         """Get blueprint from an OCI registry"""
-        url = urlparse(self.path)
+        client = BlueprintOCIClient(self.path.replace("oci://", "https://"))
-        ref = "latest"
-        path = url.path[1:]
-        if ":" in url.path:
-            path, _, ref = path.partition(":")
-        client = NewClient(
-            f"https://{url.hostname}",
-            WithUserAgent(authentik_user_agent()),
-            WithUsernamePassword(url.username, url.password),
-            WithDefaultName(path),
-            WithDebug(True),
-        )
-        LOGGER.info("Fetching OCI manifests for blueprint", instance=self)
-        manifest_request = client.NewRequest(
-            "GET",
-            "/v2/<name>/manifests/<reference>",
-            WithReference(ref),
-        ).SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
         try:
-            manifest_response = client.Do(manifest_request)
+            manifests = client.fetch_manifests()
-            manifest_response.raise_for_status()
+            return client.fetch_blobs(manifests)
-        except RequestException as exc:
+        except OCIException as exc:
             raise BlueprintRetrievalFailed(exc) from exc
-        manifest = manifest_response.json()
-        if "errors" in manifest:
-            raise BlueprintRetrievalFailed(manifest["errors"])
 
-        blob = None
+    def retrieve_file(self) -> str:
-        for layer in manifest.get("layers", []):
+        """Get blueprint from path"""
-            if layer.get("mediaType", "") == OCI_MEDIA_TYPE:
-                blob = layer.get("digest")
-                LOGGER.debug("Found layer with matching media type", instance=self, blob=blob)
-        if not blob:
-            raise BlueprintRetrievalFailed("Blob not found")
-
-        blob_request = client.NewRequest(
-            "GET",
-            "/v2/<name>/blobs/<digest>",
-            WithDigest(blob),
-        )
         try:
-            blob_response = client.Do(blob_request)
+            full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path))
-            blob_response.raise_for_status()
+            with full_path.open("r", encoding="utf-8") as _file:
-            return blob_response.text
+                return _file.read()
-        except RequestException as exc:
+        except (IOError, OSError) as exc:
             raise BlueprintRetrievalFailed(exc) from exc
 
     def retrieve(self) -> str:
         """Retrieve blueprint contents"""
         if self.path.startswith("oci://"):
             return self.retrieve_oci()
-        full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path))
+        if self.path != "":
-        with full_path.open("r", encoding="utf-8") as _file:
+            return self.retrieve_file()
-            return _file.read()
+        return self.content
 
     @property
     def serializer(self) -> Serializer:

authentik/blueprints/tests/test_oci.py

@@ -2,7 +2,8 @@
 from django.test import TransactionTestCase
 from requests_mock import Mocker
 
-from authentik.blueprints.models import OCI_MEDIA_TYPE, BlueprintInstance, BlueprintRetrievalFailed
+from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed
+from authentik.blueprints.v1.oci import OCI_MEDIA_TYPE
 
 
 class TestBlueprintOCI(TransactionTestCase):

authentik/blueprints/tests/test_v1_api.py

@@ -43,3 +43,28 @@ class TestBlueprintsV1API(APITestCase):
                     "6871c0003f5c07be5c3316d9d4a08444bd8fed1b3f03294e51e44522"
                 ),
             )
+
+    def test_api_blank(self):
+        """Test blank"""
+        res = self.client.post(
+            reverse("authentik_api:blueprintinstance-list"),
+            data={
+                "name": "foo",
+            },
+        )
+        self.assertEqual(res.status_code, 400)
+        self.assertJSONEqual(
+            res.content.decode(), {"non_field_errors": ["Either path or content must be set."]}
+        )
+
+    def test_api_content(self):
+        """Test blank"""
+        res = self.client.post(
+            reverse("authentik_api:blueprintinstance-list"),
+            data={
+                "name": "foo",
+                "content": '{"version": 3}',
+            },
+        )
+        self.assertEqual(res.status_code, 400)
+        self.assertJSONEqual(res.content.decode(), {"content": ["Failed to validate blueprint"]})

authentik/blueprints/v1/oci.py

@@ -0,0 +1,98 @@
+"""OCI Client"""
+from typing import Any
+from urllib.parse import ParseResult, urlparse
+
+from opencontainers.distribution.reggie import (
+    NewClient,
+    WithDebug,
+    WithDefaultName,
+    WithDigest,
+    WithReference,
+    WithUserAgent,
+    WithUsernamePassword,
+)
+from requests.exceptions import RequestException
+from structlog import get_logger
+from structlog.stdlib import BoundLogger
+
+from authentik.lib.sentry import SentryIgnoredException
+from authentik.lib.utils.http import authentik_user_agent
+
+OCI_MEDIA_TYPE = "application/vnd.goauthentik.blueprint.v1+yaml"
+
+
+class OCIException(SentryIgnoredException):
+    """OCI-related errors"""
+
+
+class BlueprintOCIClient:
+    """Blueprint OCI Client"""
+
+    url: ParseResult
+    sanitized_url: str
+    logger: BoundLogger
+    ref: str
+    client: NewClient
+
+    def __init__(self, url: str) -> None:
+        self._parse_url(url)
+        self.logger = get_logger().bind(url=self.sanitized_url)
+
+        self.ref = "latest"
+        path = self.url.path[1:]
+        if ":" in self.url.path:
+            path, _, self.ref = path.partition(":")
+        self.client = NewClient(
+            f"https://{self.url.hostname}",
+            WithUserAgent(authentik_user_agent()),
+            WithUsernamePassword(self.url.username, self.url.password),
+            WithDefaultName(path),
+            WithDebug(True),
+        )
+
+    def _parse_url(self, url: str):
+        self.url = urlparse(url)
+        netloc = self.url.netloc
+        if "@" in netloc:
+            netloc = netloc[netloc.index("@") + 1 :]
+        self.sanitized_url = self.url._replace(netloc=netloc).geturl()
+
+    def fetch_manifests(self) -> dict[str, Any]:
+        """Fetch manifests for ref"""
+        self.logger.info("Fetching OCI manifests for blueprint")
+        manifest_request = self.client.NewRequest(
+            "GET",
+            "/v2/<name>/manifests/<reference>",
+            WithReference(self.ref),
+        ).SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
+        try:
+            manifest_response = self.client.Do(manifest_request)
+            manifest_response.raise_for_status()
+        except RequestException as exc:
+            raise OCIException(exc) from exc
+        manifest = manifest_response.json()
+        if "errors" in manifest:
+            raise OCIException(manifest["errors"])
+        return manifest
+
+    def fetch_blobs(self, manifest: dict[str, Any]):
+        """Fetch blob based on manifest info"""
+        blob = None
+        for layer in manifest.get("layers", []):
+            if layer.get("mediaType", "") == OCI_MEDIA_TYPE:
+                blob = layer.get("digest")
+                self.logger.debug("Found layer with matching media type", blob=blob)
+        if not blob:
+            raise OCIException("Blob not found")
+
+        blob_request = self.client.NewRequest(
+            "GET",
+            "/v2/<name>/blobs/<digest>",
+            WithDigest(blob),
+        )
+        try:
+            blob_response = self.client.Do(blob_request)
+            blob_response.raise_for_status()
+            return blob_response.text
+        except RequestException as exc:
+            raise OCIException(exc) from exc

schema.yml

@@ -25918,6 +25918,7 @@ components:
           type: string
         path:
           type: string
+          default: ''
         context:
           type: object
           additionalProperties: {}
@@ -25943,13 +25944,14 @@ components:
           type: object
           additionalProperties: {}
           readOnly: true
+        content:
+          type: string
       required:
       - last_applied
       - last_applied_hash
       - managed_models
       - metadata
       - name
-      - path
       - pk
       - status
     BlueprintInstanceRequest:
@@ -25961,15 +25963,16 @@ components:
           minLength: 1
         path:
           type: string
-          minLength: 1
+          default: ''
         context:
           type: object
           additionalProperties: {}
         enabled:
           type: boolean
+        content:
+          type: string
       required:
       - name
-      - path
     BlueprintInstanceStatusEnum:
       enum:
       - successful
@@ -33147,12 +33150,14 @@ components:
           minLength: 1
         path:
           type: string
-          minLength: 1
+          default: ''
         context:
           type: object
           additionalProperties: {}
         enabled:
           type: boolean
+        content:
+          type: string
     PatchedCaptchaStageRequest:
       type: object
       description: CaptchaStage Serializer

web/src/admin/blueprints/BlueprintForm.ts

@@ -20,26 +20,27 @@ import PFToggleGroup from "@patternfly/patternfly/components/ToggleGroup/toggle-
 import { BlueprintFile, BlueprintInstance, ManagedApi } from "@goauthentik/api";
 
 enum blueprintSource {
-    local,
+    file,
     oci,
+    internal,
 }
 
 @customElement("ak-blueprint-form")
 export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
     @state()
-    source: blueprintSource = blueprintSource.local;
+    source: blueprintSource = blueprintSource.file;
 
-    loadInstance(pk: string): Promise<BlueprintInstance> {
+    async loadInstance(pk: string): Promise<BlueprintInstance> {
-        return new ManagedApi(DEFAULT_CONFIG)
+        const inst = await new ManagedApi(DEFAULT_CONFIG).managedBlueprintsRetrieve({
-            .managedBlueprintsRetrieve({
             instanceUuid: pk,
-            })
+        });
-            .then((inst) => {
+        if (inst.path?.startsWith("oci://")) {
-                if (inst.path.startsWith("oci://")) {
             this.source = blueprintSource.oci;
         }
+        if (inst.content !== "") {
+            this.source = blueprintSource.internal;
+        }
         return inst;
-            });
     }
 
     getSuccessMessage(): string {
@@ -102,12 +103,12 @@ export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
                         <div class="pf-c-toggle-group__item">
                             <button
                                 class="pf-c-toggle-group__button ${this.source ===
-                                blueprintSource.local
+                                blueprintSource.file
                                     ? "pf-m-selected"
                                     : ""}"
                                 type="button"
                                 @click=${() => {
-                                    this.source = blueprintSource.local;
+                                    this.source = blueprintSource.file;
                                 }}
                             >
                                 <span class="pf-c-toggle-group__text">${t`Local path`}</span>
@@ -128,10 +129,25 @@ export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
                                 <span class="pf-c-toggle-group__text">${t`OCI Registry`}</span>
                             </button>
                         </div>
+                        <div class="pf-c-divider pf-m-vertical" role="separator"></div>
+                        <div class="pf-c-toggle-group__item">
+                            <button
+                                class="pf-c-toggle-group__button ${this.source ===
+                                blueprintSource.internal
+                                    ? "pf-m-selected"
+                                    : ""}"
+                                type="button"
+                                @click=${() => {
+                                    this.source = blueprintSource.internal;
+                                }}
+                            >
+                                <span class="pf-c-toggle-group__text">${t`Internal`}</span>
+                            </button>
+                        </div>
                     </div>
                 </div>
                 <div class="pf-c-card__footer">
-                    ${this.source === blueprintSource.local
+                    ${this.source === blueprintSource.file
                         ? html`<ak-form-element-horizontal label=${t`Path`} name="path">
                               <ak-search-select
                                   .fetchObjects=${async (
@@ -187,6 +203,15 @@ export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
                               </p>
                           </ak-form-element-horizontal>`
                         : html``}
+                    ${this.source === blueprintSource.internal
+                        ? html`<ak-form-element-horizontal label=${t`Blueprint`} name="content">
+                              <ak-codemirror
+                                  mode="yaml"
+                                  .parseValue=${false}
+                                  value="${ifDefined(this.instance?.content)}"
+                              ></ak-codemirror>
+                          </ak-form-element-horizontal>`
+                        : html``}
                 </div>
             </div>
 

web/src/elements/CodeMirror.ts

@@ -28,6 +28,9 @@ export class CodeMirrorTextarea<T> extends AKElement {
     @property()
     name?: string;
 
+    @property({ type: Boolean })
+    parseValue = true;
+
     editor?: EditorView;
 
     _value?: string;
@@ -67,6 +70,9 @@ export class CodeMirrorTextarea<T> extends AKElement {
     }
 
     get value(): T | string {
+        if (!this.parseValue) {
+            return this.getInnerValue();
+        }
         try {
             switch (this.mode.toLowerCase()) {
                 case "yaml":

website/developer-docs/blueprints/index.md

@@ -22,7 +22,7 @@ Blueprints are yaml files, whose format is described further in [File structure]
 
 Starting with authentik 2022.8, blueprints are used to manage authentik default flows and other system objects. These blueprints can be disabled/replaced with custom blueprints in certain circumstances.
 
-## Storage - Local
+## Storage - File
 
 The authentik container by default looks for blueprints in `/blueprints`. Underneath this directory, there are a couple default subdirectories:
 
@@ -49,3 +49,13 @@ To push a blueprint to an OCI-compatible registry, [ORAS](https://oras.land/) ca
 ```
 oras push ghcr.io/<username>/blueprint/<blueprint name>:latest <yaml file>:application/vnd.goauthentik.blueprint.v1+yaml
 ```
+
+## Storage - Internal
+
+:::info
+Requires authentik 2023.1
+:::
+
+Blueprints can be stored in authentik's database, which allows blueprints to be managed via external configuration management tools like Terraform.
+
+Modifying the contents of a blueprint will trigger its reconciliation. Blueprints are validated on submission to prevent invalid blueprints from being saved.