blueprints: internal storage (#4397)

* rework oci client

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>

* add blueprint content

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add UI

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* make path optional

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add validation

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* add tests

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

* fix

Signed-off-by: Jens Langhammer <jens@goauthentik.io>

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>
Signed-off-by: Jens Langhammer <jens@goauthentik.io>
This commit is contained in:
Jens L 2023-01-10 22:00:34 +01:00 committed by GitHub
parent f2961cb536
commit 1ed24a5eef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 249 additions and 80 deletions

View File

@ -1,4 +1,5 @@
"""Serializer mixin for managed models"""
from django.utils.translation import gettext_lazy as _
from drf_spectacular.utils import extend_schema, inline_serializer
from rest_framework.decorators import action
from rest_framework.exceptions import ValidationError
@ -11,6 +12,7 @@ from rest_framework.viewsets import ModelViewSet
from authentik.api.decorators import permission_required
from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed
from authentik.blueprints.v1.importer import Importer
from authentik.blueprints.v1.tasks import apply_blueprint, blueprints_find_dict
from authentik.core.api.used_by import UsedByMixin
from authentik.core.api.utils import PassiveSerializer
@ -40,6 +42,21 @@ class BlueprintInstanceSerializer(ModelSerializer):
raise ValidationError(exc) from exc
return path
def validate_content(self, content: str) -> str:
"""Ensure content (if set) is a valid blueprint"""
if content == "":
return content
context = self.instance.context if self.instance else {}
valid, logs = Importer(content, context).validate()
if not valid:
raise ValidationError(_("Failed to validate blueprint"), *[x["msg"] for x in logs])
return content
def validate(self, attrs: dict) -> dict:
if attrs.get("path", "") == "" and attrs.get("content", "") == "":
raise ValidationError(_("Either path or content must be set."))
return super().validate(attrs)
class Meta:
model = BlueprintInstance
@ -54,6 +71,7 @@ class BlueprintInstanceSerializer(ModelSerializer):
"enabled",
"managed_models",
"metadata",
"content",
]
extra_kwargs = {
"status": {"read_only": True},

View File

@ -0,0 +1,23 @@
# Generated by Django 4.1.5 on 2023-01-10 19:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("authentik_blueprints", "0001_initial"),
]
operations = [
migrations.AddField(
model_name="blueprintinstance",
name="content",
field=models.TextField(blank=True, default=""),
),
migrations.AlterField(
model_name="blueprintinstance",
name="path",
field=models.TextField(blank=True, default=""),
),
]

View File

@ -1,30 +1,18 @@
"""blueprint models"""
from pathlib import Path
from urllib.parse import urlparse
from uuid import uuid4
from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.utils.translation import gettext_lazy as _
from opencontainers.distribution.reggie import (
NewClient,
WithDebug,
WithDefaultName,
WithDigest,
WithReference,
WithUserAgent,
WithUsernamePassword,
)
from requests.exceptions import RequestException
from rest_framework.serializers import Serializer
from structlog import get_logger
from authentik.blueprints.v1.oci import BlueprintOCIClient, OCIException
from authentik.lib.config import CONFIG
from authentik.lib.models import CreatedUpdatedModel, SerializerModel
from authentik.lib.sentry import SentryIgnoredException
from authentik.lib.utils.http import authentik_user_agent
OCI_MEDIA_TYPE = "application/vnd.goauthentik.blueprint.v1+yaml"
LOGGER = get_logger()
@ -74,7 +62,8 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel):
name = models.TextField()
metadata = models.JSONField(default=dict)
path = models.TextField()
path = models.TextField(default="", blank=True)
content = models.TextField(default="", blank=True)
context = models.JSONField(default=dict)
last_applied = models.DateTimeField(auto_now=True)
last_applied_hash = models.TextField()
@ -86,60 +75,29 @@ class BlueprintInstance(SerializerModel, ManagedModel, CreatedUpdatedModel):
def retrieve_oci(self) -> str:
"""Get blueprint from an OCI registry"""
url = urlparse(self.path)
ref = "latest"
path = url.path[1:]
if ":" in url.path:
path, _, ref = path.partition(":")
client = NewClient(
f"https://{url.hostname}",
WithUserAgent(authentik_user_agent()),
WithUsernamePassword(url.username, url.password),
WithDefaultName(path),
WithDebug(True),
)
LOGGER.info("Fetching OCI manifests for blueprint", instance=self)
manifest_request = client.NewRequest(
"GET",
"/v2/<name>/manifests/<reference>",
WithReference(ref),
).SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
client = BlueprintOCIClient(self.path.replace("oci://", "https://"))
try:
manifest_response = client.Do(manifest_request)
manifest_response.raise_for_status()
except RequestException as exc:
manifests = client.fetch_manifests()
return client.fetch_blobs(manifests)
except OCIException as exc:
raise BlueprintRetrievalFailed(exc) from exc
manifest = manifest_response.json()
if "errors" in manifest:
raise BlueprintRetrievalFailed(manifest["errors"])
blob = None
for layer in manifest.get("layers", []):
if layer.get("mediaType", "") == OCI_MEDIA_TYPE:
blob = layer.get("digest")
LOGGER.debug("Found layer with matching media type", instance=self, blob=blob)
if not blob:
raise BlueprintRetrievalFailed("Blob not found")
blob_request = client.NewRequest(
"GET",
"/v2/<name>/blobs/<digest>",
WithDigest(blob),
)
def retrieve_file(self) -> str:
"""Get blueprint from path"""
try:
blob_response = client.Do(blob_request)
blob_response.raise_for_status()
return blob_response.text
except RequestException as exc:
full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path))
with full_path.open("r", encoding="utf-8") as _file:
return _file.read()
except (IOError, OSError) as exc:
raise BlueprintRetrievalFailed(exc) from exc
def retrieve(self) -> str:
"""Retrieve blueprint contents"""
if self.path.startswith("oci://"):
return self.retrieve_oci()
full_path = Path(CONFIG.y("blueprints_dir")).joinpath(Path(self.path))
with full_path.open("r", encoding="utf-8") as _file:
return _file.read()
if self.path != "":
return self.retrieve_file()
return self.content
@property
def serializer(self) -> Serializer:

View File

@ -2,7 +2,8 @@
from django.test import TransactionTestCase
from requests_mock import Mocker
from authentik.blueprints.models import OCI_MEDIA_TYPE, BlueprintInstance, BlueprintRetrievalFailed
from authentik.blueprints.models import BlueprintInstance, BlueprintRetrievalFailed
from authentik.blueprints.v1.oci import OCI_MEDIA_TYPE
class TestBlueprintOCI(TransactionTestCase):

View File

@ -43,3 +43,28 @@ class TestBlueprintsV1API(APITestCase):
"6871c0003f5c07be5c3316d9d4a08444bd8fed1b3f03294e51e44522"
),
)
def test_api_blank(self):
"""Test blank"""
res = self.client.post(
reverse("authentik_api:blueprintinstance-list"),
data={
"name": "foo",
},
)
self.assertEqual(res.status_code, 400)
self.assertJSONEqual(
res.content.decode(), {"non_field_errors": ["Either path or content must be set."]}
)
def test_api_content(self):
"""Test blank"""
res = self.client.post(
reverse("authentik_api:blueprintinstance-list"),
data={
"name": "foo",
"content": '{"version": 3}',
},
)
self.assertEqual(res.status_code, 400)
self.assertJSONEqual(res.content.decode(), {"content": ["Failed to validate blueprint"]})

View File

@ -0,0 +1,98 @@
"""OCI Client"""
from typing import Any
from urllib.parse import ParseResult, urlparse
from opencontainers.distribution.reggie import (
NewClient,
WithDebug,
WithDefaultName,
WithDigest,
WithReference,
WithUserAgent,
WithUsernamePassword,
)
from requests.exceptions import RequestException
from structlog import get_logger
from structlog.stdlib import BoundLogger
from authentik.lib.sentry import SentryIgnoredException
from authentik.lib.utils.http import authentik_user_agent
OCI_MEDIA_TYPE = "application/vnd.goauthentik.blueprint.v1+yaml"
class OCIException(SentryIgnoredException):
"""OCI-related errors"""
class BlueprintOCIClient:
"""Blueprint OCI Client"""
url: ParseResult
sanitized_url: str
logger: BoundLogger
ref: str
client: NewClient
def __init__(self, url: str) -> None:
self._parse_url(url)
self.logger = get_logger().bind(url=self.sanitized_url)
self.ref = "latest"
path = self.url.path[1:]
if ":" in self.url.path:
path, _, self.ref = path.partition(":")
self.client = NewClient(
f"https://{self.url.hostname}",
WithUserAgent(authentik_user_agent()),
WithUsernamePassword(self.url.username, self.url.password),
WithDefaultName(path),
WithDebug(True),
)
def _parse_url(self, url: str):
self.url = urlparse(url)
netloc = self.url.netloc
if "@" in netloc:
netloc = netloc[netloc.index("@") + 1 :]
self.sanitized_url = self.url._replace(netloc=netloc).geturl()
def fetch_manifests(self) -> dict[str, Any]:
"""Fetch manifests for ref"""
self.logger.info("Fetching OCI manifests for blueprint")
manifest_request = self.client.NewRequest(
"GET",
"/v2/<name>/manifests/<reference>",
WithReference(self.ref),
).SetHeader("Accept", "application/vnd.oci.image.manifest.v1+json")
try:
manifest_response = self.client.Do(manifest_request)
manifest_response.raise_for_status()
except RequestException as exc:
raise OCIException(exc) from exc
manifest = manifest_response.json()
if "errors" in manifest:
raise OCIException(manifest["errors"])
return manifest
def fetch_blobs(self, manifest: dict[str, Any]):
"""Fetch blob based on manifest info"""
blob = None
for layer in manifest.get("layers", []):
if layer.get("mediaType", "") == OCI_MEDIA_TYPE:
blob = layer.get("digest")
self.logger.debug("Found layer with matching media type", blob=blob)
if not blob:
raise OCIException("Blob not found")
blob_request = self.client.NewRequest(
"GET",
"/v2/<name>/blobs/<digest>",
WithDigest(blob),
)
try:
blob_response = self.client.Do(blob_request)
blob_response.raise_for_status()
return blob_response.text
except RequestException as exc:
raise OCIException(exc) from exc

View File

@ -25918,6 +25918,7 @@ components:
type: string
path:
type: string
default: ''
context:
type: object
additionalProperties: {}
@ -25943,13 +25944,14 @@ components:
type: object
additionalProperties: {}
readOnly: true
content:
type: string
required:
- last_applied
- last_applied_hash
- managed_models
- metadata
- name
- path
- pk
- status
BlueprintInstanceRequest:
@ -25961,15 +25963,16 @@ components:
minLength: 1
path:
type: string
minLength: 1
default: ''
context:
type: object
additionalProperties: {}
enabled:
type: boolean
content:
type: string
required:
- name
- path
BlueprintInstanceStatusEnum:
enum:
- successful
@ -33147,12 +33150,14 @@ components:
minLength: 1
path:
type: string
minLength: 1
default: ''
context:
type: object
additionalProperties: {}
enabled:
type: boolean
content:
type: string
PatchedCaptchaStageRequest:
type: object
description: CaptchaStage Serializer

View File

@ -20,26 +20,27 @@ import PFToggleGroup from "@patternfly/patternfly/components/ToggleGroup/toggle-
import { BlueprintFile, BlueprintInstance, ManagedApi } from "@goauthentik/api";
enum blueprintSource {
local,
file,
oci,
internal,
}
@customElement("ak-blueprint-form")
export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
@state()
source: blueprintSource = blueprintSource.local;
source: blueprintSource = blueprintSource.file;
loadInstance(pk: string): Promise<BlueprintInstance> {
return new ManagedApi(DEFAULT_CONFIG)
.managedBlueprintsRetrieve({
instanceUuid: pk,
})
.then((inst) => {
if (inst.path.startsWith("oci://")) {
this.source = blueprintSource.oci;
}
return inst;
});
async loadInstance(pk: string): Promise<BlueprintInstance> {
const inst = await new ManagedApi(DEFAULT_CONFIG).managedBlueprintsRetrieve({
instanceUuid: pk,
});
if (inst.path?.startsWith("oci://")) {
this.source = blueprintSource.oci;
}
if (inst.content !== "") {
this.source = blueprintSource.internal;
}
return inst;
}
getSuccessMessage(): string {
@ -102,12 +103,12 @@ export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
<div class="pf-c-toggle-group__item">
<button
class="pf-c-toggle-group__button ${this.source ===
blueprintSource.local
blueprintSource.file
? "pf-m-selected"
: ""}"
type="button"
@click=${() => {
this.source = blueprintSource.local;
this.source = blueprintSource.file;
}}
>
<span class="pf-c-toggle-group__text">${t`Local path`}</span>
@ -128,10 +129,25 @@ export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
<span class="pf-c-toggle-group__text">${t`OCI Registry`}</span>
</button>
</div>
<div class="pf-c-divider pf-m-vertical" role="separator"></div>
<div class="pf-c-toggle-group__item">
<button
class="pf-c-toggle-group__button ${this.source ===
blueprintSource.internal
? "pf-m-selected"
: ""}"
type="button"
@click=${() => {
this.source = blueprintSource.internal;
}}
>
<span class="pf-c-toggle-group__text">${t`Internal`}</span>
</button>
</div>
</div>
</div>
<div class="pf-c-card__footer">
${this.source === blueprintSource.local
${this.source === blueprintSource.file
? html`<ak-form-element-horizontal label=${t`Path`} name="path">
<ak-search-select
.fetchObjects=${async (
@ -187,6 +203,15 @@ export class BlueprintForm extends ModelForm<BlueprintInstance, string> {
</p>
</ak-form-element-horizontal>`
: html``}
${this.source === blueprintSource.internal
? html`<ak-form-element-horizontal label=${t`Blueprint`} name="content">
<ak-codemirror
mode="yaml"
.parseValue=${false}
value="${ifDefined(this.instance?.content)}"
></ak-codemirror>
</ak-form-element-horizontal>`
: html``}
</div>
</div>

View File

@ -28,6 +28,9 @@ export class CodeMirrorTextarea<T> extends AKElement {
@property()
name?: string;
@property({ type: Boolean })
parseValue = true;
editor?: EditorView;
_value?: string;
@ -67,6 +70,9 @@ export class CodeMirrorTextarea<T> extends AKElement {
}
get value(): T | string {
if (!this.parseValue) {
return this.getInnerValue();
}
try {
switch (this.mode.toLowerCase()) {
case "yaml":

View File

@ -22,7 +22,7 @@ Blueprints are yaml files, whose format is described further in [File structure]
Starting with authentik 2022.8, blueprints are used to manage authentik default flows and other system objects. These blueprints can be disabled/replaced with custom blueprints in certain circumstances.
## Storage - Local
## Storage - File
The authentik container by default looks for blueprints in `/blueprints`. Underneath this directory, there are a couple default subdirectories:
@ -49,3 +49,13 @@ To push a blueprint to an OCI-compatible registry, [ORAS](https://oras.land/) ca
```
oras push ghcr.io/<username>/blueprint/<blueprint name>:latest <yaml file>:application/vnd.goauthentik.blueprint.v1+yaml
```
## Storage - Internal
:::info
Requires authentik 2023.1
:::
Blueprints can be stored in authentik's database, which allows blueprints to be managed via external configuration management tools like Terraform.
Modifying the contents of a blueprint will trigger its reconciliation. Blueprints are validated on submission to prevent invalid blueprints from being saved.