trustchain-oc1-orchestral
/
authentik
Archived
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

outposts: initial service connection implementation

This commit is contained in:
Jens Langhammer 2020-11-04 10:41:18 +01:00
parent 34793f7cef
commit 706448dc14
11 changed files with 214 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
passbook/outposts/api.py
View File

@ -13,7 +13,7 @@ class OutpostSerializer(ModelSerializer):
class Meta:
model = Outpost
fields = ["pk", "name", "providers", "_config"]
fields = ["pk", "name", "providers", "service_connection", "_config"]
class OutpostViewSet(ModelViewSet):

55
passbook/outposts/apps.py
View File

@ -1,7 +1,19 @@
"""passbook outposts app config"""
from importlib import import_module
from os import R_OK, access
from os.path import expanduser
from pathlib import Path
from socket import gethostname
from urllib.parse import urlparse
from django.apps import AppConfig
from django.db import ProgrammingError
from docker.constants import DEFAULT_UNIX_SOCKET
from kubernetes.config.incluster_config import SERVICE_TOKEN_FILENAME
from kubernetes.config.kube_config import KUBE_CONFIG_DEFAULT_LOCATION
from structlog import get_logger
LOGGER = get_logger()
class PassbookOutpostConfig(AppConfig):
@ -14,3 +26,46 @@ class PassbookOutpostConfig(AppConfig):
def ready(self):
import_module("passbook.outposts.signals")
try:
self.init_local_connection()
except (ProgrammingError):
pass
def init_local_connection(self):
# Check if local kubernetes or docker connections should be created
from passbook.outposts.models import (
KubernetesServiceConnection,
DockerServiceConnection,
)
if Path(SERVICE_TOKEN_FILENAME).exists():
LOGGER.debug("Detected in-cluster Kubernetes Config")
if not KubernetesServiceConnection.objects.filter(local=True).exists():
LOGGER.debug("Created Service Connection for in-cluster")
KubernetesServiceConnection.objects.create(
name="Local Kubernetes Cluster", local=True, config={}
)
# For development, check for the existence of a kubeconfig file
kubeconfig_path = expanduser(KUBE_CONFIG_DEFAULT_LOCATION)
if Path(kubeconfig_path).exists():
LOGGER.debug("Detected kubeconfig")
if not KubernetesServiceConnection.objects.filter(
name=gethostname()
).exists():
LOGGER.debug("Creating kubeconfig Service Connection")
with open(kubeconfig_path, "r") as _kubeconfig:
KubernetesServiceConnection.objects.create(
name=gethostname(), config=_kubeconfig.read()
)
unix_socket_path = urlparse(DEFAULT_UNIX_SOCKET).path
socket = Path(unix_socket_path)
if socket.exists() and access(socket, R_OK):
LOGGER.debug("Detected local docker socket")
if not DockerServiceConnection.objects.filter(local=True).exists():
LOGGER.debug("Created Service Connection for docker")
DockerServiceConnection.objects.create(
name="Local Docker connection",
local=True,
url=unix_socket_path,
tls=True,
)

3
passbook/outposts/controllers/base.py
View File

@ -5,7 +5,7 @@ from structlog import get_logger
from structlog.testing import capture_logs
from passbook.lib.sentry import SentryIgnoredException
from passbook.outposts.models import Outpost
from passbook.outposts.models import Outpost, OutpostServiceConnection
class ControllerException(SentryIgnoredException):
@ -18,6 +18,7 @@ class BaseController:
deployment_ports: Dict[str, int]
outpost: Outpost
connection: OutpostServiceConnection
def __init__(self, outpost: Outpost):
self.outpost = outpost

13
passbook/outposts/controllers/docker.py
View File

@ -3,14 +3,14 @@ from time import sleep
from typing import Dict, Tuple
from django.conf import settings
from docker import DockerClient, from_env
from docker import DockerClient
from docker.errors import DockerException, NotFound
from docker.models.containers import Container
from yaml import safe_dump
from passbook import __version__
from passbook.outposts.controllers.base import BaseController, ControllerException
from passbook.outposts.models import Outpost
from passbook.outposts.models import DockerServiceConnection, Outpost
class DockerController(BaseController):
@ -19,13 +19,20 @@ class DockerController(BaseController):
client: DockerClient
container: Container
connection: DockerServiceConnection
image_base = "beryju/passbook"
def __init__(self, outpost: Outpost) -> None:
super().__init__(outpost)
try:
self.client = from_env()
if self.connection.local:
self.client = DockerClient.from_env()
else:
self.client = DockerClient(
base_url=self.connection.url,
tls=self.connection.tls,
)
except DockerException as exc:
raise ControllerException from exc

8
passbook/outposts/controllers/kubernetes.py
View File

@ -5,6 +5,7 @@ from typing import Dict, List, Type
from kubernetes.client import OpenApiException
from kubernetes.config import load_incluster_config, load_kube_config
from kubernetes.config.config_exception import ConfigException
from kubernetes.config.kube_config import load_kube_config_from_dict
from structlog.testing import capture_logs
from yaml import dump_all
@ -13,7 +14,7 @@ from passbook.outposts.controllers.k8s.base import KubernetesObjectReconciler
from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
from passbook.outposts.controllers.k8s.secret import SecretReconciler
from passbook.outposts.controllers.k8s.service import ServiceReconciler
from passbook.outposts.models import Outpost
from passbook.outposts.models import KubernetesServiceConnection, Outpost
class KubernetesController(BaseController):
@ -22,10 +23,15 @@ class KubernetesController(BaseController):
reconcilers: Dict[str, Type[KubernetesObjectReconciler]]
reconcile_order: List[str]
connection: KubernetesServiceConnection
def __init__(self, outpost: Outpost) -> None:
super().__init__(outpost)
try:
if self.connection.local:
load_incluster_config()
else:
load_kube_config_from_dict(self.connection.config)
except ConfigException:
load_kube_config()
self.reconcilers = {

2
passbook/outposts/forms.py
View File

@ -21,7 +21,7 @@ class OutpostForm(forms.ModelForm):
fields = [
"name",
"type",
"deployment_type",
"service_connection",
"providers",
"_config",
]

9
passbook/outposts/migrations/0009_fix_missing_token_identifier.py
View File

@ -6,10 +6,15 @@ from django.db.backends.base.schema import BaseDatabaseSchemaEditor
def fix_missing_token_identifier(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
User = apps.get_model("passbook_core", "User")
Token = apps.get_model("passbook_core", "Token")
from passbook.outposts.models import Outpost
for outpost in Outpost.objects.using(schema_editor.connection.alias).all():
token = outpost.token
for outpost in Outpost.objects.using(schema_editor.connection.alias).all().only('pk'):
user_identifier = outpost.user_identifier
user = User.objects.get(username=user_identifier)
tokens = Token.objects.filter(user=user)
for token in tokens:
if token.identifier != outpost.token_identifier:
token.identifier = outpost.token_identifier
token.save()

73
passbook/outposts/migrations/0010_service_connection.py Normal file
View File

@ -0,0 +1,73 @@
# Generated by Django 3.1.3 on 2020-11-04 09:11
import uuid
import django.db.models.deletion
from django.apps.registry import Apps
from django.db import migrations, models
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
def migrate_to_service_connection(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
db_alias = schema_editor.connection.alias
Outpost = apps.get_model("passbook_outposts", "Outpost")
DockerServiceConnection = apps.get_model("passbook_outposts", "DockerServiceConnection")
KubernetesServiceConnection = apps.get_model("passbook_outposts", "KubernetesServiceConnection")
from passbook.outposts.apps import PassbookOutpostConfig
# Ensure that local connection have been created
PassbookOutpostConfig.init_local_connection(None)
docker = DockerServiceConnection.objects.filter(local=True)
k8s = KubernetesServiceConnection.objects.filter(local=True)
for outpost in Outpost.objects.using(db_alias).all().exclude(deployment_type="custom"):
if outpost.deployment_type == "kubernetes":
outpost.service_connection = k8s
elif outpost.deployment_type == "docker":
outpost.service_connection = docker
outpost.save()
class Migration(migrations.Migration):
dependencies = [
('passbook_outposts', '0009_fix_missing_token_identifier'),
]
operations = [
migrations.CreateModel(
name='OutpostServiceConnection',
fields=[
('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('name', models.TextField()),
('local', models.BooleanField(default=False, help_text='If enabled, use the local connection. Required Docker socket/Kubernetes Integration', unique=True)),
],
),
migrations.CreateModel(
name='DockerServiceConnection',
fields=[
('outpostserviceconnection_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_outposts.outpostserviceconnection')),
('url', models.TextField()),
('tls', models.BooleanField()),
],
bases=('passbook_outposts.outpostserviceconnection',),
),
migrations.CreateModel(
name='KubernetesServiceConnection',
fields=[
('outpostserviceconnection_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_outposts.outpostserviceconnection')),
('config', models.JSONField()),
],
bases=('passbook_outposts.outpostserviceconnection',),
),
migrations.AddField(
model_name='outpost',
name='service_connection',
field=models.ForeignKey(blank=True, default=None, help_text='Select Service-Connection passbook should use to manage this outpost. Leave empty if passbook should not handle the deployment.', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='passbook_outposts.outpostserviceconnection'),
),
migrations.RunPython(migrate_to_service_connection),
migrations.RemoveField(
model_name='outpost',
name='deployment_type',
),
]

68
passbook/outposts/models.py
View File

@ -12,6 +12,7 @@ from django.http import HttpRequest
from django.utils.translation import gettext_lazy as _
from guardian.models import UserObjectPermission
from guardian.shortcuts import assign_perm
from model_utils.managers import InheritanceManager
from packaging.version import LegacyVersion, Version, parse
from passbook import __version__
@ -60,19 +61,44 @@ class OutpostType(models.TextChoices):
PROXY = "proxy"
class OutpostDeploymentType(models.TextChoices):
"""Deployment types that are managed through passbook"""
KUBERNETES = "kubernetes"
DOCKER = "docker"
CUSTOM = "custom"
def default_outpost_config():
"""Get default outpost config"""
return asdict(OutpostConfig(passbook_host=""))
class OutpostServiceConnection(models.Model):
"""Connection details for an Outpost Controller, like Docker or Kubernetes"""
uuid = models.UUIDField(default=uuid4, editable=False, primary_key=True)
name = models.TextField()
local = models.BooleanField(
default=False,
unique=True,
help_text=_(
(
"If enabled, use the local connection. Required Docker "
"socket/Kubernetes Integration"
)
),
)
objects = InheritanceManager()
class DockerServiceConnection(OutpostServiceConnection):
"""Service Connection to a docker endpoint"""
url = models.TextField()
tls = models.BooleanField()
class KubernetesServiceConnection(OutpostServiceConnection):
"""Service Connection to a kubernetes cluster"""
config = models.JSONField()
class Outpost(models.Model):
"""Outpost instance which manages a service user and token"""
@ -80,13 +106,20 @@ class Outpost(models.Model):
name = models.TextField()
type = models.TextField(choices=OutpostType.choices, default=OutpostType.PROXY)
deployment_type = models.TextField(
choices=OutpostDeploymentType.choices,
default=OutpostDeploymentType.CUSTOM,
service_connection = models.ForeignKey(
OutpostServiceConnection,
default=None,
null=True,
blank=True,
help_text=_(
"Select between passbook-managed deployment types or a custom deployment."
),
(
"Select Service-Connection passbook should use to manage this outpost. "
"Leave empty if passbook should not handle the deployment."
)
),
on_delete=models.SET_DEFAULT,
)
_config = models.JSONField(default=default_outpost_config)
providers = models.ManyToManyField(Provider)
@ -111,12 +144,17 @@ class Outpost(models.Model):
"""Get outpost's health status"""
return OutpostState.for_outpost(self)
@property
def user_identifier(self):
"""Username for service user"""
return f"pb-outpost-{self.uuid.hex}"
@property
def user(self) -> User:
"""Get/create user with access to all required objects"""
users = User.objects.filter(username=f"pb-outpost-{self.uuid.hex}")
users = User.objects.filter(username=self.user_identifier)
if not users.exists():
user: User = User.objects.create(username=f"pb-outpost-{self.uuid.hex}")
user: User = User.objects.create(username=self.user_identifier)
user.set_unusable_password()
user.save()
else:

12
passbook/outposts/tasks.py
View File

@ -10,13 +10,7 @@ from structlog import get_logger
from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
from passbook.lib.utils.reflection import path_to_class
from passbook.outposts.controllers.base import ControllerException
from passbook.outposts.models import (
Outpost,
OutpostDeploymentType,
OutpostModel,
OutpostState,
OutpostType,
)
from passbook.outposts.models import Outpost, OutpostModel, OutpostState, OutpostType
from passbook.providers.proxy.controllers.docker import ProxyDockerController
from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController
from passbook.root.celery import CELERY_APP
@ -27,9 +21,7 @@ LOGGER = get_logger()
@CELERY_APP.task()
def outpost_controller_all():
"""Launch Controller for all Outposts which support it"""
for outpost in Outpost.objects.exclude(
deployment_type=OutpostDeploymentType.CUSTOM
):
for outpost in Outpost.objects.exclude(service_connection=None):
outpost_controller.delay(outpost.pk.hex)

1
passbook/policies/apps.py
View File

@ -12,5 +12,4 @@ class PassbookPoliciesConfig(AppConfig):
verbose_name = "passbook Policies"
def ready(self):
"""Load policy cache clearing signals"""
import_module("passbook.policies.signals")