outposts: initial service connection implementation
This commit is contained in:
parent
34793f7cef
commit
706448dc14
|
@ -13,7 +13,7 @@ class OutpostSerializer(ModelSerializer):
|
|||
class Meta:
|
||||
|
||||
model = Outpost
|
||||
fields = ["pk", "name", "providers", "_config"]
|
||||
fields = ["pk", "name", "providers", "service_connection", "_config"]
|
||||
|
||||
|
||||
class OutpostViewSet(ModelViewSet):
|
||||
|
|
|
@ -1,7 +1,19 @@
|
|||
"""passbook outposts app config"""
|
||||
from importlib import import_module
|
||||
from os import R_OK, access
|
||||
from os.path import expanduser
|
||||
from pathlib import Path
|
||||
from socket import gethostname
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.db import ProgrammingError
|
||||
from docker.constants import DEFAULT_UNIX_SOCKET
|
||||
from kubernetes.config.incluster_config import SERVICE_TOKEN_FILENAME
|
||||
from kubernetes.config.kube_config import KUBE_CONFIG_DEFAULT_LOCATION
|
||||
from structlog import get_logger
|
||||
|
||||
LOGGER = get_logger()
|
||||
|
||||
|
||||
class PassbookOutpostConfig(AppConfig):
|
||||
|
@ -14,3 +26,46 @@ class PassbookOutpostConfig(AppConfig):
|
|||
|
||||
def ready(self):
|
||||
import_module("passbook.outposts.signals")
|
||||
try:
|
||||
self.init_local_connection()
|
||||
except (ProgrammingError):
|
||||
pass
|
||||
|
||||
def init_local_connection(self):
|
||||
# Check if local kubernetes or docker connections should be created
|
||||
from passbook.outposts.models import (
|
||||
KubernetesServiceConnection,
|
||||
DockerServiceConnection,
|
||||
)
|
||||
|
||||
if Path(SERVICE_TOKEN_FILENAME).exists():
|
||||
LOGGER.debug("Detected in-cluster Kubernetes Config")
|
||||
if not KubernetesServiceConnection.objects.filter(local=True).exists():
|
||||
LOGGER.debug("Created Service Connection for in-cluster")
|
||||
KubernetesServiceConnection.objects.create(
|
||||
name="Local Kubernetes Cluster", local=True, config={}
|
||||
)
|
||||
# For development, check for the existence of a kubeconfig file
|
||||
kubeconfig_path = expanduser(KUBE_CONFIG_DEFAULT_LOCATION)
|
||||
if Path(kubeconfig_path).exists():
|
||||
LOGGER.debug("Detected kubeconfig")
|
||||
if not KubernetesServiceConnection.objects.filter(
|
||||
name=gethostname()
|
||||
).exists():
|
||||
LOGGER.debug("Creating kubeconfig Service Connection")
|
||||
with open(kubeconfig_path, "r") as _kubeconfig:
|
||||
KubernetesServiceConnection.objects.create(
|
||||
name=gethostname(), config=_kubeconfig.read()
|
||||
)
|
||||
unix_socket_path = urlparse(DEFAULT_UNIX_SOCKET).path
|
||||
socket = Path(unix_socket_path)
|
||||
if socket.exists() and access(socket, R_OK):
|
||||
LOGGER.debug("Detected local docker socket")
|
||||
if not DockerServiceConnection.objects.filter(local=True).exists():
|
||||
LOGGER.debug("Created Service Connection for docker")
|
||||
DockerServiceConnection.objects.create(
|
||||
name="Local Docker connection",
|
||||
local=True,
|
||||
url=unix_socket_path,
|
||||
tls=True,
|
||||
)
|
||||
|
|
|
@ -5,7 +5,7 @@ from structlog import get_logger
|
|||
from structlog.testing import capture_logs
|
||||
|
||||
from passbook.lib.sentry import SentryIgnoredException
|
||||
from passbook.outposts.models import Outpost
|
||||
from passbook.outposts.models import Outpost, OutpostServiceConnection
|
||||
|
||||
|
||||
class ControllerException(SentryIgnoredException):
|
||||
|
@ -18,6 +18,7 @@ class BaseController:
|
|||
deployment_ports: Dict[str, int]
|
||||
|
||||
outpost: Outpost
|
||||
connection: OutpostServiceConnection
|
||||
|
||||
def __init__(self, outpost: Outpost):
|
||||
self.outpost = outpost
|
||||
|
|
|
@ -3,14 +3,14 @@ from time import sleep
|
|||
from typing import Dict, Tuple
|
||||
|
||||
from django.conf import settings
|
||||
from docker import DockerClient, from_env
|
||||
from docker import DockerClient
|
||||
from docker.errors import DockerException, NotFound
|
||||
from docker.models.containers import Container
|
||||
from yaml import safe_dump
|
||||
|
||||
from passbook import __version__
|
||||
from passbook.outposts.controllers.base import BaseController, ControllerException
|
||||
from passbook.outposts.models import Outpost
|
||||
from passbook.outposts.models import DockerServiceConnection, Outpost
|
||||
|
||||
|
||||
class DockerController(BaseController):
|
||||
|
@ -19,13 +19,20 @@ class DockerController(BaseController):
|
|||
client: DockerClient
|
||||
|
||||
container: Container
|
||||
connection: DockerServiceConnection
|
||||
|
||||
image_base = "beryju/passbook"
|
||||
|
||||
def __init__(self, outpost: Outpost) -> None:
|
||||
super().__init__(outpost)
|
||||
try:
|
||||
self.client = from_env()
|
||||
if self.connection.local:
|
||||
self.client = DockerClient.from_env()
|
||||
else:
|
||||
self.client = DockerClient(
|
||||
base_url=self.connection.url,
|
||||
tls=self.connection.tls,
|
||||
)
|
||||
except DockerException as exc:
|
||||
raise ControllerException from exc
|
||||
|
||||
|
|
|
@ -5,6 +5,7 @@ from typing import Dict, List, Type
|
|||
from kubernetes.client import OpenApiException
|
||||
from kubernetes.config import load_incluster_config, load_kube_config
|
||||
from kubernetes.config.config_exception import ConfigException
|
||||
from kubernetes.config.kube_config import load_kube_config_from_dict
|
||||
from structlog.testing import capture_logs
|
||||
from yaml import dump_all
|
||||
|
||||
|
@ -13,7 +14,7 @@ from passbook.outposts.controllers.k8s.base import KubernetesObjectReconciler
|
|||
from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
|
||||
from passbook.outposts.controllers.k8s.secret import SecretReconciler
|
||||
from passbook.outposts.controllers.k8s.service import ServiceReconciler
|
||||
from passbook.outposts.models import Outpost
|
||||
from passbook.outposts.models import KubernetesServiceConnection, Outpost
|
||||
|
||||
|
||||
class KubernetesController(BaseController):
|
||||
|
@ -22,10 +23,15 @@ class KubernetesController(BaseController):
|
|||
reconcilers: Dict[str, Type[KubernetesObjectReconciler]]
|
||||
reconcile_order: List[str]
|
||||
|
||||
connection: KubernetesServiceConnection
|
||||
|
||||
def __init__(self, outpost: Outpost) -> None:
|
||||
super().__init__(outpost)
|
||||
try:
|
||||
load_incluster_config()
|
||||
if self.connection.local:
|
||||
load_incluster_config()
|
||||
else:
|
||||
load_kube_config_from_dict(self.connection.config)
|
||||
except ConfigException:
|
||||
load_kube_config()
|
||||
self.reconcilers = {
|
||||
|
|
|
@ -21,7 +21,7 @@ class OutpostForm(forms.ModelForm):
|
|||
fields = [
|
||||
"name",
|
||||
"type",
|
||||
"deployment_type",
|
||||
"service_connection",
|
||||
"providers",
|
||||
"_config",
|
||||
]
|
||||
|
|
|
@ -6,13 +6,18 @@ from django.db.backends.base.schema import BaseDatabaseSchemaEditor
|
|||
|
||||
|
||||
def fix_missing_token_identifier(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
|
||||
User = apps.get_model("passbook_core", "User")
|
||||
Token = apps.get_model("passbook_core", "Token")
|
||||
from passbook.outposts.models import Outpost
|
||||
|
||||
for outpost in Outpost.objects.using(schema_editor.connection.alias).all():
|
||||
token = outpost.token
|
||||
if token.identifier != outpost.token_identifier:
|
||||
token.identifier = outpost.token_identifier
|
||||
token.save()
|
||||
for outpost in Outpost.objects.using(schema_editor.connection.alias).all().only('pk'):
|
||||
user_identifier = outpost.user_identifier
|
||||
user = User.objects.get(username=user_identifier)
|
||||
tokens = Token.objects.filter(user=user)
|
||||
for token in tokens:
|
||||
if token.identifier != outpost.token_identifier:
|
||||
token.identifier = outpost.token_identifier
|
||||
token.save()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
|
|
@ -0,0 +1,73 @@
|
|||
# Generated by Django 3.1.3 on 2020-11-04 09:11
|
||||
|
||||
import uuid
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.apps.registry import Apps
|
||||
from django.db import migrations, models
|
||||
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
|
||||
|
||||
|
||||
def migrate_to_service_connection(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
|
||||
db_alias = schema_editor.connection.alias
|
||||
Outpost = apps.get_model("passbook_outposts", "Outpost")
|
||||
DockerServiceConnection = apps.get_model("passbook_outposts", "DockerServiceConnection")
|
||||
KubernetesServiceConnection = apps.get_model("passbook_outposts", "KubernetesServiceConnection")
|
||||
from passbook.outposts.apps import PassbookOutpostConfig
|
||||
# Ensure that local connection have been created
|
||||
PassbookOutpostConfig.init_local_connection(None)
|
||||
|
||||
docker = DockerServiceConnection.objects.filter(local=True)
|
||||
k8s = KubernetesServiceConnection.objects.filter(local=True)
|
||||
|
||||
for outpost in Outpost.objects.using(db_alias).all().exclude(deployment_type="custom"):
|
||||
if outpost.deployment_type == "kubernetes":
|
||||
outpost.service_connection = k8s
|
||||
elif outpost.deployment_type == "docker":
|
||||
outpost.service_connection = docker
|
||||
outpost.save()
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('passbook_outposts', '0009_fix_missing_token_identifier'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='OutpostServiceConnection',
|
||||
fields=[
|
||||
('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
|
||||
('name', models.TextField()),
|
||||
('local', models.BooleanField(default=False, help_text='If enabled, use the local connection. Required Docker socket/Kubernetes Integration', unique=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='DockerServiceConnection',
|
||||
fields=[
|
||||
('outpostserviceconnection_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_outposts.outpostserviceconnection')),
|
||||
('url', models.TextField()),
|
||||
('tls', models.BooleanField()),
|
||||
],
|
||||
bases=('passbook_outposts.outpostserviceconnection',),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='KubernetesServiceConnection',
|
||||
fields=[
|
||||
('outpostserviceconnection_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_outposts.outpostserviceconnection')),
|
||||
('config', models.JSONField()),
|
||||
],
|
||||
bases=('passbook_outposts.outpostserviceconnection',),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='outpost',
|
||||
name='service_connection',
|
||||
field=models.ForeignKey(blank=True, default=None, help_text='Select Service-Connection passbook should use to manage this outpost. Leave empty if passbook should not handle the deployment.', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='passbook_outposts.outpostserviceconnection'),
|
||||
),
|
||||
migrations.RunPython(migrate_to_service_connection),
|
||||
migrations.RemoveField(
|
||||
model_name='outpost',
|
||||
name='deployment_type',
|
||||
),
|
||||
]
|
|
@ -12,6 +12,7 @@ from django.http import HttpRequest
|
|||
from django.utils.translation import gettext_lazy as _
|
||||
from guardian.models import UserObjectPermission
|
||||
from guardian.shortcuts import assign_perm
|
||||
from model_utils.managers import InheritanceManager
|
||||
from packaging.version import LegacyVersion, Version, parse
|
||||
|
||||
from passbook import __version__
|
||||
|
@ -60,19 +61,44 @@ class OutpostType(models.TextChoices):
|
|||
PROXY = "proxy"
|
||||
|
||||
|
||||
class OutpostDeploymentType(models.TextChoices):
|
||||
"""Deployment types that are managed through passbook"""
|
||||
|
||||
KUBERNETES = "kubernetes"
|
||||
DOCKER = "docker"
|
||||
CUSTOM = "custom"
|
||||
|
||||
|
||||
def default_outpost_config():
|
||||
"""Get default outpost config"""
|
||||
return asdict(OutpostConfig(passbook_host=""))
|
||||
|
||||
|
||||
class OutpostServiceConnection(models.Model):
|
||||
"""Connection details for an Outpost Controller, like Docker or Kubernetes"""
|
||||
|
||||
uuid = models.UUIDField(default=uuid4, editable=False, primary_key=True)
|
||||
name = models.TextField()
|
||||
|
||||
local = models.BooleanField(
|
||||
default=False,
|
||||
unique=True,
|
||||
help_text=_(
|
||||
(
|
||||
"If enabled, use the local connection. Required Docker "
|
||||
"socket/Kubernetes Integration"
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
objects = InheritanceManager()
|
||||
|
||||
|
||||
class DockerServiceConnection(OutpostServiceConnection):
|
||||
"""Service Connection to a docker endpoint"""
|
||||
|
||||
url = models.TextField()
|
||||
tls = models.BooleanField()
|
||||
|
||||
|
||||
class KubernetesServiceConnection(OutpostServiceConnection):
|
||||
"""Service Connection to a kubernetes cluster"""
|
||||
|
||||
config = models.JSONField()
|
||||
|
||||
|
||||
class Outpost(models.Model):
|
||||
"""Outpost instance which manages a service user and token"""
|
||||
|
||||
|
@ -80,13 +106,20 @@ class Outpost(models.Model):
|
|||
name = models.TextField()
|
||||
|
||||
type = models.TextField(choices=OutpostType.choices, default=OutpostType.PROXY)
|
||||
deployment_type = models.TextField(
|
||||
choices=OutpostDeploymentType.choices,
|
||||
default=OutpostDeploymentType.CUSTOM,
|
||||
service_connection = models.ForeignKey(
|
||||
OutpostServiceConnection,
|
||||
default=None,
|
||||
null=True,
|
||||
blank=True,
|
||||
help_text=_(
|
||||
"Select between passbook-managed deployment types or a custom deployment."
|
||||
(
|
||||
"Select Service-Connection passbook should use to manage this outpost. "
|
||||
"Leave empty if passbook should not handle the deployment."
|
||||
)
|
||||
),
|
||||
on_delete=models.SET_DEFAULT,
|
||||
)
|
||||
|
||||
_config = models.JSONField(default=default_outpost_config)
|
||||
|
||||
providers = models.ManyToManyField(Provider)
|
||||
|
@ -111,12 +144,17 @@ class Outpost(models.Model):
|
|||
"""Get outpost's health status"""
|
||||
return OutpostState.for_outpost(self)
|
||||
|
||||
@property
|
||||
def user_identifier(self):
|
||||
"""Username for service user"""
|
||||
return f"pb-outpost-{self.uuid.hex}"
|
||||
|
||||
@property
|
||||
def user(self) -> User:
|
||||
"""Get/create user with access to all required objects"""
|
||||
users = User.objects.filter(username=f"pb-outpost-{self.uuid.hex}")
|
||||
users = User.objects.filter(username=self.user_identifier)
|
||||
if not users.exists():
|
||||
user: User = User.objects.create(username=f"pb-outpost-{self.uuid.hex}")
|
||||
user: User = User.objects.create(username=self.user_identifier)
|
||||
user.set_unusable_password()
|
||||
user.save()
|
||||
else:
|
||||
|
|
|
@ -10,13 +10,7 @@ from structlog import get_logger
|
|||
from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
|
||||
from passbook.lib.utils.reflection import path_to_class
|
||||
from passbook.outposts.controllers.base import ControllerException
|
||||
from passbook.outposts.models import (
|
||||
Outpost,
|
||||
OutpostDeploymentType,
|
||||
OutpostModel,
|
||||
OutpostState,
|
||||
OutpostType,
|
||||
)
|
||||
from passbook.outposts.models import Outpost, OutpostModel, OutpostState, OutpostType
|
||||
from passbook.providers.proxy.controllers.docker import ProxyDockerController
|
||||
from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController
|
||||
from passbook.root.celery import CELERY_APP
|
||||
|
@ -27,9 +21,7 @@ LOGGER = get_logger()
|
|||
@CELERY_APP.task()
|
||||
def outpost_controller_all():
|
||||
"""Launch Controller for all Outposts which support it"""
|
||||
for outpost in Outpost.objects.exclude(
|
||||
deployment_type=OutpostDeploymentType.CUSTOM
|
||||
):
|
||||
for outpost in Outpost.objects.exclude(service_connection=None):
|
||||
outpost_controller.delay(outpost.pk.hex)
|
||||
|
||||
|
||||
|
|
|
@ -12,5 +12,4 @@ class PassbookPoliciesConfig(AppConfig):
|
|||
verbose_name = "passbook Policies"
|
||||
|
||||
def ready(self):
|
||||
"""Load policy cache clearing signals"""
|
||||
import_module("passbook.policies.signals")
|
||||
|
|
Reference in New Issue