outposts: initial service connection implementation

passbook/outposts/api.py

@@ -13,7 +13,7 @@ class OutpostSerializer(ModelSerializer):
     class Meta:
 
         model = Outpost
-        fields = ["pk", "name", "providers", "_config"]
+        fields = ["pk", "name", "providers", "service_connection", "_config"]
 
 
 class OutpostViewSet(ModelViewSet):

passbook/outposts/apps.py

@@ -1,7 +1,19 @@
 """passbook outposts app config"""
 from importlib import import_module
+from os import R_OK, access
+from os.path import expanduser
+from pathlib import Path
+from socket import gethostname
+from urllib.parse import urlparse
 
 from django.apps import AppConfig
+from django.db import ProgrammingError
+from docker.constants import DEFAULT_UNIX_SOCKET
+from kubernetes.config.incluster_config import SERVICE_TOKEN_FILENAME
+from kubernetes.config.kube_config import KUBE_CONFIG_DEFAULT_LOCATION
+from structlog import get_logger
+
+LOGGER = get_logger()
 
 
 class PassbookOutpostConfig(AppConfig):
@@ -14,3 +26,46 @@ class PassbookOutpostConfig(AppConfig):
 
     def ready(self):
         import_module("passbook.outposts.signals")
+        try:
+            self.init_local_connection()
+        except (ProgrammingError):
+            pass
+
+    def init_local_connection(self):
+        # Check if local kubernetes or docker connections should be created
+        from passbook.outposts.models import (
+            KubernetesServiceConnection,
+            DockerServiceConnection,
+        )
+
+        if Path(SERVICE_TOKEN_FILENAME).exists():
+            LOGGER.debug("Detected in-cluster Kubernetes Config")
+            if not KubernetesServiceConnection.objects.filter(local=True).exists():
+                LOGGER.debug("Created Service Connection for in-cluster")
+                KubernetesServiceConnection.objects.create(
+                    name="Local Kubernetes Cluster", local=True, config={}
+                )
+        # For development, check for the existence of a kubeconfig file
+        kubeconfig_path = expanduser(KUBE_CONFIG_DEFAULT_LOCATION)
+        if Path(kubeconfig_path).exists():
+            LOGGER.debug("Detected kubeconfig")
+            if not KubernetesServiceConnection.objects.filter(
+                name=gethostname()
+            ).exists():
+                LOGGER.debug("Creating kubeconfig Service Connection")
+                with open(kubeconfig_path, "r") as _kubeconfig:
+                    KubernetesServiceConnection.objects.create(
+                        name=gethostname(), config=_kubeconfig.read()
+                    )
+        unix_socket_path = urlparse(DEFAULT_UNIX_SOCKET).path
+        socket = Path(unix_socket_path)
+        if socket.exists() and access(socket, R_OK):
+            LOGGER.debug("Detected local docker socket")
+            if not DockerServiceConnection.objects.filter(local=True).exists():
+                LOGGER.debug("Created Service Connection for docker")
+                DockerServiceConnection.objects.create(
+                    name="Local Docker connection",
+                    local=True,
+                    url=unix_socket_path,
+                    tls=True,
+                )

passbook/outposts/controllers/base.py

@@ -5,7 +5,7 @@ from structlog import get_logger
 from structlog.testing import capture_logs
 
 from passbook.lib.sentry import SentryIgnoredException
-from passbook.outposts.models import Outpost
+from passbook.outposts.models import Outpost, OutpostServiceConnection
 
 
 class ControllerException(SentryIgnoredException):
@@ -18,6 +18,7 @@ class BaseController:
     deployment_ports: Dict[str, int]
 
     outpost: Outpost
+    connection: OutpostServiceConnection
 
     def __init__(self, outpost: Outpost):
         self.outpost = outpost

passbook/outposts/controllers/docker.py

@@ -3,14 +3,14 @@ from time import sleep
 from typing import Dict, Tuple
 
 from django.conf import settings
-from docker import DockerClient, from_env
+from docker import DockerClient
 from docker.errors import DockerException, NotFound
 from docker.models.containers import Container
 from yaml import safe_dump
 
 from passbook import __version__
 from passbook.outposts.controllers.base import BaseController, ControllerException
-from passbook.outposts.models import Outpost
+from passbook.outposts.models import DockerServiceConnection, Outpost
 
 
 class DockerController(BaseController):
@@ -19,13 +19,20 @@ class DockerController(BaseController):
     client: DockerClient
 
     container: Container
+    connection: DockerServiceConnection
 
     image_base = "beryju/passbook"
 
     def __init__(self, outpost: Outpost) -> None:
         super().__init__(outpost)
         try:
-            self.client = from_env()
+            if self.connection.local:
+                self.client = DockerClient.from_env()
+            else:
+                self.client = DockerClient(
+                    base_url=self.connection.url,
+                    tls=self.connection.tls,
+                )
         except DockerException as exc:
             raise ControllerException from exc
 

passbook/outposts/controllers/kubernetes.py

@@ -5,6 +5,7 @@ from typing import Dict, List, Type
 from kubernetes.client import OpenApiException
 from kubernetes.config import load_incluster_config, load_kube_config
 from kubernetes.config.config_exception import ConfigException
+from kubernetes.config.kube_config import load_kube_config_from_dict
 from structlog.testing import capture_logs
 from yaml import dump_all
 
@@ -13,7 +14,7 @@ from passbook.outposts.controllers.k8s.base import KubernetesObjectReconciler
 from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
 from passbook.outposts.controllers.k8s.secret import SecretReconciler
 from passbook.outposts.controllers.k8s.service import ServiceReconciler
-from passbook.outposts.models import Outpost
+from passbook.outposts.models import KubernetesServiceConnection, Outpost
 
 
 class KubernetesController(BaseController):
@@ -22,10 +23,15 @@ class KubernetesController(BaseController):
     reconcilers: Dict[str, Type[KubernetesObjectReconciler]]
     reconcile_order: List[str]
 
+    connection: KubernetesServiceConnection
+
     def __init__(self, outpost: Outpost) -> None:
         super().__init__(outpost)
         try:
+            if self.connection.local:
                 load_incluster_config()
+            else:
+                load_kube_config_from_dict(self.connection.config)
         except ConfigException:
             load_kube_config()
         self.reconcilers = {

passbook/outposts/forms.py

@@ -21,7 +21,7 @@ class OutpostForm(forms.ModelForm):
         fields = [
             "name",
             "type",
-            "deployment_type",
+            "service_connection",
             "providers",
             "_config",
         ]

passbook/outposts/migrations/0009_fix_missing_token_identifier.py

@@ -6,10 +6,15 @@ from django.db.backends.base.schema import BaseDatabaseSchemaEditor
 
 
 def fix_missing_token_identifier(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
+    User = apps.get_model("passbook_core", "User")
+    Token = apps.get_model("passbook_core", "Token")
     from passbook.outposts.models import Outpost
 
-    for outpost in Outpost.objects.using(schema_editor.connection.alias).all():
+    for outpost in Outpost.objects.using(schema_editor.connection.alias).all().only('pk'):
-        token = outpost.token
+        user_identifier = outpost.user_identifier
+        user = User.objects.get(username=user_identifier)
+        tokens = Token.objects.filter(user=user)
+        for token in tokens:
             if token.identifier != outpost.token_identifier:
                 token.identifier = outpost.token_identifier
                 token.save()

passbook/outposts/migrations/0010_service_connection.py

@@ -0,0 +1,73 @@
+# Generated by Django 3.1.3 on 2020-11-04 09:11
+
+import uuid
+
+import django.db.models.deletion
+from django.apps.registry import Apps
+from django.db import migrations, models
+from django.db.backends.base.schema import BaseDatabaseSchemaEditor
+
+
+def migrate_to_service_connection(apps: Apps, schema_editor: BaseDatabaseSchemaEditor):
+    db_alias = schema_editor.connection.alias
+    Outpost = apps.get_model("passbook_outposts", "Outpost")
+    DockerServiceConnection = apps.get_model("passbook_outposts", "DockerServiceConnection")
+    KubernetesServiceConnection = apps.get_model("passbook_outposts", "KubernetesServiceConnection")
+    from passbook.outposts.apps import PassbookOutpostConfig
+    # Ensure that local connection have been created
+    PassbookOutpostConfig.init_local_connection(None)
+
+    docker = DockerServiceConnection.objects.filter(local=True)
+    k8s = KubernetesServiceConnection.objects.filter(local=True)
+
+    for outpost in Outpost.objects.using(db_alias).all().exclude(deployment_type="custom"):
+        if outpost.deployment_type == "kubernetes":
+            outpost.service_connection = k8s
+        elif outpost.deployment_type == "docker":
+            outpost.service_connection = docker
+        outpost.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('passbook_outposts', '0009_fix_missing_token_identifier'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='OutpostServiceConnection',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.TextField()),
+                ('local', models.BooleanField(default=False, help_text='If enabled, use the local connection. Required Docker socket/Kubernetes Integration', unique=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='DockerServiceConnection',
+            fields=[
+                ('outpostserviceconnection_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_outposts.outpostserviceconnection')),
+                ('url', models.TextField()),
+                ('tls', models.BooleanField()),
+            ],
+            bases=('passbook_outposts.outpostserviceconnection',),
+        ),
+        migrations.CreateModel(
+            name='KubernetesServiceConnection',
+            fields=[
+                ('outpostserviceconnection_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='passbook_outposts.outpostserviceconnection')),
+                ('config', models.JSONField()),
+            ],
+            bases=('passbook_outposts.outpostserviceconnection',),
+        ),
+        migrations.AddField(
+            model_name='outpost',
+            name='service_connection',
+            field=models.ForeignKey(blank=True, default=None, help_text='Select Service-Connection passbook should use to manage this outpost. Leave empty if passbook should not handle the deployment.', null=True, on_delete=django.db.models.deletion.SET_DEFAULT, to='passbook_outposts.outpostserviceconnection'),
+        ),
+        migrations.RunPython(migrate_to_service_connection),
+        migrations.RemoveField(
+            model_name='outpost',
+            name='deployment_type',
+        ),
+    ]

passbook/outposts/models.py

@@ -12,6 +12,7 @@ from django.http import HttpRequest
 from django.utils.translation import gettext_lazy as _
 from guardian.models import UserObjectPermission
 from guardian.shortcuts import assign_perm
+from model_utils.managers import InheritanceManager
 from packaging.version import LegacyVersion, Version, parse
 
 from passbook import __version__
@@ -60,19 +61,44 @@ class OutpostType(models.TextChoices):
     PROXY = "proxy"
 
 
-class OutpostDeploymentType(models.TextChoices):
-    """Deployment types that are managed through passbook"""
-
-    KUBERNETES = "kubernetes"
-    DOCKER = "docker"
-    CUSTOM = "custom"
-
-
 def default_outpost_config():
     """Get default outpost config"""
     return asdict(OutpostConfig(passbook_host=""))
 
 
+class OutpostServiceConnection(models.Model):
+    """Connection details for an Outpost Controller, like Docker or Kubernetes"""
+
+    uuid = models.UUIDField(default=uuid4, editable=False, primary_key=True)
+    name = models.TextField()
+
+    local = models.BooleanField(
+        default=False,
+        unique=True,
+        help_text=_(
+            (
+                "If enabled, use the local connection. Required Docker "
+                "socket/Kubernetes Integration"
+            )
+        ),
+    )
+
+    objects = InheritanceManager()
+
+
+class DockerServiceConnection(OutpostServiceConnection):
+    """Service Connection to a docker endpoint"""
+
+    url = models.TextField()
+    tls = models.BooleanField()
+
+
+class KubernetesServiceConnection(OutpostServiceConnection):
+    """Service Connection to a kubernetes cluster"""
+
+    config = models.JSONField()
+
+
 class Outpost(models.Model):
     """Outpost instance which manages a service user and token"""
 
@@ -80,13 +106,20 @@ class Outpost(models.Model):
     name = models.TextField()
 
     type = models.TextField(choices=OutpostType.choices, default=OutpostType.PROXY)
-    deployment_type = models.TextField(
+    service_connection = models.ForeignKey(
-        choices=OutpostDeploymentType.choices,
+        OutpostServiceConnection,
-        default=OutpostDeploymentType.CUSTOM,
+        default=None,
+        null=True,
+        blank=True,
         help_text=_(
-            "Select between passbook-managed deployment types or a custom deployment."
+            (
-        ),
+                "Select Service-Connection passbook should use to manage this outpost. "
+                "Leave empty if passbook should not handle the deployment."
             )
+        ),
+        on_delete=models.SET_DEFAULT,
+    )
+
     _config = models.JSONField(default=default_outpost_config)
 
     providers = models.ManyToManyField(Provider)
@@ -111,12 +144,17 @@ class Outpost(models.Model):
         """Get outpost's health status"""
         return OutpostState.for_outpost(self)
 
+    @property
+    def user_identifier(self):
+        """Username for service user"""
+        return f"pb-outpost-{self.uuid.hex}"
+
     @property
     def user(self) -> User:
         """Get/create user with access to all required objects"""
-        users = User.objects.filter(username=f"pb-outpost-{self.uuid.hex}")
+        users = User.objects.filter(username=self.user_identifier)
         if not users.exists():
-            user: User = User.objects.create(username=f"pb-outpost-{self.uuid.hex}")
+            user: User = User.objects.create(username=self.user_identifier)
             user.set_unusable_password()
             user.save()
         else:

passbook/outposts/tasks.py

@@ -10,13 +10,7 @@ from structlog import get_logger
 from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
 from passbook.lib.utils.reflection import path_to_class
 from passbook.outposts.controllers.base import ControllerException
-from passbook.outposts.models import (
+from passbook.outposts.models import Outpost, OutpostModel, OutpostState, OutpostType
-    Outpost,
-    OutpostDeploymentType,
-    OutpostModel,
-    OutpostState,
-    OutpostType,
-)
 from passbook.providers.proxy.controllers.docker import ProxyDockerController
 from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController
 from passbook.root.celery import CELERY_APP
@@ -27,9 +21,7 @@ LOGGER = get_logger()
 @CELERY_APP.task()
 def outpost_controller_all():
     """Launch Controller for all Outposts which support it"""
-    for outpost in Outpost.objects.exclude(
+    for outpost in Outpost.objects.exclude(service_connection=None):
-        deployment_type=OutpostDeploymentType.CUSTOM
-    ):
         outpost_controller.delay(outpost.pk.hex)
 
 

passbook/policies/apps.py

@@ -12,5 +12,4 @@ class PassbookPoliciesConfig(AppConfig):
     verbose_name = "passbook Policies"
 
     def ready(self):
-        """Load policy cache clearing signals"""
         import_module("passbook.policies.signals")