outpost: rewrite kubernetes controller

passbook/outposts/controllers/k8s/base.py

@@ -0,0 +1,105 @@
+"""Base Kubernetes Reconciler"""
+from typing import Generic, TypeVar
+
+from kubernetes.client import V1ObjectMeta
+from kubernetes.client.rest import ApiException
+from structlog import get_logger
+
+from passbook import __version__
+from passbook.lib.sentry import SentryIgnoredException
+from passbook.outposts.models import Outpost
+
+# pylint: disable=invalid-name
+T = TypeVar("T")
+
+
+class ReconcileTrigger(SentryIgnoredException):
+    """Base trigger raised by child classes to notify us"""
+
+
+class NeedsRecreate(ReconcileTrigger):
+    """Exception to trigger a complete recreate of the Kubernetes Object"""
+
+
+class NeedsUpdate(ReconcileTrigger):
+    """Exception to trigger an update to the Kubernetes Object"""
+
+
+class KubernetesObjectReconciler(Generic[T]):
+    """Base Kubernetes Reconciler, handles the basic logic."""
+
+    def __init__(self, outpost: Outpost):
+        self.outpost = outpost
+        self.namespace = ""
+        self.logger = get_logger(
+            component=f"k8s-reconciler-{self.__class__.__name__}", outpost=outpost
+        )
+
+    def run(self):
+        """Create object if it doesn't exist, update if needed or recreate if needed."""
+        current = None
+        reference = self.get_reference_object()
+        try:
+            try:
+                current = self.retrieve()
+            except ApiException as exc:
+                if exc.status == 404:
+                    self.logger.debug("Failed to get current, triggering recreate")
+                    raise NeedsRecreate from exc
+                self.logger.debug("Other unhandled error", exc=exc)
+            else:
+                self.logger.debug("Got current, running reconcile")
+                self.reconcile(current, reference)
+        except NeedsRecreate:
+            self.logger.debug("Recreate requested")
+            if current:
+                self.logger.debug("Deleted old")
+                self.delete(current)
+            else:
+                self.logger.debug("No old found, creating")
+            self.logger.debug("Created")
+            self.create(reference)
+        except NeedsUpdate:
+            self.logger.debug("Updating")
+            self.update(current, reference)
+        else:
+            self.logger.debug("Nothing to do...")
+
+    def get_reference_object(self) -> T:
+        """Return object as it should be"""
+        raise NotImplementedError
+
+    def reconcile(self, current: T, reference: T):
+        """Check what operations should be done, should be raised as
+        ReconcileTrigger"""
+        raise NotImplementedError
+
+    def create(self, reference: T):
+        """API Wrapper to create object"""
+        raise NotImplementedError
+
+    def retrieve(self) -> T:
+        """API Wrapper to retrive object"""
+        raise NotImplementedError
+
+    def delete(self, reference: T):
+        """API Wrapper to delete object"""
+        raise NotImplementedError
+
+    def update(self, current: T, reference: T):
+        """API Wrapper to update object"""
+        raise NotImplementedError
+
+    def get_object_meta(self, **kwargs) -> V1ObjectMeta:
+        """Get common object metadata"""
+        return V1ObjectMeta(
+            namespace=self.namespace,
+            labels={
+                "app.kubernetes.io/name": f"passbook-{self.outpost.type.lower()}",
+                "app.kubernetes.io/instance": self.outpost.name,
+                "app.kubernetes.io/version": __version__,
+                "app.kubernetes.io/managed-by": "passbook.beryju.org",
+                "passbook.beryju.org/outpost-uuid": self.outpost.uuid.hex,
+            },
+            **kwargs,
+        )

passbook/outposts/controllers/k8s/deployment.py

@@ -0,0 +1,120 @@
+"""Kubernetes Deployment Reconciler"""
+from typing import Dict
+
+from kubernetes.client import (
+    AppsV1Api,
+    V1Container,
+    V1ContainerPort,
+    V1Deployment,
+    V1DeploymentSpec,
+    V1EnvVar,
+    V1EnvVarSource,
+    V1LabelSelector,
+    V1ObjectMeta,
+    V1PodSpec,
+    V1PodTemplateSpec,
+    V1SecretKeySelector,
+)
+
+from passbook import __version__
+from passbook.outposts.controllers.k8s.base import (
+    KubernetesObjectReconciler,
+    NeedsUpdate,
+)
+from passbook.outposts.models import Outpost
+
+
+class DeploymentReconciler(KubernetesObjectReconciler[V1Deployment]):
+    """Kubernetes Deployment Reconciler"""
+
+    image_base = "beryju/passbook"
+
+    deployment_ports: Dict[str, int]
+
+    def __init__(self, outpost: Outpost) -> None:
+        super().__init__(outpost)
+        self.api = AppsV1Api()
+        self.deployment_ports = {}
+
+    def reconcile(self, current: V1Deployment, reference: V1Deployment):
+        if current.spec.replicas != reference.spec.replicas:
+            raise NeedsUpdate()
+        if (
+            current.spec.template.spec.containers[0].image
+            != reference.spec.template.spec.containers[0].image
+        ):
+            raise NeedsUpdate()
+
+    def get_reference_object(self) -> V1Deployment:
+        """Get deployment object for outpost"""
+        # Generate V1ContainerPort objects
+        container_ports = []
+        for port_name, port in self.deployment_ports.items():
+            container_ports.append(V1ContainerPort(container_port=port, name=port_name))
+        meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}")
+        return V1Deployment(
+            metadata=meta,
+            spec=V1DeploymentSpec(
+                replicas=self.outpost.config.kubernetes_replicas,
+                selector=V1LabelSelector(match_labels=meta.labels),
+                template=V1PodTemplateSpec(
+                    metadata=V1ObjectMeta(labels=meta.labels),
+                    spec=V1PodSpec(
+                        containers=[
+                            V1Container(
+                                name=self.outpost.type,
+                                image=f"{self.image_base}-{self.outpost.type}:{__version__}",
+                                ports=container_ports,
+                                env=[
+                                    V1EnvVar(
+                                        name="PASSBOOK_HOST",
+                                        value_from=V1EnvVarSource(
+                                            secret_key_ref=V1SecretKeySelector(
+                                                name=f"passbook-outpost-{self.outpost.name}-api",
+                                                key="passbook_host",
+                                            )
+                                        ),
+                                    ),
+                                    V1EnvVar(
+                                        name="PASSBOOK_TOKEN",
+                                        value_from=V1EnvVarSource(
+                                            secret_key_ref=V1SecretKeySelector(
+                                                name=f"passbook-outpost-{self.outpost.name}-api",
+                                                key="token",
+                                            )
+                                        ),
+                                    ),
+                                    V1EnvVar(
+                                        name="PASSBOOK_INSECURE",
+                                        value_from=V1EnvVarSource(
+                                            secret_key_ref=V1SecretKeySelector(
+                                                name=f"passbook-outpost-{self.outpost.name}-api",
+                                                key="passbook_host_insecure",
+                                            )
+                                        ),
+                                    ),
+                                ],
+                            )
+                        ]
+                    ),
+                ),
+            ),
+        )
+
+    def create(self, reference: V1Deployment):
+        return self.api.create_namespaced_deployment(self.namespace, reference)
+
+    def delete(self, reference: V1Deployment):
+        return self.api.delete_namespaced_deployment(
+            reference.metadata.name, self.namespace
+        )
+
+    def retrieve(self) -> V1Deployment:
+        return self.api.read_namespaced_deployment(
+            f"passbook-outpost-{self.outpost.name}", self.namespace
+        )
+
+    def update(self, current: V1Deployment, reference: V1Deployment):
+        return self.api.patch_namespaced_deployment(
+            current.metadata.name, self.namespace, reference
+        )

passbook/outposts/controllers/k8s/secret.py

@@ -0,0 +1,60 @@
+"""Kubernetes Secret Reconciler"""
+from base64 import b64encode
+
+from kubernetes.client import CoreV1Api, V1Secret
+
+from passbook.outposts.controllers.k8s.base import (
+    KubernetesObjectReconciler,
+    NeedsUpdate,
+)
+from passbook.outposts.models import Outpost
+
+
+def b64string(source: str) -> str:
+    """Base64 Encode string"""
+    return b64encode(source.encode()).decode("utf-8")
+
+
+class SecretReconciler(KubernetesObjectReconciler[V1Secret]):
+    """Kubernetes Secret Reconciler"""
+
+    def __init__(self, outpost: Outpost) -> None:
+        super().__init__(outpost)
+        self.api = CoreV1Api()
+
+    def reconcile(self, current: V1Secret, reference: V1Secret):
+        for key in reference.data.keys():
+            if current.data[key] != reference.data[key]:
+                raise NeedsUpdate()
+
+    def get_reference_object(self) -> V1Secret:
+        """Get deployment object for outpost"""
+        meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}-api")
+        return V1Secret(
+            metadata=meta,
+            data={
+                "passbook_host": b64string(self.outpost.config.passbook_host),
+                "passbook_host_insecure": b64string(
+                    str(self.outpost.config.passbook_host_insecure)
+                ),
+                "token": b64string(self.outpost.token.token_uuid.hex),
+            },
+        )
+
+    def create(self, reference: V1Secret):
+        return self.api.create_namespaced_secret(self.namespace, reference)
+
+    def delete(self, reference: V1Secret):
+        return self.api.delete_namespaced_secret(
+            reference.metadata.name, self.namespace
+        )
+
+    def retrieve(self) -> V1Secret:
+        return self.api.read_namespaced_secret(
+            f"passbook-outpost-{self.outpost.name}-api", self.namespace
+        )
+
+    def update(self, current: V1Secret, reference: V1Secret):
+        return self.api.patch_namespaced_secret(
+            current.metadata.name, self.namespace, reference
+        )

passbook/outposts/controllers/k8s/service.py

@@ -0,0 +1,57 @@
+"""Kubernetes Service Reconciler"""
+from typing import Dict
+
+from kubernetes.client import CoreV1Api, V1Service, V1ServicePort, V1ServiceSpec
+
+from passbook.outposts.controllers.k8s.base import (
+    KubernetesObjectReconciler,
+    NeedsUpdate,
+)
+from passbook.outposts.models import Outpost
+
+
+class ServiceReconciler(KubernetesObjectReconciler[V1Service]):
+    """Kubernetes Service Reconciler"""
+
+    deployment_ports: Dict[str, int]
+
+    def __init__(self, outpost: Outpost) -> None:
+        super().__init__(outpost)
+        self.api = CoreV1Api()
+        self.deployment_ports = {}
+
+    def reconcile(self, current: V1Service, reference: V1Service):
+        if len(current.spec.ports) != len(reference.spec.ports):
+            raise NeedsUpdate()
+        for port in reference.spec.ports:
+            if port not in current.spec.ports:
+                raise NeedsUpdate()
+
+    def get_reference_object(self) -> V1Service:
+        """Get deployment object for outpost"""
+        meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}")
+        ports = []
+        for port_name, port in self.deployment_ports.items():
+            ports.append(V1ServicePort(name=port_name, port=port))
+        return V1Service(
+            metadata=meta,
+            spec=V1ServiceSpec(ports=ports, selector=meta.labels, type="ClusterIP"),
+        )
+
+    def create(self, reference: V1Service):
+        return self.api.create_namespaced_service(self.namespace, reference)
+
+    def delete(self, reference: V1Service):
+        return self.api.delete_namespaced_service(
+            reference.metadata.name, self.namespace
+        )
+
+    def retrieve(self) -> V1Service:
+        return self.api.read_namespaced_service(
+            f"passbook-outpost-{self.outpost.name}", self.namespace
+        )
+
+    def update(self, current: V1Service, reference: V1Service):
+        return self.api.patch_namespaced_service(
+            current.metadata.name, self.namespace, reference
+        )

passbook/outposts/controllers/kubernetes.py

@@ -1,156 +1,63 @@
 """Kubernetes deployment controller"""
-from base64 import b64encode
 from io import StringIO
 
-from kubernetes.client import (
-    V1Container,
-    V1ContainerPort,
-    V1Deployment,
-    V1DeploymentSpec,
-    V1EnvVar,
-    V1EnvVarSource,
-    V1LabelSelector,
-    V1ObjectMeta,
-    V1PodSpec,
-    V1PodTemplateSpec,
-    V1Secret,
-    V1SecretKeySelector,
-    V1Service,
-    V1ServicePort,
-    V1ServiceSpec,
-)
+from kubernetes.config import load_incluster_config, load_kube_config
+from kubernetes.config.config_exception import ConfigException
 from yaml import dump_all
 
-from passbook import __version__
 from passbook.outposts.controllers.base import BaseController
-
-
-def b64encode_str(input_string: str) -> str:
-    """base64 encode string"""
-    return b64encode(input_string.encode()).decode()
+from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler
+from passbook.outposts.controllers.k8s.secret import SecretReconciler
+from passbook.outposts.controllers.k8s.service import ServiceReconciler
 
 
 class KubernetesController(BaseController):
     """Manage deployment of outpost in kubernetes"""
 
-    image_base = "beryju/passbook"
+    def __init__(self, outpost_pk: str) -> None:
+        super().__init__(outpost_pk)
+        try:
+            load_incluster_config()
+        except ConfigException:
+            load_kube_config()
 
     def run(self):
         """Called by scheduled task to reconcile deployment/service/etc"""
-        # TODO
+        namespace = self.outpost.config.kubernetes_namespace
+
+        secret_reconciler = SecretReconciler(self.outpost)
+        secret_reconciler.namespace = namespace
+        secret_reconciler.run()
+
+        deployment_reconciler = DeploymentReconciler(self.outpost)
+        deployment_reconciler.namespace = namespace
+        deployment_reconciler.deployment_ports = self.deployment_ports
+        deployment_reconciler.run()
+
+        service_reconciler = ServiceReconciler(self.outpost)
+        service_reconciler.namespace = namespace
+        service_reconciler.deployment_ports = self.deployment_ports
+        service_reconciler.run()
 
     def get_static_deployment(self) -> str:
+        secret_reconciler = SecretReconciler(self.outpost)
+        secret_reconciler.namespace = ""
+
+        deployment_reconciler = DeploymentReconciler(self.outpost)
+        deployment_reconciler.namespace = ""
+        deployment_reconciler.deployment_ports = self.deployment_ports
+
+        service_reconciler = ServiceReconciler(self.outpost)
+        service_reconciler.namespace = ""
+        service_reconciler.deployment_ports = self.deployment_ports
         with StringIO() as _str:
             dump_all(
                 [
-                    self.get_deployment_secret().to_dict(),
-                    self.get_deployment().to_dict(),
-                    self.get_service().to_dict(),
+                    secret_reconciler.get_reference_object().to_dict(),
+                    deployment_reconciler.get_reference_object().to_dict(),
+                    service_reconciler.get_reference_object().to_dict(),
                 ],
                 stream=_str,
                 default_flow_style=False,
             )
             return _str.getvalue()
-
-    def get_object_meta(self, **kwargs) -> V1ObjectMeta:
-        """Get common object metadata"""
-        return V1ObjectMeta(
-            namespace="self.instance.namespace",
-            labels={
-                "app.kubernetes.io/name": f"passbook-{self.outpost.type.lower()}",
-                "app.kubernetes.io/instance": self.outpost.name,
-                "app.kubernetes.io/version": __version__,
-                "app.kubernetes.io/managed-by": "passbook.beryju.org",
-                "passbook.beryju.org/outpost/uuid": self.outpost.uuid.hex,
-            },
-            **kwargs,
-        )
-
-    def get_deployment_secret(self) -> V1Secret:
-        """Get secret with token and passbook host"""
-        return V1Secret(
-            api_version="v1",
-            kind="secret",
-            type="Opaque",
-            metadata=self.get_object_meta(
-                name=f"passbook-outpost-{self.outpost.name}-api"
-            ),
-            data={
-                "passbook_host": b64encode_str(self.outpost.config.passbook_host),
-                "passbook_host_insecure": b64encode_str(
-                    str(self.outpost.config.passbook_host_insecure)
-                ),
-                "token": b64encode_str(self.outpost.token.token_uuid.hex),
-            },
-        )
-
-    def get_service(self) -> V1Service:
-        """Get service object for outpost based on ports defined"""
-        meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}")
-        ports = []
-        for port_name, port in self.deployment_ports.items():
-            ports.append(V1ServicePort(name=port_name, port=port))
-        return V1Service(
-            api_version="v1",
-            kind="service",
-            metadata=meta,
-            spec=V1ServiceSpec(ports=ports, selector=meta.labels, type="ClusterIP"),
-        )
-
-    def get_deployment(self) -> V1Deployment:
-        """Get deployment object for outpost"""
-        # Generate V1ContainerPort objects
-        container_ports = []
-        for port_name, port in self.deployment_ports.items():
-            container_ports.append(V1ContainerPort(container_port=port, name=port_name))
-        meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}")
-        return V1Deployment(
-            api_version="apps/v1",
-            kind="deployment",
-            metadata=meta,
-            spec=V1DeploymentSpec(
-                replicas=1,
-                selector=V1LabelSelector(match_labels=meta.labels),
-                template=V1PodTemplateSpec(
-                    metadata=V1ObjectMeta(labels=meta.labels),
-                    spec=V1PodSpec(
-                        containers=[
-                            V1Container(
-                                name=self.outpost.type,
-                                image=f"{self.image_base}-{self.outpost.type}:{__version__}",
-                                ports=container_ports,
-                                env=[
-                                    V1EnvVar(
-                                        name="PASSBOOK_HOST",
-                                        value_from=V1EnvVarSource(
-                                            secret_key_ref=V1SecretKeySelector(
-                                                name=f"passbook-outpost-{self.outpost.name}-api",
-                                                key="passbook_host",
-                                            )
-                                        ),
-                                    ),
-                                    V1EnvVar(
-                                        name="PASSBOOK_TOKEN",
-                                        value_from=V1EnvVarSource(
-                                            secret_key_ref=V1SecretKeySelector(
-                                                name=f"passbook-outpost-{self.outpost.name}-api",
-                                                key="token",
-                                            )
-                                        ),
-                                    ),
-                                    V1EnvVar(
-                                        name="PASSBOOK_INSECURE",
-                                        value_from=V1EnvVarSource(
-                                            secret_key_ref=V1SecretKeySelector(
-                                                name=f"passbook-outpost-{self.outpost.name}-api",
-                                                key="passbook_host_insecure",
-                                            )
-                                        ),
-                                    ),
-                                ],
-                            )
-                        ]
-                    ),
-                ),
-            ),
-        )

passbook/outposts/migrations/0008_auto_20201014_1547.py

@@ -0,0 +1,26 @@
+# Generated by Django 3.1.2 on 2020-10-14 15:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("passbook_outposts", "0007_remove_outpost_channels"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="outpost",
+            name="deployment_type",
+            field=models.TextField(
+                choices=[
+                    ("kubernetes", "Kubernetes"),
+                    ("docker", "Docker"),
+                    ("custom", "Custom"),
+                ],
+                default="custom",
+                help_text="Select between passbook-managed deployment types or a custom deployment.",
+            ),
+        ),
+    ]

passbook/outposts/models.py

@@ -36,6 +36,9 @@ class OutpostConfig:
         "error_reporting.environment", "customer"
     )
 
+    kubernetes_replicas: int = field(default=1)
+    kubernetes_namespace: str = field(default="default")
+
 
 class OutpostModel(Model):
     """Base model for providers that need more objects than just themselves"""
@@ -58,7 +61,7 @@ class OutpostType(models.TextChoices):
 class OutpostDeploymentType(models.TextChoices):
     """Deployment types that are managed through passbook"""
 
-    # KUBERNETES = "kubernetes"
+    KUBERNETES = "kubernetes"
     DOCKER = "docker"
     CUSTOM = "custom"