diff --git a/passbook/outposts/controllers/k8s/__init__.py b/passbook/outposts/controllers/k8s/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/passbook/outposts/controllers/k8s/base.py b/passbook/outposts/controllers/k8s/base.py new file mode 100644 index 000000000..a2b6a5c04 --- /dev/null +++ b/passbook/outposts/controllers/k8s/base.py @@ -0,0 +1,105 @@ +"""Base Kubernetes Reconciler""" +from typing import Generic, TypeVar + +from kubernetes.client import V1ObjectMeta +from kubernetes.client.rest import ApiException +from structlog import get_logger + +from passbook import __version__ +from passbook.lib.sentry import SentryIgnoredException +from passbook.outposts.models import Outpost + +# pylint: disable=invalid-name +T = TypeVar("T") + + +class ReconcileTrigger(SentryIgnoredException): + """Base trigger raised by child classes to notify us""" + + +class NeedsRecreate(ReconcileTrigger): + """Exception to trigger a complete recreate of the Kubernetes Object""" + + +class NeedsUpdate(ReconcileTrigger): + """Exception to trigger an update to the Kubernetes Object""" + + +class KubernetesObjectReconciler(Generic[T]): + """Base Kubernetes Reconciler, handles the basic logic.""" + + def __init__(self, outpost: Outpost): + self.outpost = outpost + self.namespace = "" + self.logger = get_logger( + component=f"k8s-reconciler-{self.__class__.__name__}", outpost=outpost + ) + + def run(self): + """Create object if it doesn't exist, update if needed or recreate if needed.""" + current = None + reference = self.get_reference_object() + try: + try: + current = self.retrieve() + except ApiException as exc: + if exc.status == 404: + self.logger.debug("Failed to get current, triggering recreate") + raise NeedsRecreate from exc + self.logger.debug("Other unhandled error", exc=exc) + else: + self.logger.debug("Got current, running reconcile") + self.reconcile(current, reference) + except NeedsRecreate: + self.logger.debug("Recreate requested") + if current: + self.logger.debug("Deleted old") + self.delete(current) + else: + self.logger.debug("No old found, creating") + self.logger.debug("Created") + self.create(reference) + except NeedsUpdate: + self.logger.debug("Updating") + self.update(current, reference) + else: + self.logger.debug("Nothing to do...") + + def get_reference_object(self) -> T: + """Return object as it should be""" + raise NotImplementedError + + def reconcile(self, current: T, reference: T): + """Check what operations should be done, should be raised as + ReconcileTrigger""" + raise NotImplementedError + + def create(self, reference: T): + """API Wrapper to create object""" + raise NotImplementedError + + def retrieve(self) -> T: + """API Wrapper to retrive object""" + raise NotImplementedError + + def delete(self, reference: T): + """API Wrapper to delete object""" + raise NotImplementedError + + def update(self, current: T, reference: T): + """API Wrapper to update object""" + raise NotImplementedError + + def get_object_meta(self, **kwargs) -> V1ObjectMeta: + """Get common object metadata""" + return V1ObjectMeta( + namespace=self.namespace, + labels={ + "app.kubernetes.io/name": f"passbook-{self.outpost.type.lower()}", + "app.kubernetes.io/instance": self.outpost.name, + "app.kubernetes.io/version": __version__, + "app.kubernetes.io/managed-by": "passbook.beryju.org", + "passbook.beryju.org/outpost-uuid": self.outpost.uuid.hex, + }, + **kwargs, + ) diff --git a/passbook/outposts/controllers/k8s/deployment.py b/passbook/outposts/controllers/k8s/deployment.py new file mode 100644 index 000000000..6cd0ba0cb --- /dev/null +++ b/passbook/outposts/controllers/k8s/deployment.py @@ -0,0 +1,120 @@ +"""Kubernetes Deployment Reconciler""" +from typing import Dict + +from kubernetes.client import ( + AppsV1Api, + V1Container, + V1ContainerPort, + V1Deployment, + V1DeploymentSpec, + V1EnvVar, + V1EnvVarSource, + V1LabelSelector, + V1ObjectMeta, + V1PodSpec, + V1PodTemplateSpec, + V1SecretKeySelector, +) + +from passbook import __version__ +from passbook.outposts.controllers.k8s.base import ( + KubernetesObjectReconciler, + NeedsUpdate, +) +from passbook.outposts.models import Outpost + + +class DeploymentReconciler(KubernetesObjectReconciler[V1Deployment]): + """Kubernetes Deployment Reconciler""" + + image_base = "beryju/passbook" + + deployment_ports: Dict[str, int] + + def __init__(self, outpost: Outpost) -> None: + super().__init__(outpost) + self.api = AppsV1Api() + self.deployment_ports = {} + + def reconcile(self, current: V1Deployment, reference: V1Deployment): + if current.spec.replicas != reference.spec.replicas: + raise NeedsUpdate() + if ( + current.spec.template.spec.containers[0].image + != reference.spec.template.spec.containers[0].image + ): + raise NeedsUpdate() + + def get_reference_object(self) -> V1Deployment: + """Get deployment object for outpost""" + # Generate V1ContainerPort objects + container_ports = [] + for port_name, port in self.deployment_ports.items(): + container_ports.append(V1ContainerPort(container_port=port, name=port_name)) + meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}") + return V1Deployment( + metadata=meta, + spec=V1DeploymentSpec( + replicas=self.outpost.config.kubernetes_replicas, + selector=V1LabelSelector(match_labels=meta.labels), + template=V1PodTemplateSpec( + metadata=V1ObjectMeta(labels=meta.labels), + spec=V1PodSpec( + containers=[ + V1Container( + name=self.outpost.type, + image=f"{self.image_base}-{self.outpost.type}:{__version__}", + ports=container_ports, + env=[ + V1EnvVar( + name="PASSBOOK_HOST", + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name=f"passbook-outpost-{self.outpost.name}-api", + key="passbook_host", + ) + ), + ), + V1EnvVar( + name="PASSBOOK_TOKEN", + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name=f"passbook-outpost-{self.outpost.name}-api", + key="token", + ) + ), + ), + V1EnvVar( + name="PASSBOOK_INSECURE", + value_from=V1EnvVarSource( + secret_key_ref=V1SecretKeySelector( + name=f"passbook-outpost-{self.outpost.name}-api", + key="passbook_host_insecure", + ) + ), + ), + ], + ) + ] + ), + ), + ), + ) + + def create(self, reference: V1Deployment): + return self.api.create_namespaced_deployment(self.namespace, reference) + + def delete(self, reference: V1Deployment): + return self.api.delete_namespaced_deployment( + reference.metadata.name, self.namespace + ) + + def retrieve(self) -> V1Deployment: + return self.api.read_namespaced_deployment( + f"passbook-outpost-{self.outpost.name}", self.namespace + ) + + def update(self, current: V1Deployment, reference: V1Deployment): + return self.api.patch_namespaced_deployment( + current.metadata.name, self.namespace, reference + ) diff --git a/passbook/outposts/controllers/k8s/secret.py b/passbook/outposts/controllers/k8s/secret.py new file mode 100644 index 000000000..6a307befe --- /dev/null +++ b/passbook/outposts/controllers/k8s/secret.py @@ -0,0 +1,60 @@ +"""Kubernetes Secret Reconciler""" +from base64 import b64encode + +from kubernetes.client import CoreV1Api, V1Secret + +from passbook.outposts.controllers.k8s.base import ( + KubernetesObjectReconciler, + NeedsUpdate, +) +from passbook.outposts.models import Outpost + + +def b64string(source: str) -> str: + """Base64 Encode string""" + return b64encode(source.encode()).decode("utf-8") + + +class SecretReconciler(KubernetesObjectReconciler[V1Secret]): + """Kubernetes Secret Reconciler""" + + def __init__(self, outpost: Outpost) -> None: + super().__init__(outpost) + self.api = CoreV1Api() + + def reconcile(self, current: V1Secret, reference: V1Secret): + for key in reference.data.keys(): + if current.data[key] != reference.data[key]: + raise NeedsUpdate() + + def get_reference_object(self) -> V1Secret: + """Get deployment object for outpost""" + meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}-api") + return V1Secret( + metadata=meta, + data={ + "passbook_host": b64string(self.outpost.config.passbook_host), + "passbook_host_insecure": b64string( + str(self.outpost.config.passbook_host_insecure) + ), + "token": b64string(self.outpost.token.token_uuid.hex), + }, + ) + + def create(self, reference: V1Secret): + return self.api.create_namespaced_secret(self.namespace, reference) + + def delete(self, reference: V1Secret): + return self.api.delete_namespaced_secret( + reference.metadata.name, self.namespace + ) + + def retrieve(self) -> V1Secret: + return self.api.read_namespaced_secret( + f"passbook-outpost-{self.outpost.name}-api", self.namespace + ) + + def update(self, current: V1Secret, reference: V1Secret): + return self.api.patch_namespaced_secret( + current.metadata.name, self.namespace, reference + ) diff --git a/passbook/outposts/controllers/k8s/service.py b/passbook/outposts/controllers/k8s/service.py new file mode 100644 index 000000000..21a8cea57 --- /dev/null +++ b/passbook/outposts/controllers/k8s/service.py @@ -0,0 +1,57 @@ +"""Kubernetes Service Reconciler""" +from typing import Dict + +from kubernetes.client import CoreV1Api, V1Service, V1ServicePort, V1ServiceSpec + +from passbook.outposts.controllers.k8s.base import ( + KubernetesObjectReconciler, + NeedsUpdate, +) +from passbook.outposts.models import Outpost + + +class ServiceReconciler(KubernetesObjectReconciler[V1Service]): + """Kubernetes Service Reconciler""" + + deployment_ports: Dict[str, int] + + def __init__(self, outpost: Outpost) -> None: + super().__init__(outpost) + self.api = CoreV1Api() + self.deployment_ports = {} + + def reconcile(self, current: V1Service, reference: V1Service): + if len(current.spec.ports) != len(reference.spec.ports): + raise NeedsUpdate() + for port in reference.spec.ports: + if port not in current.spec.ports: + raise NeedsUpdate() + + def get_reference_object(self) -> V1Service: + """Get deployment object for outpost""" + meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}") + ports = [] + for port_name, port in self.deployment_ports.items(): + ports.append(V1ServicePort(name=port_name, port=port)) + return V1Service( + metadata=meta, + spec=V1ServiceSpec(ports=ports, selector=meta.labels, type="ClusterIP"), + ) + + def create(self, reference: V1Service): + return self.api.create_namespaced_service(self.namespace, reference) + + def delete(self, reference: V1Service): + return self.api.delete_namespaced_service( + reference.metadata.name, self.namespace + ) + + def retrieve(self) -> V1Service: + return self.api.read_namespaced_service( + f"passbook-outpost-{self.outpost.name}", self.namespace + ) + + def update(self, current: V1Service, reference: V1Service): + return self.api.patch_namespaced_service( + current.metadata.name, self.namespace, reference + ) diff --git a/passbook/outposts/controllers/kubernetes.py b/passbook/outposts/controllers/kubernetes.py index aafa55430..ed28bd636 100644 --- a/passbook/outposts/controllers/kubernetes.py +++ b/passbook/outposts/controllers/kubernetes.py @@ -1,156 +1,63 @@ """Kubernetes deployment controller""" -from base64 import b64encode from io import StringIO -from kubernetes.client import ( - V1Container, - V1ContainerPort, - V1Deployment, - V1DeploymentSpec, - V1EnvVar, - V1EnvVarSource, - V1LabelSelector, - V1ObjectMeta, - V1PodSpec, - V1PodTemplateSpec, - V1Secret, - V1SecretKeySelector, - V1Service, - V1ServicePort, - V1ServiceSpec, -) +from kubernetes.config import load_incluster_config, load_kube_config +from kubernetes.config.config_exception import ConfigException from yaml import dump_all -from passbook import __version__ from passbook.outposts.controllers.base import BaseController - - -def b64encode_str(input_string: str) -> str: - """base64 encode string""" - return b64encode(input_string.encode()).decode() +from passbook.outposts.controllers.k8s.deployment import DeploymentReconciler +from passbook.outposts.controllers.k8s.secret import SecretReconciler +from passbook.outposts.controllers.k8s.service import ServiceReconciler class KubernetesController(BaseController): """Manage deployment of outpost in kubernetes""" - image_base = "beryju/passbook" + def __init__(self, outpost_pk: str) -> None: + super().__init__(outpost_pk) + try: + load_incluster_config() + except ConfigException: + load_kube_config() def run(self): """Called by scheduled task to reconcile deployment/service/etc""" - # TODO + namespace = self.outpost.config.kubernetes_namespace + + secret_reconciler = SecretReconciler(self.outpost) + secret_reconciler.namespace = namespace + secret_reconciler.run() + + deployment_reconciler = DeploymentReconciler(self.outpost) + deployment_reconciler.namespace = namespace + deployment_reconciler.deployment_ports = self.deployment_ports + deployment_reconciler.run() + + service_reconciler = ServiceReconciler(self.outpost) + service_reconciler.namespace = namespace + service_reconciler.deployment_ports = self.deployment_ports + service_reconciler.run() def get_static_deployment(self) -> str: + secret_reconciler = SecretReconciler(self.outpost) + secret_reconciler.namespace = "" + + deployment_reconciler = DeploymentReconciler(self.outpost) + deployment_reconciler.namespace = "" + deployment_reconciler.deployment_ports = self.deployment_ports + + service_reconciler = ServiceReconciler(self.outpost) + service_reconciler.namespace = "" + service_reconciler.deployment_ports = self.deployment_ports with StringIO() as _str: dump_all( [ - self.get_deployment_secret().to_dict(), - self.get_deployment().to_dict(), - self.get_service().to_dict(), + secret_reconciler.get_reference_object().to_dict(), + deployment_reconciler.get_reference_object().to_dict(), + service_reconciler.get_reference_object().to_dict(), ], stream=_str, default_flow_style=False, ) return _str.getvalue() - - def get_object_meta(self, **kwargs) -> V1ObjectMeta: - """Get common object metadata""" - return V1ObjectMeta( - namespace="self.instance.namespace", - labels={ - "app.kubernetes.io/name": f"passbook-{self.outpost.type.lower()}", - "app.kubernetes.io/instance": self.outpost.name, - "app.kubernetes.io/version": __version__, - "app.kubernetes.io/managed-by": "passbook.beryju.org", - "passbook.beryju.org/outpost/uuid": self.outpost.uuid.hex, - }, - **kwargs, - ) - - def get_deployment_secret(self) -> V1Secret: - """Get secret with token and passbook host""" - return V1Secret( - api_version="v1", - kind="secret", - type="Opaque", - metadata=self.get_object_meta( - name=f"passbook-outpost-{self.outpost.name}-api" - ), - data={ - "passbook_host": b64encode_str(self.outpost.config.passbook_host), - "passbook_host_insecure": b64encode_str( - str(self.outpost.config.passbook_host_insecure) - ), - "token": b64encode_str(self.outpost.token.token_uuid.hex), - }, - ) - - def get_service(self) -> V1Service: - """Get service object for outpost based on ports defined""" - meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}") - ports = [] - for port_name, port in self.deployment_ports.items(): - ports.append(V1ServicePort(name=port_name, port=port)) - return V1Service( - api_version="v1", - kind="service", - metadata=meta, - spec=V1ServiceSpec(ports=ports, selector=meta.labels, type="ClusterIP"), - ) - - def get_deployment(self) -> V1Deployment: - """Get deployment object for outpost""" - # Generate V1ContainerPort objects - container_ports = [] - for port_name, port in self.deployment_ports.items(): - container_ports.append(V1ContainerPort(container_port=port, name=port_name)) - meta = self.get_object_meta(name=f"passbook-outpost-{self.outpost.name}") - return V1Deployment( - api_version="apps/v1", - kind="deployment", - metadata=meta, - spec=V1DeploymentSpec( - replicas=1, - selector=V1LabelSelector(match_labels=meta.labels), - template=V1PodTemplateSpec( - metadata=V1ObjectMeta(labels=meta.labels), - spec=V1PodSpec( - containers=[ - V1Container( - name=self.outpost.type, - image=f"{self.image_base}-{self.outpost.type}:{__version__}", - ports=container_ports, - env=[ - V1EnvVar( - name="PASSBOOK_HOST", - value_from=V1EnvVarSource( - secret_key_ref=V1SecretKeySelector( - name=f"passbook-outpost-{self.outpost.name}-api", - key="passbook_host", - ) - ), - ), - V1EnvVar( - name="PASSBOOK_TOKEN", - value_from=V1EnvVarSource( - secret_key_ref=V1SecretKeySelector( - name=f"passbook-outpost-{self.outpost.name}-api", - key="token", - ) - ), - ), - V1EnvVar( - name="PASSBOOK_INSECURE", - value_from=V1EnvVarSource( - secret_key_ref=V1SecretKeySelector( - name=f"passbook-outpost-{self.outpost.name}-api", - key="passbook_host_insecure", - ) - ), - ), - ], - ) - ] - ), - ), - ), - ) diff --git a/passbook/outposts/migrations/0008_auto_20201014_1547.py b/passbook/outposts/migrations/0008_auto_20201014_1547.py new file mode 100644 index 000000000..e5e181aa4 --- /dev/null +++ b/passbook/outposts/migrations/0008_auto_20201014_1547.py @@ -0,0 +1,26 @@ +# Generated by Django 3.1.2 on 2020-10-14 15:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("passbook_outposts", "0007_remove_outpost_channels"), + ] + + operations = [ + migrations.AlterField( + model_name="outpost", + name="deployment_type", + field=models.TextField( + choices=[ + ("kubernetes", "Kubernetes"), + ("docker", "Docker"), + ("custom", "Custom"), + ], + default="custom", + help_text="Select between passbook-managed deployment types or a custom deployment.", + ), + ), + ] diff --git a/passbook/outposts/models.py b/passbook/outposts/models.py index 49fffe6ef..2331ddbc3 100644 --- a/passbook/outposts/models.py +++ b/passbook/outposts/models.py @@ -36,6 +36,9 @@ class OutpostConfig: "error_reporting.environment", "customer" ) + kubernetes_replicas: int = field(default=1) + kubernetes_namespace: str = field(default="default") + class OutpostModel(Model): """Base model for providers that need more objects than just themselves""" @@ -58,7 +61,7 @@ class OutpostType(models.TextChoices): class OutpostDeploymentType(models.TextChoices): """Deployment types that are managed through passbook""" - # KUBERNETES = "kubernetes" + KUBERNETES = "kubernetes" DOCKER = "docker" CUSTOM = "custom"