outpost: enable docker controller

docker-compose.yml

@@ -52,6 +52,7 @@ services:
       PASSBOOK_POSTGRESQL__PASSWORD: ${PG_PASS}
     volumes:
       - ./backups:/backups
+      - /var/run/docker.socket:/var/run/docker.socket
     env_file:
       - .env
   static:

passbook/core/models.py

@@ -313,9 +313,7 @@ class Token(ExpiringModel):
     description = models.TextField(default="", blank=True)
 
     def __str__(self):
-        return (
+        return f"Token {self.identifier} (expires={self.expires})"
-            f"Token {self.identifier} (expires={self.expires})"
-        )
 
     class Meta:
 

passbook/outposts/controllers/docker.py

@@ -1,4 +1,6 @@
 """Docker controller"""
+from typing import Dict, Tuple
+
 from docker import DockerClient, from_env
 from docker.errors import NotFound
 from docker.models.containers import Container
@@ -21,36 +23,80 @@ class DockerController(BaseController):
         super().__init__(outpost_pk)
         self.client = from_env()
 
-    def _get_container(self) -> Container:
+    def _get_env(self) -> Dict[str, str]:
+        return {
+            "PASSBOOK_HOST": self.outpost.config.passbook_host,
+            "PASSBOOK_INSECURE": str(self.outpost.config.passbook_host_insecure),
+            "PASSBOOK_TOKEN": self.outpost.token.token_uuid.hex,
+        }
+
+    def _comp_env(self, container: Container) -> bool:
+        """Check if container's env is equal to what we would set. Return true if container needs
+        to be rebuilt."""
+        should_be = self._get_env()
+        container_env = container.attrs.get("Config", {}).get("Env", {})
+        for key, expected_value in should_be.items():
+            if key not in container_env:
+                continue
+            if container_env[key] != expected_value:
+                return True
+        return False
+
+    def _get_container(self) -> Tuple[Container, bool]:
         container_name = f"passbook-proxy-{self.outpost.uuid.hex}"
         try:
-            return self.client.containers.get(container_name)
+            return self.client.containers.get(container_name), False
         except NotFound:
-            return self.client.containers.create(
+            self.logger.info("Container does not exist, creating")
-                image=f"{self.image_base}-{self.outpost.type}:{__version__}",
+            image_name = f"{self.image_base}-{self.outpost.type}:{__version__}"
+            self.client.images.pull(image_name)
+            return (
+                self.client.containers.create(
+                    image=image_name,
                     name=f"passbook-proxy-{self.outpost.uuid.hex}",
                     detach=True,
                     ports={x: x for _, x in self.deployment_ports.items()},
-                environment={
+                    environment=self._get_env(),
-                    "PASSBOOK_HOST": self.outpost.config.passbook_host,
-                    "PASSBOOK_INSECURE": str(
-                        self.outpost.config.passbook_host_insecure
                 ),
-                    "PASSBOOK_TOKEN": self.outpost.token.token_uuid.hex,
+                True,
-                },
             )
 
     def run(self):
-        container = self._get_container()
+        container, has_been_created = self._get_container()
+        if has_been_created:
+            return None
         # Check if the container is out of date, delete it and retry
         if len(container.image.tags) > 0:
-            tag: str = container.iamge.tags[0]
+            tag: str = container.image.tags[0]
             _, _, version = tag.partition(":")
             if version != __version__:
+                self.logger.info(
+                    "Container has mismatched version, re-creating...",
+                    has=version,
+                    should=__version__,
+                )
                 container.kill()
                 container.remove(force=True)
                 return self.run()
+        # Check that container values match our values
+        if self._comp_env(container):
+            self.logger.info("Container has outdated config, re-creating...")
+            container.kill()
+            container.remove(force=True)
+            return self.run()
+        # Check that container is healthy
+        if (
+            container.status == "running"
+            and container.attrs.get("State", {}).get("Health", {}).get("Status", "")
+            != "healthy"
+        ):
+            # At this point we know the config is correct, but the container isn't healthy,
+            # so we just restart it with the same config
+            self.logger.info("Container is unhealthy, restarting...")
+            container.restart()
+        # Check that container is running
         if container.status != "running":
+            self.logger.info("Container is not running, restarting...")
             container.start()
         return None
 

passbook/outposts/settings.py

@@ -1,10 +1,10 @@
 """Outposts Settings"""
-# from celery.schedules import crontab
+from celery.schedules import crontab
 
-# CELERY_BEAT_SCHEDULE = {
+CELERY_BEAT_SCHEDULE = {
-#     "outposts_k8s": {
+    "outposts_k8s": {
-#         "task": "passbook.outposts.tasks.outpost_k8s_controller",
+        "task": "passbook.outposts.tasks.outpost_controller",
-#         "schedule": crontab(minute="*/5"),  # Run every 5 minutes
+        "schedule": crontab(minute="*/5"),  # Run every 5 minutes
-#         "options": {"queue": "passbook_scheduled"},
+        "options": {"queue": "passbook_scheduled"},
-#     }
+    }
-# }
+}