outposts: periodically update state of service connection, show state in UI

passbook/admin/templates/administration/outpost_service_connection/list.html

@@ -50,6 +50,7 @@
                     <th role="columnheader" scope="col">{% trans 'Name' %}</th>
                     <th role="columnheader" scope="col">{% trans 'Type' %}</th>
                     <th role="columnheader" scope="col">{% trans 'Local?' %}</th>
+                    <th role="columnheader" scope="col">{% trans 'Status' %}</th>
                     <th role="cell"></th>
                 </tr>
             </thead>
@@ -69,6 +70,15 @@
                             {{ sc.local|yesno:"Yes,No" }}
                         </span>
                     </td>
+                    <td role="cell">
+                        <span>
+                            {% if sc.state.healthy %}
+                            <i class="fas fa-check pf-m-success"></i> {{ sc.state.version }}
+                            {% else %}
+                            <i class="fas fa-times pf-m-danger"></i> {% trans 'Unhealthy' %}
+                            {% endif %}
+                        </span>
+                    </td>
                     <td>
                         <a class="pf-c-button pf-m-secondary" href="{% url 'passbook_admin:outpost-service-connection-update' pk=sc.pk %}?back={{ request.get_full_path }}">{% trans 'Edit' %}</a>
                         <a class="pf-c-button pf-m-danger" href="{% url 'passbook_admin:outpost-service-connection-delete' pk=sc.pk %}?back={{ request.get_full_path }}">{% trans 'Delete' %}</a>

passbook/outposts/forms.py

@@ -48,6 +48,11 @@ class DockerServiceConnectionForm(forms.ModelForm):
         fields = ["name", "local", "url", "tls"]
         widgets = {
             "name": forms.TextInput,
+            "url": forms.TextInput,
+        }
+        labels = {
+            "url": _("URL"),
+            "tls": _("TLS"),
         }
 
 

passbook/outposts/models.py

@@ -25,6 +25,7 @@ from kubernetes.config.incluster_config import load_incluster_config
 from kubernetes.config.kube_config import load_kube_config, load_kube_config_from_dict
 from model_utils.managers import InheritanceManager
 from packaging.version import LegacyVersion, Version, parse
+from urllib3.exceptions import HTTPError
 
 from passbook import __version__
 from passbook.core.models import Provider, Token, TokenIntents, User
@@ -115,9 +116,9 @@ class OutpostServiceConnection(models.Model):
         """Get state of service connection"""
         state_key = f"outpost_service_connection_{self.pk.hex}"
         state = cache.get(state_key, None)
-        if state:
+        if not state:
             state = self._get_state()
-            cache.set(state_key, state)
+            cache.set(state_key, state, timeout=0)
         return state
 
     def _get_state(self) -> OutpostServiceConnectionState:
@@ -209,7 +210,7 @@ class KubernetesServiceConnection(OutpostServiceConnection):
             return OutpostServiceConnectionState(
                 version=version.git_version, healthy=True
             )
-        except OpenApiException:
+        except (OpenApiException, HTTPError):
             return OutpostServiceConnectionState(version="", healthy=False)
 
     def client(self) -> ApiClient:

passbook/outposts/settings.py

@@ -7,4 +7,9 @@ CELERY_BEAT_SCHEDULE = {
         "schedule": crontab(minute="*/5"),
         "options": {"queue": "passbook_scheduled"},
     },
+    "outposts_service_connection_check": {
+        "task": "passbook.outposts.tasks.outpost_service_connection_monitor",
+        "schedule": crontab(minute=0, hour="*"),
+        "options": {"queue": "passbook_scheduled"},
+    },
 }

passbook/outposts/tasks.py

@@ -3,6 +3,7 @@ from typing import Any
 
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
+from django.core.cache import cache
 from django.db.models.base import Model
 from django.utils.text import slugify
 from structlog import get_logger
@@ -15,6 +16,7 @@ from passbook.outposts.models import (
     KubernetesServiceConnection,
     Outpost,
     OutpostModel,
+    OutpostServiceConnection,
     OutpostState,
     OutpostType,
 )
@@ -32,6 +34,25 @@ def outpost_controller_all():
         outpost_controller.delay(outpost.pk.hex)
 
 
+@CELERY_APP.task()
+def outpost_service_connection_state(state_pk: Any):
+    """Update cached state of a service connection"""
+    connection: OutpostServiceConnection = (
+        OutpostServiceConnection.objects.filter(pk=state_pk).select_subclasses().first()
+    )
+    cache.delete(f"outpost_service_connection_{connection.pk.hex}")
+    _ = connection.state
+
+
+@CELERY_APP.task(bind=True, base=MonitoredTask)
+def outpost_service_connection_monitor(self: MonitoredTask):
+    """Regularly check the state of Outpost Service Connections"""
+    for connection in OutpostServiceConnection.objects.select_subclasses():
+        cache.delete(f"outpost_service_connection_{connection.pk.hex}")
+        _ = connection.state
+    self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL))
+
+
 @CELERY_APP.task(bind=True, base=MonitoredTask)
 def outpost_controller(self: MonitoredTask, outpost_pk: str):
     """Create/update/monitor the deployment of an Outpost"""
@@ -92,6 +113,10 @@ def outpost_post_save(model_class: str, model_pk: Any):
         outpost_send_update(instance)
         return
 
+    if isinstance(instance, OutpostServiceConnection):
+        LOGGER.debug("triggering ServiceConnection state update", instance=instance)
+        outpost_service_connection_state.delay(instance.pk)
+
     for field in instance._meta.get_fields():
         # Each field is checked if it has a `related_model` attribute (when ForeginKeys or M2Ms)
         # are used, and if it has a value