From 5cb7f0794ec85664d482b5be6fc585e5d9ea7a61 Mon Sep 17 00:00:00 2001 From: Jens Langhammer Date: Sun, 8 Nov 2020 21:31:27 +0100 Subject: [PATCH] outposts: periodically update state of service connection, show state in UI --- .../outpost_service_connection/list.html | 10 ++++++++ passbook/outposts/forms.py | 5 ++++ passbook/outposts/models.py | 7 +++--- passbook/outposts/settings.py | 5 ++++ passbook/outposts/tasks.py | 25 +++++++++++++++++++ 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/passbook/admin/templates/administration/outpost_service_connection/list.html b/passbook/admin/templates/administration/outpost_service_connection/list.html index fdaf45c20..8b7d64798 100644 --- a/passbook/admin/templates/administration/outpost_service_connection/list.html +++ b/passbook/admin/templates/administration/outpost_service_connection/list.html @@ -50,6 +50,7 @@ {% trans 'Name' %} {% trans 'Type' %} {% trans 'Local?' %} + {% trans 'Status' %} @@ -69,6 +70,15 @@ {{ sc.local|yesno:"Yes,No" }} + + + {% if sc.state.healthy %} + {{ sc.state.version }} + {% else %} + {% trans 'Unhealthy' %} + {% endif %} + + {% trans 'Edit' %} {% trans 'Delete' %} diff --git a/passbook/outposts/forms.py b/passbook/outposts/forms.py index a0de27461..81dd7f31a 100644 --- a/passbook/outposts/forms.py +++ b/passbook/outposts/forms.py @@ -48,6 +48,11 @@ class DockerServiceConnectionForm(forms.ModelForm): fields = ["name", "local", "url", "tls"] widgets = { "name": forms.TextInput, + "url": forms.TextInput, + } + labels = { + "url": _("URL"), + "tls": _("TLS"), } diff --git a/passbook/outposts/models.py b/passbook/outposts/models.py index 26fe22456..b1675bff8 100644 --- a/passbook/outposts/models.py +++ b/passbook/outposts/models.py @@ -25,6 +25,7 @@ from kubernetes.config.incluster_config import load_incluster_config from kubernetes.config.kube_config import load_kube_config, load_kube_config_from_dict from model_utils.managers import InheritanceManager from packaging.version import LegacyVersion, Version, parse +from urllib3.exceptions import HTTPError from passbook import __version__ from passbook.core.models import Provider, Token, TokenIntents, User @@ -115,9 +116,9 @@ class OutpostServiceConnection(models.Model): """Get state of service connection""" state_key = f"outpost_service_connection_{self.pk.hex}" state = cache.get(state_key, None) - if state: + if not state: state = self._get_state() - cache.set(state_key, state) + cache.set(state_key, state, timeout=0) return state def _get_state(self) -> OutpostServiceConnectionState: @@ -209,7 +210,7 @@ class KubernetesServiceConnection(OutpostServiceConnection): return OutpostServiceConnectionState( version=version.git_version, healthy=True ) - except OpenApiException: + except (OpenApiException, HTTPError): return OutpostServiceConnectionState(version="", healthy=False) def client(self) -> ApiClient: diff --git a/passbook/outposts/settings.py b/passbook/outposts/settings.py index 0a7d803a6..9f4a08d56 100644 --- a/passbook/outposts/settings.py +++ b/passbook/outposts/settings.py @@ -7,4 +7,9 @@ CELERY_BEAT_SCHEDULE = { "schedule": crontab(minute="*/5"), "options": {"queue": "passbook_scheduled"}, }, + "outposts_service_connection_check": { + "task": "passbook.outposts.tasks.outpost_service_connection_monitor", + "schedule": crontab(minute=0, hour="*"), + "options": {"queue": "passbook_scheduled"}, + }, } diff --git a/passbook/outposts/tasks.py b/passbook/outposts/tasks.py index 8a368d626..dca2a748f 100644 --- a/passbook/outposts/tasks.py +++ b/passbook/outposts/tasks.py @@ -3,6 +3,7 @@ from typing import Any from asgiref.sync import async_to_sync from channels.layers import get_channel_layer +from django.core.cache import cache from django.db.models.base import Model from django.utils.text import slugify from structlog import get_logger @@ -15,6 +16,7 @@ from passbook.outposts.models import ( KubernetesServiceConnection, Outpost, OutpostModel, + OutpostServiceConnection, OutpostState, OutpostType, ) @@ -32,6 +34,25 @@ def outpost_controller_all(): outpost_controller.delay(outpost.pk.hex) +@CELERY_APP.task() +def outpost_service_connection_state(state_pk: Any): + """Update cached state of a service connection""" + connection: OutpostServiceConnection = ( + OutpostServiceConnection.objects.filter(pk=state_pk).select_subclasses().first() + ) + cache.delete(f"outpost_service_connection_{connection.pk.hex}") + _ = connection.state + + +@CELERY_APP.task(bind=True, base=MonitoredTask) +def outpost_service_connection_monitor(self: MonitoredTask): + """Regularly check the state of Outpost Service Connections""" + for connection in OutpostServiceConnection.objects.select_subclasses(): + cache.delete(f"outpost_service_connection_{connection.pk.hex}") + _ = connection.state + self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL)) + + @CELERY_APP.task(bind=True, base=MonitoredTask) def outpost_controller(self: MonitoredTask, outpost_pk: str): """Create/update/monitor the deployment of an Outpost""" @@ -92,6 +113,10 @@ def outpost_post_save(model_class: str, model_pk: Any): outpost_send_update(instance) return + if isinstance(instance, OutpostServiceConnection): + LOGGER.debug("triggering ServiceConnection state update", instance=instance) + outpost_service_connection_state.delay(instance.pk) + for field in instance._meta.get_fields(): # Each field is checked if it has a `related_model` attribute (when ForeginKeys or M2Ms) # are used, and if it has a value