root: implement monitored tasks

lifecycle/bootstrap.sh

@@ -4,7 +4,7 @@ printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap",
 if [[ "$1" == "server" ]]; then
     gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
 elif [[ "$1" == "worker" ]]; then
-    celery -A passbook.root.celery worker --autoscale 10,3 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled
+    celery -A passbook.root.celery worker --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled
 elif [[ "$1" == "migrate" ]]; then
     # Run system migrations first, run normal migrations after
     python -m lifecycle.migrate

passbook/admin/tasks.py

@@ -3,6 +3,7 @@ from django.core.cache import cache
 from requests import RequestException, get
 from structlog import get_logger
 
+from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
 from passbook.root.celery import CELERY_APP
 
 LOGGER = get_logger()
@@ -10,8 +11,8 @@ VERSION_CACHE_KEY = "passbook_latest_version"
 VERSION_CACHE_TIMEOUT = 2 * 60 * 60  # 2 hours
 
 
-@CELERY_APP.task()
-def update_latest_version():
+@CELERY_APP.task(bind=True, base=MonitoredTask)
+def update_latest_version(self: MonitoredTask):
     """Update latest version info"""
     try:
         data = get(
@@ -19,5 +20,11 @@ def update_latest_version():
         ).json()
         tag_name = data.get("tag_name")
         cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], VERSION_CACHE_TIMEOUT)
-    except (RequestException, IndexError):
+        self.set_status(
+            TaskResult(
+                TaskResultStatus.SUCCESSFUL, ["Successfully updated latest Version"]
+            )
+        )
+    except (RequestException, IndexError) as exc:
         cache.set(VERSION_CACHE_KEY, "0.0.0", VERSION_CACHE_TIMEOUT)
+        self.set_status(TaskResult(TaskResultStatus.ERROR, [str(exc)]))

passbook/admin/templates/administration/base.html

@@ -146,6 +146,12 @@
                         {% trans 'Groups' %}
                     </a>
                 </li>
+                <li class="pf-c-nav__item">
+                    <a href="{% url 'passbook_admin:tasks' %}"
+                        class="pf-c-nav__link {% is_active 'passbook_admin:tasks' %}">
+                        {% trans 'System Tasks' %}
+                    </a>
+                </li>
             </ul>
         </nav>
     </div>

passbook/admin/templates/administration/task/list.html

@@ -0,0 +1,71 @@
+{% extends "administration/base.html" %}
+
+{% load i18n %}
+{% load humanize %}
+{% load passbook_utils %}
+
+{% block content %}
+<section class="pf-c-page__main-section pf-m-light">
+    <div class="pf-c-content">
+        <h1>
+            <i class="fas fa-key"></i>
+            {% trans 'System Tasks' %}
+        </h1>
+        <p>{% trans "Background tasks." %}</p>
+    </div>
+</section>
+<section class="pf-c-page__main-section pf-m-no-padding-mobile">
+    <div class="pf-c-card">
+        <table class="pf-c-table pf-m-compact pf-m-grid-xl" role="grid">
+            <thead>
+                <tr role="row">
+                    <th role="columnheader" scope="col">{% trans 'Identifier' %}</th>
+                    <th role="columnheader" scope="col">{% trans 'Description' %}</th>
+                    <th role="columnheader" scope="col">{% trans 'Last Status' %}</th>
+                    <th role="columnheader" scope="col">{% trans 'Status' %}</th>
+                    <th role="columnheader" scope="col">{% trans 'Messages' %}</th>
+                </tr>
+            </thead>
+            <tbody role="rowgroup">
+                {% for key, task in object_list.items %}
+                <tr role="row">
+                    <th role="columnheader">
+                        <pre>{{ key }}</pre>
+                    </th>
+                    <td role="cell">
+                        <span>
+                            {{ task.task_description }}
+                        </span>
+                    </td>
+                    <td role="cell">
+                        <span>
+                            {{ task.finish_timestamp|naturaltime }}
+                        </span>
+                    </td>
+                    <td role="cell">
+                        <span>
+                            {% if task.result.status == task_successful %}
+                            <i class="fas fa-check pf-m-success"></i> {% trans 'Successful' %}
+                            {% elif task.result.status == task_warning %}
+                            <i class="fas fa-exclamation-triangle pf-m-warning"></i> {% trans 'Warning' %}
+                            {% elif task.result.status == task_error %}
+                            <i class="fas fa-times pf-m-danger"></i> {% trans 'Error' %}
+                            {% else %}
+                            <i class="fas fa-question-circle"></i> {% trans 'Unknown' %}
+                            {% endif %}
+                        </span>
+                    </td>
+                    <td>
+                        {% for message in task.result.messages %}
+                        <div>
+                            {{ message }}
+                        </div>
+                        {% endfor %}
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</section>
+{% endblock %}

passbook/admin/urls.py

@@ -17,6 +17,7 @@ from passbook.admin.views import (
     stages_bindings,
     stages_invitations,
     stages_prompts,
+    tasks,
     tokens,
     users,
 )
@@ -311,4 +312,10 @@ urlpatterns = [
         outposts.OutpostDeleteView.as_view(),
         name="outpost-delete",
     ),
+    # Tasks
+    path(
+        "tasks/",
+        tasks.TaskListView.as_view(),
+        name="tasks",
+    ),
 ]

passbook/admin/views/tasks.py

@@ -0,0 +1,22 @@
+"""passbook Tasks List"""
+from typing import Any, Dict
+
+from django.core.cache import cache
+from django.views.generic.base import TemplateView
+
+from passbook.admin.mixins import AdminRequiredMixin
+from passbook.lib.tasks import TaskResultStatus
+
+
+class TaskListView(AdminRequiredMixin, TemplateView):
+    """Show list of all background tasks"""
+
+    template_name = "administration/task/list.html"
+
+    def get_context_data(self, **kwargs: Any) -> Dict[str, Any]:
+        kwargs = super().get_context_data(**kwargs)
+        kwargs["object_list"] = cache.get_many(cache.keys("task_*"))
+        kwargs["task_successful"] = TaskResultStatus.SUCCESSFUL
+        kwargs["task_warning"] = TaskResultStatus.WARNING
+        kwargs["task_error"] = TaskResultStatus.ERROR
+        return kwargs

passbook/core/tasks.py

@@ -3,14 +3,16 @@ from django.utils.timezone import now
 from structlog import get_logger
 
 from passbook.core.models import ExpiringModel
+from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
 from passbook.root.celery import CELERY_APP
 
 LOGGER = get_logger()
 
 
-@CELERY_APP.task()
-def clean_expired_models():
+@CELERY_APP.task(bind=True, base=MonitoredTask)
+def clean_expired_models(self: MonitoredTask):
     """Remove expired objects"""
+    messages = []
     for cls in ExpiringModel.__subclasses__():
         cls: ExpiringModel
         amount, _ = (
@@ -20,3 +22,5 @@ def clean_expired_models():
             .delete()
         )
         LOGGER.debug("Deleted expired models", model=cls, amount=amount)
+        messages.append(f"Deleted {amount} expired {cls._meta.verbose_name_plural}")
+    self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL, messages))

passbook/core/tests/test_tasks.py

@@ -14,5 +14,5 @@ class TestTasks(TestCase):
         """Test Token cleanup task"""
         Token.objects.create(expires=now(), user=get_anonymous_user())
         self.assertEqual(Token.objects.all().count(), 1)
-        clean_expired_models()
+        clean_expired_models.delay()
         self.assertEqual(Token.objects.all().count(), 0)

passbook/lib/tasks.py

@@ -0,0 +1,88 @@
+"""Monitored tasks"""
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import List, Optional
+
+from celery import Task
+from django.core.cache import cache
+
+
+class TaskResultStatus(Enum):
+    """Possible states of tasks"""
+
+    SUCCESSFUL = 1
+    WARNING = 2
+    ERROR = 4
+
+
+@dataclass
+class TaskResult:
+    """Result of a task run, this class is created by the task itself
+    and used by self.set_status"""
+
+    status: TaskResultStatus
+
+    messages: List[str] = field(default_factory=list)
+
+    error: Optional[Exception] = field(default=None)
+
+    # Optional UID used in cache for tasks that run in different instances
+    uid: Optional[str] = field(default=None)
+
+
+@dataclass
+class TaskInfo:
+    """Info about a task run"""
+
+    task_name: str
+    finish_timestamp: datetime
+
+    result: TaskResult
+
+    task_description: Optional[str] = field(default=None)
+
+    def save(self):
+        """Save task into cache"""
+        key = f"task_{self.task_name}"
+        if self.result.uid:
+            key += f"_{self.result.uid}"
+            self.task_name += f"_{self.result.uid}"
+        cache.set(key, self)
+
+
+class MonitoredTask(Task):
+    """Task which can save its state to the cache"""
+
+    _result: TaskResult
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._result = TaskResult(status=TaskResultStatus.ERROR, messages=[])
+
+    def set_status(self, result: TaskResult):
+        """Set result for current run, will overwrite previous result."""
+        self._result = result
+
+    # pylint: disable=too-many-arguments
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        TaskInfo(
+            task_name=self.__name__,
+            task_description=self.__doc__,
+            finish_timestamp=datetime.now(),
+            result=self._result,
+        ).save()
+        return super().after_return(status, retval, task_id, args, kwargs, einfo=einfo)
+
+    # pylint: disable=too-many-arguments
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        TaskInfo(
+            task_name=self.__name__,
+            task_description=self.__doc__,
+            finish_timestamp=datetime.now(),
+            result=self._result,
+        ).save()
+        return super().on_failure(exc, task_id, args, kwargs, einfo=einfo)
+
+    def run(self, *args, **kwargs):
+        raise NotImplementedError

passbook/providers/proxy/tests.py

@@ -7,9 +7,7 @@ from django.test import TestCase
 
 from passbook.flows.models import Flow
 from passbook.outposts.models import Outpost, OutpostDeploymentType, OutpostType
-from passbook.providers.proxy.controllers.kubernetes import (
-    ProxyKubernetesController,
-)
+from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController
 from passbook.providers.proxy.models import ProxyProvider
 
 

passbook/stages/consent/tests.py

@@ -127,7 +127,7 @@ class TestConsentStage(TestCase):
             ).exists()
         )
         sleep(1)
-        clean_expired_models()
+        clean_expired_models.delay()
         self.assertFalse(
             UserConsent.objects.filter(
                 user=self.user, application=self.application

passbook/static/static/passbook/passbook.css

@@ -52,6 +52,9 @@
 .pf-m-success {
     color: var(--pf-global--success-color--100);
 }
+.pf-m-warning {
+    color: var(--pf-global--warning-color--100);
+}
 .pf-m-danger {
     color: var(--pf-global--danger-color--100);
 }