root: implement monitored tasks
This commit is contained in:
parent
17060238f0
commit
91ce7f7363
|
@ -4,7 +4,7 @@ printf '{"event": "Bootstrap completed", "level": "info", "logger": "bootstrap",
|
||||||
if [[ "$1" == "server" ]]; then
|
if [[ "$1" == "server" ]]; then
|
||||||
gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
|
gunicorn -c /lifecycle/gunicorn.conf.py passbook.root.asgi:application
|
||||||
elif [[ "$1" == "worker" ]]; then
|
elif [[ "$1" == "worker" ]]; then
|
||||||
celery -A passbook.root.celery worker --autoscale 10,3 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled
|
celery -A passbook.root.celery worker --autoscale 3,1 -E -B -s /tmp/celerybeat-schedule -Q passbook,passbook_scheduled
|
||||||
elif [[ "$1" == "migrate" ]]; then
|
elif [[ "$1" == "migrate" ]]; then
|
||||||
# Run system migrations first, run normal migrations after
|
# Run system migrations first, run normal migrations after
|
||||||
python -m lifecycle.migrate
|
python -m lifecycle.migrate
|
||||||
|
|
|
@ -3,6 +3,7 @@ from django.core.cache import cache
|
||||||
from requests import RequestException, get
|
from requests import RequestException, get
|
||||||
from structlog import get_logger
|
from structlog import get_logger
|
||||||
|
|
||||||
|
from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
|
||||||
from passbook.root.celery import CELERY_APP
|
from passbook.root.celery import CELERY_APP
|
||||||
|
|
||||||
LOGGER = get_logger()
|
LOGGER = get_logger()
|
||||||
|
@ -10,8 +11,8 @@ VERSION_CACHE_KEY = "passbook_latest_version"
|
||||||
VERSION_CACHE_TIMEOUT = 2 * 60 * 60 # 2 hours
|
VERSION_CACHE_TIMEOUT = 2 * 60 * 60 # 2 hours
|
||||||
|
|
||||||
|
|
||||||
@CELERY_APP.task()
|
@CELERY_APP.task(bind=True, base=MonitoredTask)
|
||||||
def update_latest_version():
|
def update_latest_version(self: MonitoredTask):
|
||||||
"""Update latest version info"""
|
"""Update latest version info"""
|
||||||
try:
|
try:
|
||||||
data = get(
|
data = get(
|
||||||
|
@ -19,5 +20,11 @@ def update_latest_version():
|
||||||
).json()
|
).json()
|
||||||
tag_name = data.get("tag_name")
|
tag_name = data.get("tag_name")
|
||||||
cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], VERSION_CACHE_TIMEOUT)
|
cache.set(VERSION_CACHE_KEY, tag_name.split("/")[1], VERSION_CACHE_TIMEOUT)
|
||||||
except (RequestException, IndexError):
|
self.set_status(
|
||||||
|
TaskResult(
|
||||||
|
TaskResultStatus.SUCCESSFUL, ["Successfully updated latest Version"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except (RequestException, IndexError) as exc:
|
||||||
cache.set(VERSION_CACHE_KEY, "0.0.0", VERSION_CACHE_TIMEOUT)
|
cache.set(VERSION_CACHE_KEY, "0.0.0", VERSION_CACHE_TIMEOUT)
|
||||||
|
self.set_status(TaskResult(TaskResultStatus.ERROR, [str(exc)]))
|
||||||
|
|
|
@ -146,6 +146,12 @@
|
||||||
{% trans 'Groups' %}
|
{% trans 'Groups' %}
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
|
<li class="pf-c-nav__item">
|
||||||
|
<a href="{% url 'passbook_admin:tasks' %}"
|
||||||
|
class="pf-c-nav__link {% is_active 'passbook_admin:tasks' %}">
|
||||||
|
{% trans 'System Tasks' %}
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
</nav>
|
</nav>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
{% extends "administration/base.html" %}
|
||||||
|
|
||||||
|
{% load i18n %}
|
||||||
|
{% load humanize %}
|
||||||
|
{% load passbook_utils %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<section class="pf-c-page__main-section pf-m-light">
|
||||||
|
<div class="pf-c-content">
|
||||||
|
<h1>
|
||||||
|
<i class="fas fa-key"></i>
|
||||||
|
{% trans 'System Tasks' %}
|
||||||
|
</h1>
|
||||||
|
<p>{% trans "Background tasks." %}</p>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
<section class="pf-c-page__main-section pf-m-no-padding-mobile">
|
||||||
|
<div class="pf-c-card">
|
||||||
|
<table class="pf-c-table pf-m-compact pf-m-grid-xl" role="grid">
|
||||||
|
<thead>
|
||||||
|
<tr role="row">
|
||||||
|
<th role="columnheader" scope="col">{% trans 'Identifier' %}</th>
|
||||||
|
<th role="columnheader" scope="col">{% trans 'Description' %}</th>
|
||||||
|
<th role="columnheader" scope="col">{% trans 'Last Status' %}</th>
|
||||||
|
<th role="columnheader" scope="col">{% trans 'Status' %}</th>
|
||||||
|
<th role="columnheader" scope="col">{% trans 'Messages' %}</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody role="rowgroup">
|
||||||
|
{% for key, task in object_list.items %}
|
||||||
|
<tr role="row">
|
||||||
|
<th role="columnheader">
|
||||||
|
<pre>{{ key }}</pre>
|
||||||
|
</th>
|
||||||
|
<td role="cell">
|
||||||
|
<span>
|
||||||
|
{{ task.task_description }}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td role="cell">
|
||||||
|
<span>
|
||||||
|
{{ task.finish_timestamp|naturaltime }}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td role="cell">
|
||||||
|
<span>
|
||||||
|
{% if task.result.status == task_successful %}
|
||||||
|
<i class="fas fa-check pf-m-success"></i> {% trans 'Successful' %}
|
||||||
|
{% elif task.result.status == task_warning %}
|
||||||
|
<i class="fas fa-exclamation-triangle pf-m-warning"></i> {% trans 'Warning' %}
|
||||||
|
{% elif task.result.status == task_error %}
|
||||||
|
<i class="fas fa-times pf-m-danger"></i> {% trans 'Error' %}
|
||||||
|
{% else %}
|
||||||
|
<i class="fas fa-question-circle"></i> {% trans 'Unknown' %}
|
||||||
|
{% endif %}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% for message in task.result.messages %}
|
||||||
|
<div>
|
||||||
|
{{ message }}
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
{% endblock %}
|
|
@ -17,6 +17,7 @@ from passbook.admin.views import (
|
||||||
stages_bindings,
|
stages_bindings,
|
||||||
stages_invitations,
|
stages_invitations,
|
||||||
stages_prompts,
|
stages_prompts,
|
||||||
|
tasks,
|
||||||
tokens,
|
tokens,
|
||||||
users,
|
users,
|
||||||
)
|
)
|
||||||
|
@ -311,4 +312,10 @@ urlpatterns = [
|
||||||
outposts.OutpostDeleteView.as_view(),
|
outposts.OutpostDeleteView.as_view(),
|
||||||
name="outpost-delete",
|
name="outpost-delete",
|
||||||
),
|
),
|
||||||
|
# Tasks
|
||||||
|
path(
|
||||||
|
"tasks/",
|
||||||
|
tasks.TaskListView.as_view(),
|
||||||
|
name="tasks",
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
"""passbook Tasks List"""
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
from django.core.cache import cache
|
||||||
|
from django.views.generic.base import TemplateView
|
||||||
|
|
||||||
|
from passbook.admin.mixins import AdminRequiredMixin
|
||||||
|
from passbook.lib.tasks import TaskResultStatus
|
||||||
|
|
||||||
|
|
||||||
|
class TaskListView(AdminRequiredMixin, TemplateView):
|
||||||
|
"""Show list of all background tasks"""
|
||||||
|
|
||||||
|
template_name = "administration/task/list.html"
|
||||||
|
|
||||||
|
def get_context_data(self, **kwargs: Any) -> Dict[str, Any]:
|
||||||
|
kwargs = super().get_context_data(**kwargs)
|
||||||
|
kwargs["object_list"] = cache.get_many(cache.keys("task_*"))
|
||||||
|
kwargs["task_successful"] = TaskResultStatus.SUCCESSFUL
|
||||||
|
kwargs["task_warning"] = TaskResultStatus.WARNING
|
||||||
|
kwargs["task_error"] = TaskResultStatus.ERROR
|
||||||
|
return kwargs
|
|
@ -3,14 +3,16 @@ from django.utils.timezone import now
|
||||||
from structlog import get_logger
|
from structlog import get_logger
|
||||||
|
|
||||||
from passbook.core.models import ExpiringModel
|
from passbook.core.models import ExpiringModel
|
||||||
|
from passbook.lib.tasks import MonitoredTask, TaskResult, TaskResultStatus
|
||||||
from passbook.root.celery import CELERY_APP
|
from passbook.root.celery import CELERY_APP
|
||||||
|
|
||||||
LOGGER = get_logger()
|
LOGGER = get_logger()
|
||||||
|
|
||||||
|
|
||||||
@CELERY_APP.task()
|
@CELERY_APP.task(bind=True, base=MonitoredTask)
|
||||||
def clean_expired_models():
|
def clean_expired_models(self: MonitoredTask):
|
||||||
"""Remove expired objects"""
|
"""Remove expired objects"""
|
||||||
|
messages = []
|
||||||
for cls in ExpiringModel.__subclasses__():
|
for cls in ExpiringModel.__subclasses__():
|
||||||
cls: ExpiringModel
|
cls: ExpiringModel
|
||||||
amount, _ = (
|
amount, _ = (
|
||||||
|
@ -20,3 +22,5 @@ def clean_expired_models():
|
||||||
.delete()
|
.delete()
|
||||||
)
|
)
|
||||||
LOGGER.debug("Deleted expired models", model=cls, amount=amount)
|
LOGGER.debug("Deleted expired models", model=cls, amount=amount)
|
||||||
|
messages.append(f"Deleted {amount} expired {cls._meta.verbose_name_plural}")
|
||||||
|
self.set_status(TaskResult(TaskResultStatus.SUCCESSFUL, messages))
|
||||||
|
|
|
@ -14,5 +14,5 @@ class TestTasks(TestCase):
|
||||||
"""Test Token cleanup task"""
|
"""Test Token cleanup task"""
|
||||||
Token.objects.create(expires=now(), user=get_anonymous_user())
|
Token.objects.create(expires=now(), user=get_anonymous_user())
|
||||||
self.assertEqual(Token.objects.all().count(), 1)
|
self.assertEqual(Token.objects.all().count(), 1)
|
||||||
clean_expired_models()
|
clean_expired_models.delay()
|
||||||
self.assertEqual(Token.objects.all().count(), 0)
|
self.assertEqual(Token.objects.all().count(), 0)
|
||||||
|
|
|
@ -0,0 +1,88 @@
|
||||||
|
"""Monitored tasks"""
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
from enum import Enum
|
||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from celery import Task
|
||||||
|
from django.core.cache import cache
|
||||||
|
|
||||||
|
|
||||||
|
class TaskResultStatus(Enum):
|
||||||
|
"""Possible states of tasks"""
|
||||||
|
|
||||||
|
SUCCESSFUL = 1
|
||||||
|
WARNING = 2
|
||||||
|
ERROR = 4
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TaskResult:
|
||||||
|
"""Result of a task run, this class is created by the task itself
|
||||||
|
and used by self.set_status"""
|
||||||
|
|
||||||
|
status: TaskResultStatus
|
||||||
|
|
||||||
|
messages: List[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
error: Optional[Exception] = field(default=None)
|
||||||
|
|
||||||
|
# Optional UID used in cache for tasks that run in different instances
|
||||||
|
uid: Optional[str] = field(default=None)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TaskInfo:
|
||||||
|
"""Info about a task run"""
|
||||||
|
|
||||||
|
task_name: str
|
||||||
|
finish_timestamp: datetime
|
||||||
|
|
||||||
|
result: TaskResult
|
||||||
|
|
||||||
|
task_description: Optional[str] = field(default=None)
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
"""Save task into cache"""
|
||||||
|
key = f"task_{self.task_name}"
|
||||||
|
if self.result.uid:
|
||||||
|
key += f"_{self.result.uid}"
|
||||||
|
self.task_name += f"_{self.result.uid}"
|
||||||
|
cache.set(key, self)
|
||||||
|
|
||||||
|
|
||||||
|
class MonitoredTask(Task):
|
||||||
|
"""Task which can save its state to the cache"""
|
||||||
|
|
||||||
|
_result: TaskResult
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs) -> None:
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self._result = TaskResult(status=TaskResultStatus.ERROR, messages=[])
|
||||||
|
|
||||||
|
def set_status(self, result: TaskResult):
|
||||||
|
"""Set result for current run, will overwrite previous result."""
|
||||||
|
self._result = result
|
||||||
|
|
||||||
|
# pylint: disable=too-many-arguments
|
||||||
|
def after_return(self, status, retval, task_id, args, kwargs, einfo):
|
||||||
|
TaskInfo(
|
||||||
|
task_name=self.__name__,
|
||||||
|
task_description=self.__doc__,
|
||||||
|
finish_timestamp=datetime.now(),
|
||||||
|
result=self._result,
|
||||||
|
).save()
|
||||||
|
return super().after_return(status, retval, task_id, args, kwargs, einfo=einfo)
|
||||||
|
|
||||||
|
# pylint: disable=too-many-arguments
|
||||||
|
def on_failure(self, exc, task_id, args, kwargs, einfo):
|
||||||
|
TaskInfo(
|
||||||
|
task_name=self.__name__,
|
||||||
|
task_description=self.__doc__,
|
||||||
|
finish_timestamp=datetime.now(),
|
||||||
|
result=self._result,
|
||||||
|
).save()
|
||||||
|
return super().on_failure(exc, task_id, args, kwargs, einfo=einfo)
|
||||||
|
|
||||||
|
def run(self, *args, **kwargs):
|
||||||
|
raise NotImplementedError
|
|
@ -7,9 +7,7 @@ from django.test import TestCase
|
||||||
|
|
||||||
from passbook.flows.models import Flow
|
from passbook.flows.models import Flow
|
||||||
from passbook.outposts.models import Outpost, OutpostDeploymentType, OutpostType
|
from passbook.outposts.models import Outpost, OutpostDeploymentType, OutpostType
|
||||||
from passbook.providers.proxy.controllers.kubernetes import (
|
from passbook.providers.proxy.controllers.kubernetes import ProxyKubernetesController
|
||||||
ProxyKubernetesController,
|
|
||||||
)
|
|
||||||
from passbook.providers.proxy.models import ProxyProvider
|
from passbook.providers.proxy.models import ProxyProvider
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -127,7 +127,7 @@ class TestConsentStage(TestCase):
|
||||||
).exists()
|
).exists()
|
||||||
)
|
)
|
||||||
sleep(1)
|
sleep(1)
|
||||||
clean_expired_models()
|
clean_expired_models.delay()
|
||||||
self.assertFalse(
|
self.assertFalse(
|
||||||
UserConsent.objects.filter(
|
UserConsent.objects.filter(
|
||||||
user=self.user, application=self.application
|
user=self.user, application=self.application
|
||||||
|
|
|
@ -52,6 +52,9 @@
|
||||||
.pf-m-success {
|
.pf-m-success {
|
||||||
color: var(--pf-global--success-color--100);
|
color: var(--pf-global--success-color--100);
|
||||||
}
|
}
|
||||||
|
.pf-m-warning {
|
||||||
|
color: var(--pf-global--warning-color--100);
|
||||||
|
}
|
||||||
.pf-m-danger {
|
.pf-m-danger {
|
||||||
color: var(--pf-global--danger-color--100);
|
color: var(--pf-global--danger-color--100);
|
||||||
}
|
}
|
||||||
|
|
Reference in New Issue