*: improve error handling for startup tasks

Signed-off-by: Jens Langhammer <jens.langhammer@beryju.org>
This commit is contained in:
Jens Langhammer 2022-08-01 23:31:47 +02:00
parent 3cd0a782af
commit 2bd29e2fdd
6 changed files with 25 additions and 8 deletions

View File

@ -3,6 +3,7 @@ import re
from django.core.cache import cache from django.core.cache import cache
from django.core.validators import URLValidator from django.core.validators import URLValidator
from django.db import DatabaseError, InternalError, ProgrammingError
from packaging.version import parse from packaging.version import parse
from requests import RequestException from requests import RequestException
from structlog.stdlib import get_logger from structlog.stdlib import get_logger
@ -39,7 +40,9 @@ def _set_prom_info():
) )
@CELERY_APP.task() @CELERY_APP.task(
throws=(DatabaseError, ProgrammingError, InternalError),
)
def clear_update_notifications(): def clear_update_notifications():
"""Clear update notifications on startup if the notification was for the version """Clear update notifications on startup if the notification was for the version
we're running now.""" we're running now."""

View File

@ -3,7 +3,7 @@ from importlib import import_module
from inspect import ismethod from inspect import ismethod
from django.apps import AppConfig from django.apps import AppConfig
from django.db import DatabaseError, ProgrammingError from django.db import DatabaseError, InternalError, ProgrammingError
from structlog.stdlib import get_logger from structlog.stdlib import get_logger
LOGGER = get_logger() LOGGER = get_logger()
@ -33,5 +33,5 @@ class ManagedAppConfig(AppConfig):
try: try:
meth() meth()
LOGGER.debug("Successfully reconciled", name=name) LOGGER.debug("Successfully reconciled", name=name)
except (ProgrammingError, DatabaseError) as exc: except (DatabaseError, ProgrammingError, InternalError) as exc:
LOGGER.debug("Failed to run reconcile", name=name, exc=exc) LOGGER.debug("Failed to run reconcile", name=name, exc=exc)

View File

@ -19,7 +19,9 @@ from authentik.lib.config import CONFIG
from authentik.root.celery import CELERY_APP from authentik.root.celery import CELERY_APP
@CELERY_APP.task() @CELERY_APP.task(
throws=(DatabaseError, ProgrammingError, InternalError),
)
@prefill_task @prefill_task
def blueprints_discover(): def blueprints_discover():
"""Find blueprints and check if they need to be created in the database""" """Find blueprints and check if they need to be created in the database"""

View File

@ -8,7 +8,7 @@ from channels.middleware import BaseMiddleware
from channels_redis.core import ChannelFull from channels_redis.core import ChannelFull
from django.conf import settings from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation, ValidationError from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation, ValidationError
from django.db import InternalError, OperationalError, ProgrammingError from django.db import DatabaseError, InternalError, OperationalError, ProgrammingError
from django.http.response import Http404 from django.http.response import Http404
from django_redis.exceptions import ConnectionInterrupted from django_redis.exceptions import ConnectionInterrupted
from docker.errors import DockerException from docker.errors import DockerException
@ -116,6 +116,7 @@ def before_send(event: dict, hint: dict) -> Optional[dict]:
# Django Errors # Django Errors
Error, Error,
ImproperlyConfigured, ImproperlyConfigured,
DatabaseError,
OperationalError, OperationalError,
InternalError, InternalError,
ProgrammingError, ProgrammingError,

View File

@ -10,6 +10,7 @@ import yaml
from asgiref.sync import async_to_sync from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer from channels.layers import get_channel_layer
from django.core.cache import cache from django.core.cache import cache
from django.db import DatabaseError, InternalError, ProgrammingError
from django.db.models.base import Model from django.db.models.base import Model
from django.utils.text import slugify from django.utils.text import slugify
from docker.constants import DEFAULT_UNIX_SOCKET from docker.constants import DEFAULT_UNIX_SOCKET
@ -87,7 +88,11 @@ def outpost_service_connection_state(connection_pk: Any):
cache.set(connection.state_key, state, timeout=None) cache.set(connection.state_key, state, timeout=None)
@CELERY_APP.task(bind=True, base=MonitoredTask) @CELERY_APP.task(
bind=True,
base=MonitoredTask,
throws=(DatabaseError, ProgrammingError, InternalError),
)
@prefill_task @prefill_task
def outpost_service_connection_monitor(self: MonitoredTask): def outpost_service_connection_monitor(self: MonitoredTask):
"""Regularly check the state of Outpost Service Connections""" """Regularly check the state of Outpost Service Connections"""
@ -102,7 +107,9 @@ def outpost_service_connection_monitor(self: MonitoredTask):
) )
@CELERY_APP.task() @CELERY_APP.task(
throws=(DatabaseError, ProgrammingError, InternalError),
)
def outpost_controller_all(): def outpost_controller_all():
"""Launch Controller for all Outposts which support it""" """Launch Controller for all Outposts which support it"""
for outpost in Outpost.objects.exclude(service_connection=None): for outpost in Outpost.objects.exclude(service_connection=None):

View File

@ -1,9 +1,13 @@
"""proxy provider tasks""" """proxy provider tasks"""
from django.db import DatabaseError, InternalError, ProgrammingError
from authentik.providers.proxy.models import ProxyProvider from authentik.providers.proxy.models import ProxyProvider
from authentik.root.celery import CELERY_APP from authentik.root.celery import CELERY_APP
@CELERY_APP.task() @CELERY_APP.task(
throws=(DatabaseError, ProgrammingError, InternalError),
)
def proxy_set_defaults(): def proxy_set_defaults():
"""Ensure correct defaults are set for all providers""" """Ensure correct defaults are set for all providers"""
for provider in ProxyProvider.objects.all(): for provider in ProxyProvider.objects.all():