django-orchestra/orchestra/contrib/tasks/__init__.py

108 lines
3.4 KiB
Python
Raw Normal View History

2015-05-03 17:44:46 +00:00
import traceback
from functools import partial, wraps, update_wrapper
from multiprocessing import Process
from uuid import uuid4
from threading import Thread
from celery import shared_task as celery_shared_task
from celery import states
from celery.decorators import periodic_task as celery_periodic_task
from django.utils import timezone
from orchestra.utils.db import close_connection
from orchestra.utils.python import AttrDict, OrderedSet
def get_id():
return str(uuid4())
def get_name(fn):
return '.'.join((fn.__module__, fn.__name__))
def keep_state(fn):
""" logs task on djcelery's TaskState model """
@wraps(fn)
def wrapper(task_id, name, *args, **kwargs):
from djcelery.models import TaskState
now = timezone.now()
state = TaskState.objects.create(state=states.STARTED, task_id=task_id, name=name, args=str(args),
kwargs=str(kwargs), tstamp=now)
try:
result = fn(*args, **kwargs)
except Exception as exc:
state.state = states.FAILURE
state.traceback = traceback.format_exc()
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return
# TODO send email
else:
state.state = states.SUCCESS
state.result = str(result)
state.runtime = (timezone.now()-now).total_seconds()
state.save()
return result
return wrapper
def apply_async(fn, name=None, method='thread'):
""" replaces celery apply_async """
def inner(fn, name, method, *args, **kwargs):
task_id = get_id()
args = (task_id, name) + args
2015-05-03 21:26:17 +00:00
thread = method(target=fn, args=args, kwargs=kwargs)
2015-05-03 17:44:46 +00:00
thread.start()
# Celery API compat
thread.request = AttrDict(id=task_id)
return thread
2015-05-03 21:45:46 +00:00
2015-05-03 17:44:46 +00:00
if name is None:
name = get_name(fn)
if method == 'thread':
method = Thread
elif method == 'process':
method = Process
else:
2015-05-03 21:45:46 +00:00
raise NotImplementedError("%s concurrency method is not supported." % method)
2015-05-03 17:44:46 +00:00
fn.apply_async = partial(inner, close_connection(keep_state(fn)), name, method)
2015-05-03 21:26:17 +00:00
fn.delay = fn.apply_async
2015-05-03 17:44:46 +00:00
return fn
def task(fn=None, **kwargs):
2015-05-03 20:08:32 +00:00
# TODO override this if 'celerybeat' in sys.argv ?
2015-05-03 17:44:46 +00:00
from . import settings
# register task
if fn is None:
2015-05-04 10:48:09 +00:00
name = kwargs.get('name', None)
2015-05-03 21:26:17 +00:00
if settings.TASKS_BACKEND in ('thread', 'process'):
def decorator(fn):
2015-05-04 10:48:09 +00:00
return apply_async(celery_shared_task(**kwargs)(fn), name=name)
2015-05-03 21:26:17 +00:00
return decorator
else:
return celery_shared_task(**kwargs)
fn = update_wraper(partial(celery_shared_task, fn))
2015-05-03 17:44:46 +00:00
if settings.TASKS_BACKEND in ('thread', 'process'):
2015-05-04 10:48:09 +00:00
fn = update_wrapper(apply_async(fn), fn)
2015-05-03 17:44:46 +00:00
return fn
def periodic_task(fn=None, **kwargs):
from . import settings
# register task
if fn is None:
2015-05-04 10:48:09 +00:00
name = kwargs.get('name', None)
2015-05-03 21:26:17 +00:00
if settings.TASKS_BACKEND in ('thread', 'process'):
def decorator(fn):
2015-05-04 10:48:09 +00:00
return apply_async(celery_periodic_task(**kwargs)(fn), name=name)
2015-05-03 21:26:17 +00:00
return decorator
else:
return celery_periodic_task(**kwargs)
2015-05-04 10:48:09 +00:00
fn = update_wraper(celery_periodic_task(fn), fn)
2015-05-03 17:44:46 +00:00
if settings.TASKS_BACKEND in ('thread', 'process'):
name = kwargs.pop('name', None)
2015-05-03 21:26:17 +00:00
fn = update_wrapper(apply_async(fn, name), fn)
2015-05-03 17:44:46 +00:00
return fn