# -*- coding: utf-8 -*-
"""
celery.concurrency.threads
~~~~~~~~~~~~~~~~~~~~~~~~~~
Pool implementation using threads.
"""
from __future__ import absolute_import
from celery.five import UserDict
from .base import apply_target, BasePool
__all__ = ['TaskPool']
class NullDict(UserDict):
def __setitem__(self, key, value):
pass
class TaskPool(BasePool):
def __init__(self, *args, **kwargs):
try:
import threadpool
except ImportError:
raise ImportError(
'The threaded pool requires the threadpool module.')
self.WorkRequest = threadpool.WorkRequest
self.ThreadPool = threadpool.ThreadPool
super(TaskPool, self).__init__(*args, **kwargs)
def on_start(self):
self._pool = self.ThreadPool(self.limit)
# threadpool stores all work requests until they are processed
# we don't need this dict, and it occupies way too much memory.
self._pool.workRequests = NullDict()
self._quick_put = self._pool.putRequest
self._quick_clear = self._pool._results_queue.queue.clear
def on_stop(self):
self._pool.dismissWorkers(self.limit, do_join=True)
def on_apply(self, target, args=None, kwargs=None, callback=None,
accept_callback=None, **_):
req = self.WorkRequest(apply_target, (target, args, kwargs, callback,
accept_callback))
self._quick_put(req)
# threadpool also has callback support,
# but for some reason the callback is not triggered
# before you've collected the results.
# Clear the results (if any), so it doesn't grow too large.
self._quick_clear()
return req