The program works with map (), but raises a TypeError with pool.map ()

Question

I have a program that has not been used for a long time, but it has already been used and worked. It uses multiprocessing, since the same task must be performed for different data many times.

Now I touched the program to add a new parameter, tested it and noticed that it was causing an error. Also an earlier version with version controls the same error. The full error looks like this:

Exception in thread Thread-2:
    Traceback (most recent call last):
        File "/usr/lib64/python2.7/threading.py", line 811, in __bootstrap_inner
            self.run()
        File "/usr/lib64/python2.7/threading.py", line 764, in run
            self.__target(*self.__args, **self.__kwargs)
        File "/usr/lib64/python2.7/multiprocessing/pool.py", line 342, in _handle_tasks
           put(task)
TypeError: 'NoneType' object is not callable

That's all. Honestly, this does not tell me much. When trying to debug this, I came up with the idea of ​​trying to use the regular map()instead of the merged version pool.map. Then the script works fine.

, , , , :

import random
import time
from multiprocessing import Pool


def do_work(x, y, z):
     time.sleep(random.random() * 2)
     print x + y + z

def do_one(arguments):
    print "doing one"
    do_work(*arguments)

def do_many(x, y, zs):
    map(do_one, [(x, y, z) for z in zs])

def do_many_pooled(x, y, zs):
    pool = Pool(2)
    pool.map(do_one, [(x, y, z) for z in zs])
    pool.close()
    pool.join()

def main():
    x = 1
    y = 2
    zs = range(10)
    print "doing many"
    do_many(x, y, zs)
    print "doing many pooled"
    do_many_pooled(x, y, zs)


if __name__ == '__main__':
    main()

, numpy . , " ", , , .

- , Traceback / , ?

Answer 1

, , put None :

File "/usr/lib64/python2.7/multiprocessing/pool.py", line 342, in _handle_tasks
   put(task)
TypeError: 'NoneType' object is not callable

Python, Pool.__init__() _task_handler _handle_tasks :

self._task_handler = threading.Thread(
    target=Pool._handle_tasks,
    args=(self._taskqueue, self._quick_put, self._outqueue, self._pool)
    )

_handle_tasks, , self._quick_put - , put:

@staticmethod
def _handle_tasks(taskqueue, put, outqueue, pool, cache):
    thread = threading.current_thread()

    for taskseq, set_length in iter(taskqueue.get, None):
        i = -1
        for i, task in enumerate(taskseq):
            if thread._state:
                debug('task handler found thread._state != RUN')
                break
            try:
                put(task)
            except Exception as e:
                job, ind = task[:2]
                try:
                    cache[job]._set(ind, (False, e))
                except KeyError:
                    pass
        else:
            if set_length:
                debug('doing set_length()')
                set_length(i+1)
            continue
        break
    else:
        debug('task handler got sentinel')

, , . Python 2.7.6, :

@staticmethod
def _handle_tasks(taskqueue, put, outqueue, pool):
    thread = threading.current_thread()

    for taskseq, set_length in iter(taskqueue.get, None):
        i = -1
        for i, task in enumerate(taskseq):
            if thread._state:
                debug('task handler found thread._state != RUN')
                break
            try:
                put(task)
            except IOError:
                debug('could not put task on queue')
                break
        else:
            if set_length:
                debug('doing set_length()')
                set_length(i+1)
            continue
        break
    else:
        debug('task handler got sentinel')

, a TypeError . , # 19425. , , Python 2.7, - .

, put() , - put . , Python. , Python?

Google :

• Python # 9755 - , .

• Python # 15881 - , .

, fail "'NoneType" ".

Python . - . , . I , Python.