Python, use multiprocessing to speed up cython

Question

the code given here is simplified, but triggers the same PicklingError. I know that there is a lot of discussion about what can and cannot be pickled, but I found a solution from them.

I am writing a simple cython script with the following function:

def pow2(int a) : 
    return a**2 

Compilation works, I can call this function in a python script.

However, I am wondering how to use this function in multiprocessing,

from multiprocessing import Pool
from fast import pow2
p = Pool(processes =4 )
y = p.map( pow2, np.arange( 10, dtype=int))

gives me a PicklingError:

dtw is the name of the package, and fast is fast.pyx.

How can I get around this problem? thanks in advance

Answer 1

multiprocessing, - , OpenMP prange. , .

• x*x x**2, pow(x, 2)):
• double
• , size % num_threads != 0

:

#cython: wraparound=False
#cython: boundscheck=False
#cython: cdivision=True
#cython: nonecheck=False
#cython: profile=False
import numpy as np
cimport numpy as np
from cython.parallel import prange

cdef void cpow2(int size, double *inp, double *out) nogil:
    cdef int i
    for i in range(size):
        out[i] = inp[i]*inp[i]

def pow2(np.ndarray[np.float64_t, ndim=1] inp,
         np.ndarray[np.float64_t, ndim=1] out,
         int num_threads=4):
    cdef int thread
    cdef np.ndarray[np.int32_t, ndim=1] sub_sizes, pos
    size = np.shape(inp)[0]
    sub_sizes = np.zeros(num_threads, np.int32) + size//num_threads
    pos = np.zeros(num_threads, np.int32)
    sub_sizes[num_threads-1] += size % num_threads
    pos[1:] = np.cumsum(sub_sizes)[:num_threads-1]
    for thread in prange(num_threads, nogil=True, chunksize=1,
                         num_threads=num_threads, schedule='static'):
        cpow2(sub_sizes[thread], &inp[pos[thread]], &out[pos[thread]])

def main():
    a = np.arange(642312323).astype(np.float64)
    pow2(a, out=a, num_threads=4)