I wrote an extension module that uses C ++ function pointers to store sequences of function calls. I want to “run” these call sequences in separate processes using the python multiprocessing module (there is no general state, so no synchronization problems).
I need to know if functional pointers (not data pointers) remain after multiprocessing does this fork() .
C ++ module:
#include <list> #include <boost/assert.hpp> #include <boost/python.hpp> #include <boost/python/stl_iterator.hpp> #include <boost/foreach.hpp> /* * Some functions to be called */ double funcA(double d) { return d; } double funcB(double d) { return d + 3.14; } double funcC(double d) { return d - 42.0; } /* * My container of function pointers (picklable to allow use with multiprocessing) */ typedef double(*func_ptr_t)(double); struct CallSequence { CallSequence() { _seq.push_back(funcA); _seq.push_back(funcB); _seq.push_back(funcC); } std::list<func_ptr_t> _seq; }; template <typename cast_type> struct CallSequence_picklesuite : boost::python::pickle_suite { BOOST_STATIC_ASSERT_MSG(sizeof(cast_type) == sizeof(func_ptr_t), CANNOT_CAST_POINTER_TO_REQUESTED_TYPE); static boost::python::list getstate(const CallSequence& cs) { boost::python::list ret; BOOST_FOREACH(func_ptr_t p, cs._seq) ret.append(reinterpret_cast<cast_type>(p)); return ret; } static void setstate(CallSequence& cs, boost::python::list l) { std::list<func_ptr_t> new_list; boost::python::stl_input_iterator<cast_type> begin(l), end; for(; begin != end; begin++) new_list.push_back(reinterpret_cast<func_ptr_t>(*begin)); cs._seq.swap(new_list); } }; /* * Run the call sequence */ double runner(const CallSequence& cs) { double ret = 0; BOOST_FOREACH(const func_ptr_t& p, cs._seq) ret += p(2.18); return ret; } BOOST_PYTHON_MODULE(my_extension) { using namespace ::boost::python; class_<CallSequence>("CallSequence") .def_pickle(CallSequence_picklesuite<unsigned int>()); def("runner", runner); }
Compiled with
$ g++ question1.cpp -lboost_python -I /usr/include/python2.7 -shared -o my_extension.so
Python code calling it through several processes:
#!/usr/bin/python from multiprocessing import Pool import my_extension def runner(sequence): return my_extension.runner(sequence) def main(): l = [my_extension.CallSequence() for _ in range(200)] pool = Pool(processes=4) print pool.map(runner, l) if __name__ == '__main__': main()
The output is executed as expected. I want to know if I can just “be lucky”, or if I can reliably expect function pointers to remain valid after fork() .
source share