All geek questions in one place

Python and Torch7 (Lua) interaction via shared library

I am trying to pass data (arrays) between python and lua, and I want to manipulate data in lua using the Torch7 framework. I figured this was best done with C, since the python and lua interface is with C. Also, some of the advantages are that copying data is not required this way (passing only pointers) and is fast.

I have implemented two programs in which lua is embedded in c and one where python passes data to c. They both work when compiling binary executables. However, when the c to lua program is used instead to collaborate with the library, it does not work.

Details: Im uses 64-bit ubuntu 14.04 and 12.04. Im using luajit 2.0.2 with lua 5.1 installed in / usr / local / The libs dependencies are in / usr / local / lib and the headers are in / usr / local / include Im using python 2.7

Program code c to lua:

tensor.lua

require 'torch'

function hi_tensor(t)
   print(‘Hi from lua')
   torch.setdefaulttensortype('torch.FloatTensor')
   print(t)
return t*2
end

cluaf.h

void multiply (float* array, int m, int n, float *result, int m1, int n1);

cluaf.c

#include <stdio.h>
#include <string.h>
#include "lua.h"
#include "lauxlib.h"
#include "lualib.h"
#include "luaT.h"
#include "TH/TH.h"

void multiply (float* array, int m, int n, float *result, int m1, int n1)
{
    lua_State *L = luaL_newstate();
    luaL_openlibs( L );

    // loading the lua file
    if (luaL_loadfile(L, "tensor.lua") || lua_pcall(L, 0, 0, 0))
    {
        printf("error: %s \n", lua_tostring(L, -1));
    }

    // convert the c array to Torch7 specific structure representing a tensor
    THFloatStorage* storage =  THFloatStorage_newWithData(array, m*n);
    THFloatTensor* tensor = THFloatTensor_newWithStorage2d(storage, 0, m, n, n, 1);
    luaT_newmetatable(L, "torch.FloatTensor", NULL, NULL, NULL, NULL);

    // load the lua function hi_tensor
    lua_getglobal(L, "hi_tensor");
    if(!lua_isfunction(L,-1))
    {
        lua_pop(L,1);
    }

    //this pushes data to the stack to be used as a parameter
    //to the hi_tensor function call
    luaT_pushudata(L, (void *)tensor, "torch.FloatTensor");

    // call the lua function hi_tensor
    if (lua_pcall(L, 1, 1, 0) != 0)
    {
        printf("error running function `hi_tensor': %s \n", lua_tostring(L, -1));
    }

    // get results returned from the lua function hi_tensor
    THFloatTensor* z = luaT_toudata(L, -1, "torch.FloatTensor");
    lua_pop(L, 1);
    THFloatStorage *storage_res =  z->storage;
    result = storage_res->data;

    return ;
}

Then to check:

luajit -b tensor.lua tensor.o

gcc -w -c -Wall -Wl,-E -fpic cluaf.c -lluajit -lluaT -lTH -lm -ldl -L /usr/local/lib

gcc -shared cluaf.o tensor.o -L/usr/local/lib -lluajit -lluaT -lTH -lm -ldl -Wl,-E -o libcluaf.so

gcc -L. -Wall -o test main.c -lcluaf

./test

Output:

Hi from lua
 1.0000  0.2000
 0.2000  5.3000
[torch.FloatTensor of dimension 2x2]

c result 2.000000 
c result 0.400000 
c result 0.400000 
c result 10.60000

So far so good. But when I try to use a shared library in python, it breaks.

test.py

from ctypes import byref, cdll, c_int
import ctypes
import numpy as np
import cython

l = cdll.LoadLibrary(‘absolute_path_to_so/libcluaf.so')

a = np.arange(4, dtype=np.float64).reshape((2,2))
b = np.arange(4, dtype=np.float64).reshape((2,2))

l.multiply.argtypes = [ctypes.POINTER(ctypes.c_float), ctypes.c_int, ctypes.c_int,     ctypes.POINTER(ctypes.c_float), ctypes.c_int, ctypes.c_int]
a_list = []
b_list = []

for i in range(a.shape[0]):
    for j in range(a.shape[1]):
            a_list.append(a[i][j])

for i in range(b.shape[0]):
     for j in range(b.shape[1]):
        b_list.append(b[i][j])

arr_a = (ctypes.c_float * len(a_list))()
arr_b = (ctypes.c_float * len(b_list))()

l.multiply(arr_a, ctypes.c_int(2), ctypes.c_int(2), arr_b, ctypes.c_int(2), ctypes.c_int(2))

I run:

python test.py

and output:

error: error loading module 'libpaths' from file '/usr/local/lib/lua/5.1/libpaths.so':
    /usr/local/lib/lua/5.1/libpaths.so: undefined symbol: lua_gettop

I searched for this error here and everywhere on the Internet, but they either suggest (1) to include -Wl, -E to export characters, or (2) to add dependencies on the links I made. (1) I have -Wl, -E, but it doesn't seem to be doing anything. (2) I have included dependencies (-L / usr / local / lib -lluajit -lluaT -lTH -lm -ldl)

python , , " lua". , .

luajit.so lua_gettop (nm/usr/local/lib/luajit.so, ) lua.h LUA_API int (lua_gettop) (lua_State * L);

, c , lua.h lua_gettop luajit.so( ).

www.luajit.org/running.html : " ELF (, Linux) , . : -Wl, -E require() - ( *.exe lua51.dll Windows) package.cpath. '

package.cpath package.path:

./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so

./?.lua;/usr/local/share/luajit-2.0.2/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua

nm libcluaf.so:

00000000002020a0 B __bss_start
00000000002020a0 b completed.6972
                 w __cxa_finalize@@GLIBC_2.2.5
0000000000000a50 t deregister_tm_clones
0000000000000ac0 t __do_global_dtors_aux
0000000000201dd8 t __do_global_dtors_aux_fini_array_entry
0000000000202098 d __dso_handle
0000000000201de8 d _DYNAMIC
00000000002020a0 D _edata
00000000002020a8 B _end
0000000000000d28 T _fini
0000000000000b00 t frame_dummy
0000000000201dd0 t __frame_dummy_init_array_entry
0000000000000ed0 r __FRAME_END__
0000000000202000 d _GLOBAL_OFFSET_TABLE_
                 w __gmon_start__
0000000000000918 T _init
                 w _ITM_deregisterTMCloneTable
                 w _ITM_registerTMCloneTable
0000000000201de0 d __JCR_END__
0000000000201de0 d __JCR_LIST__
                 w _Jv_RegisterClasses
                 U lua_getfield
0000000000000d99 R luaJIT_BC_tensor
                 U luaL_loadfile
                 U luaL_newstate
                 U luaL_openlibs
                 U lua_pcall
                 U lua_settop
                 U luaT_newmetatable
                 U lua_tolstring
                 U luaT_pushudata
                 U luaT_toudata
                 U lua_type
0000000000000b35 T multiply
                 U printf@@GLIBC_2.2.5
0000000000000a80 t register_tm_clones
                 U THFloatStorage_newWithData
                 U THFloatTensor_newWithStorage2d
00000000002020a0 d __TMC_END__

+4
3

Linux Lua Lua, Lua API. -Wl,-E. , . : RTLD_GLOBAL dlopen. , , RTLD_LOCAL , , , Linux . , :

Lua (JIT), RTLD_GLOBAL, ( libcluaf.so):

from ctypes import byref, cdll, c_int
import ctypes

lualib = ctypes.CDLL("libluajit-5.1.so", mode=ctypes.RTLD_GLOBAL)
l = cdll.LoadLibrary('absolute_path_to_so/libcluaf.so')
# ...

Lua (JIT), RTLD_NOLOAD dlopen. POSIX, , , C . . .

+2

lua_gettop - , Lua.so, luajit.so. , , , , , -, c Lua, luajit. .

, lib ctypes python, luajit lib ? , , , ctypes . , ctypes lib luajit, , , luajit . , , Python.

, , : python, luajit ctypes .

0

To exchange data between python / numpy and lua / torch, you can try a library called " lutorpy ". It does exactly what you are trying to do, exchange memory and pass only a pointer using the asNumpyArray () method.

import lutorpy as lua
import numpy as np

## run lua code in python with minimal modification:  replace ":" to "._"
t = torch.DoubleTensor(10,3)
print(t._size()) # the corresponding lua version is t:size()

## convert torch tensor to numpy array
### Note: the underlying object are sharing the same memory, so the conversion is instant
arr = t.asNumpyArray()
print(arr.shape)

## or, you can convert numpy array to torch tensor
xn = np.random.randn(100)
## convert the numpy array into torch tensor
xt = torch.fromNumpyArray(xn)
0
source

Source: https://habr.com/ru/post/1548177/

More articles:


All Articles