2019-06-24

Seamless calling between python and c/c++

主要整理c/c++ call python 以及 python call c/c++ 的基本使用。

prerequisite

how python search the package when import module

The main logic is to build the c/c++ code into the .so file and add the .so into the search file of the python package. How to check the python search path?

import sys
print (sys.path)

If we modify the environment variable export PYTHONPATH=<path_for_so_file>, this path will be added into the sys.path

If we want to show the path for specific module, we could use

import numpy
print(numpy.__file__)

关于cpython，pybind11，swig

参考这个
https://www.zhihu.com/question/323926607

显而易见，使用pybind11是当前主流的方式

one important thing before linking

to detect if the current version of python, not mattter for python2 or python3 is static (with the suffix of .a) or the dynamic (with the suffix of .so) is important thing. Since it requires different operations to link the packages. the fpic and enable shared parameters are necessary for linking into a shared python libraray. Check this discussion to get more details:

https://discourse.paraview.org/t/undefined-symbol-pyexc-valueerror/5494/4

In this case, the python library is not linked firstly, then we set particular options as OFF, the python library is failed to link with the other binaries because it did not compiled by the fpic label.

this is an example of embed python into the c++ manually:

cpp file:
#include <Python.h>
#include <stdio.h>

int main(int argc, char *argv[])
{
  setenv("PYTHONPATH", ".", 0);
  char hostname[] = "localhost";
  PyObject *pName, *pModule, *pFunc;
  PyObject *pArgs, *pValue;
  Py_Initialize();
  pName = PyUnicode_FromString("GetHostname");
  pModule = PyImport_Import(pName);
  Py_DECREF(pName);

  if(pModule != NULL) {
    pFunc = PyObject_GetAttrString(pModule, "GetHostname");

    if (pFunc && PyCallable_Check(pFunc)) {
      pArgs = PyTuple_New(1);
      pValue = PyUnicode_FromString(hostname);
      PyTuple_SetItem(pArgs, 0, pValue);
      pValue = PyObject_CallObject(pFunc, pArgs);
      Py_DECREF(pArgs);
      if (pValue != NULL) {
    printf("The IP address is %s\n", PyBytes_AS_STRING(pValue));
    Py_DECREF(pValue);
      }
      else {
    Py_DECREF(pFunc);
    Py_DECREF(pModule);
    PyErr_Print();
    fprintf(stderr, "Call Failed\n");
    return 1;
      }
    }
    else {
      if (PyErr_Occurred())
    PyErr_Print();
      fprintf(stderr, "Cannot find function\n");
    }
    Py_XDECREF(pFunc);
    Py_DECREF(pModule);
  }
  else {
    PyErr_Print();
    fprintf(stderr, "Failed to load file\n");
    return 1;
  }
  Py_Finalize();
  return 0;
}

and the python file:

import socket

def GetHostname(hostname):
    addr = socket.gethostbyname(hostname)
    return addr

compile by the cmake:

cmake_minimum_required (VERSION 3.14)
project(GetHostname)
find_package("Python2" QUIET REQUIRED COMPONENTS Interpreter)

find_package( PythonInterp "2" REQUIRED )
find_package( PythonLibs "2" REQUIRED )
include_directories ( ${PYTHON_INCLUDE_DIRS} )
add_executable (GetHostname GetHostname.cc)
target_link_libraries(GetHostname ${PYTHON_LIBRARIES})

compile by the g++ manually

g++ $(python-config --cflags) -o GetHostname $(python-config --ldflags) ../GetHostname.cc

The running reuslts looks like this:

$ ./GetHostname 
The IP address is 127.0.0.1

call c/c++ code in python

use pybind11

It is straight forward with the help of the pybind11 and the documents online is clear with sample code.

call python from the c/c++ code

use pybind11

using embed to put the intepreter in c/c++

use cpython

official documents:
https://docs.python.org/2/extending/embedding.html

即使使用了pybind11，最后编译成的.so文件还是cpython的

关于一个实际bug的解决

具体可以参考这个

最终的原因是由于一个函数和系统的函数命名一样了，导致的这个错误（不使用namespace的缺点）

虽然仅仅知道怎么解决，还不知道原因为什么是这样，但这个过程有一些收获

比如知道如何来查看cpython的代码然后怎么通过对应的版本来互相匹配找到正确的分支比如看这里比如知道了cpython的实现里 dprintf是被重新定义过的没有使用库函数的那个dprintf 需要查看底层的打印信息的话需要重新编译python然后需要加上哪些环境变量比如参考这个.

然后要把c和系统的东西再更上一层楼的话看看Cpython的实现是一个不错的途径 reference中也列出了一些资料

然后对namespace又有了一些新的认识像这种case 使用 namespace 就不容易出现类似的问题了或者有良好的命名习惯一个project中的函数都使用特定的前缀开始

然后从c/c++ call python的时候首先保证python自己是可以正常运行通过python 可以正常进行import操作，以及python的module是可以被load 的。

关于mpi4Py

use mpi4py with pybind11
https://stackoverflow.com/questions/49259704/pybind11-possible-to-use-mpi4py

download and compile mpi4py 之前注意使用module check 到底是 openmpi还是mpich 保证编译和执行的时候使用的是一种mpi的实现

https://groups.google.com/forum/#!topic/mpi4py/jPqNrr_8UWY

https://mpi4py.readthedocs.io/en/stable/tutorial.html#wrapping-with-swig

source code

https://bitbucket.org/mpi4py/mpi4py/src/18c52a947443765dea0adaee0b702c044d9f150c/demo/python-config?at=master&fileviewer=file-view-default

wrap with python
https://mpi4py.readthedocs.io/en/stable/tutorial.html#wrapping-with-swig

https://stackoverflow.com/questions/12459906/how-to-pass-mpi-information-to-ctypes-in-python

Attempting to use an MPI routine before initializing MPICH
https://bitbucket.org/mpi4py/mpi4py/issues/107/compiling-problem-for-swig-demo

get numpy array from python

//sample code at the cpp end
#include <iostream>
#include <pybind11/embed.h> // everything needed for embedding
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include <typeinfo>
namespace py = pybind11;

int getDataAndProcess(py::module pyReader, std::size_t starIndex)
{

  //allocate the buffer for numpy array
  std::size_t bufferSize = 10;
  py::array_t<uint16_t> array(bufferSize, 0);
  auto rdata = pyReader.attr("load_next_block")(starIndex);
  array = rdata.cast<py::array_t<uint16_t>>();
  py::buffer_info info = array.request();
  uint16_t *ptr1 = (uint16_t *)info.ptr;
  std::cout << "process the data at the c++ end " << starIndex
            << "\n";
  for (int i = 0; i < 10; i++)
  {
    std::cout << " " << *ptr1;
    ptr1++;
  }
  std::cout << "\n";
}

int main()
{

  // init python interpreter
  py::scoped_interpreter guard{};
  py::print("test python interpreter");
  auto sys = py::module::import("sys");
  sys.attr("path").attr("append")(
      "/home/zhe/Documents/cworkspace/pybindTest/pyReader");
  py::module pyreader = py::module::import("read_py");
  // get the data and process it
  for (int i = 0; i < 5; i++)
  {
    getDataAndProcess(pyreader, i);
  }

  return 0;
}

//code at the python end
import numpy as np

import sys
import numpy as np

def load_next_block(index):
    
    block_data = np.ones((10,), dtype=np.uint16)*index
    print ("data at python:")
    print (block_data)
    return block_data


if __name__ == '__main__':
    for i in range (5):
        load_next_block(i)

For compiling, juse use the add_executable. The .cmake configuration will be generated after the build and install for the bybind

cmake_minimum_required(VERSION 2.8.12)
project(pyreader)

set (pybind11_DIR /home/zhe/Documents/cworkspace/build_pybind11/install/share/cmake/pybind11)
find_package(pybind11 REQUIRED)

add_executable(pyreader read_cpp.cpp)
target_link_libraries(pyreader pybind11::embed)