Dynamic loading libpython with pybind11 - c++11

Question

I'm trying to build some shared library with pybind11 from Mac OSX. I'm running into the error:
dyld: Symbol not found: _PyBaseObject_Type
Referenced from: /Users/xxxxx/work/test_dynamic_linking/./example
Expected in: flat namespace
in /Users/xxxxx/work/test_dynamic_linking/./example
Abort trap: 6
What I'm trying to achieve is to turn off build time linking, but dynamic loading libpython in runtime with dlopen. Note the -Wl,-undefined,dynamic_lookup flag in the cmake file. I'm doing it this way because I want to build a wheel, and linking to libpython is not a good idea AFAIU.
Below is a minimial reproducible example. I'm confused that if you call functions like Py_DecodeLocale() or Py_InitializeEx() directly from main.cpp, it works fine. But calling pybind11::initialize_interpreter() fails with the error above.
If I do
nm -gU /opt/anaconda3/envs/py38/lib/libpython3.8.dylib | grep PyBaseObject
the symbol _PyBaseObject_Type is indeed defined in the lib:
0000000000334528 D _PyBaseObject_Type
If I create a wrapper shared library which wraps the calls to pybind11 functions, and dlopen it from main.cpp, it works fine. This makes me more confused.
Cmake file:
cmake_minimum_required(VERSION 3.4)
project(example)
set (CMAKE_CXX_STANDARD 11)
set(UNDEFINED_SYMBOLS_IGNORE_FLAG "-Wl,-undefined,dynamic_lookup")
string(APPEND CMAKE_EXE_LINKER_FLAGS " ${UNDEFINED_SYMBOLS_IGNORE_FLAG}")
string(APPEND CMAKE_SHARED_LINKER_FLAGS " ${UNDEFINED_SYMBOLS_IGNORE_FLAG}")
include_directories(pybind11/include)
include_directories(/opt/anaconda3/envs/py38/include/python3.8)
add_library(pywrapper SHARED ${CMAKE_CURRENT_SOURCE_DIR}/wrapper.cpp)
add_executable(example main.cpp)
pyembed.hpp:
#pragma once
#include "pybind11/embed.h"
namespace py = pybind11;
void initialize_interpreter_func();
struct pybind_wrap_api {
decltype(&initialize_interpreter_func) initialize_interpreter;
};
wrapper.cpp:
#include "pyembed.hpp"
#include <set>
#include <vector>
#include <iostream>
#include "pybind11/embed.h"
#include "pybind11/stl.h"
namespace py = pybind11;
void initialize_interpreter_func() {
pybind11::initialize_interpreter();
}
pybind_wrap_api init_pybind_wrap_api() noexcept {
return {
&initialize_interpreter_func,
};
}
__attribute__((visibility("default"))) pybind_wrap_api pybind_wrapper_api =
init_pybind_wrap_api();
main.cpp:
#include <pybind11/embed.h> // everything needed for embedding
#include "pyembed.hpp"
#include <stdlib.h>
#include <dlfcn.h>
#include <iostream>
#include <string>
namespace py = pybind11;
static void* pylib_handle = nullptr;
static void* pybind_wrapper_handle = nullptr;
pybind_wrap_api* wrappers = nullptr;
int main() {
std::string path_libpython = "/opt/anaconda3/envs/py38/lib/libpython3.8.dylib";
pylib_handle = dlopen(path_libpython.c_str(), RTLD_NOW | RTLD_GLOBAL);
if(!pylib_handle) {
std::cout << "load libpython failed..." << std::endl;
} else {
std::cout << "load libpython succeeded..." << std::endl;
}
std::string path_wrapper = "./libpywrapper.dylib";
pybind_wrapper_handle = dlopen(path_wrapper.c_str(), RTLD_NOW | RTLD_GLOBAL);
wrappers = static_cast<pybind_wrap_api*>(dlsym(pybind_wrapper_handle, "pybind_wrapper_api"));
std::string pythonhome = "/opt/anaconda3/envs/py38";
setenv("PYTHONHOME", pythonhome.c_str(), 1);
std::string pythonpath = "/opt/anaconda3/envs/py38/lib/python3.8/site-packages";
setenv("PYTHONPATH", pythonpath.c_str(), true);
// this line will cause it to fail with the symbol not found error
py::initialize_interpreter();
// if comment out the previous line and do the following line, it works fine. I'm confused why is so.
//wrappers->initialize_interpreter();
return 0;
}
Then do
cmake . && make && ./example