diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d2c8cb77..471de231f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ if(NOT CMAKE_BUILD_TYPE) FORCE) endif() -project(heyoka VERSION 6.0.0 LANGUAGES CXX C) +project(heyoka VERSION 6.1.0 LANGUAGES CXX C) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/yacma") @@ -194,6 +194,7 @@ set(HEYOKA_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/debug.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/aligned_buffer.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/type_traits.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/get_dl_path.cpp" # NOTE: this will be an empty file in case we are not # building with support for real. "${CMAKE_CURRENT_SOURCE_DIR}/src/detail/real_helpers.cpp" @@ -336,7 +337,7 @@ if(HEYOKA_WITH_SLEEF) endif() # Setup the heyoka ABI version number. -set(HEYOKA_ABI_VERSION 30) +set(HEYOKA_ABI_VERSION 31) if(HEYOKA_BUILD_STATIC_LIBRARY) # Setup of the heyoka static library. diff --git a/doc/changelog.rst b/doc/changelog.rst index c853c70cc..f97b75bd5 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,12 +1,15 @@ Changelog ========= -6.0.1 (unreleased) +6.1.0 (unreleased) ------------------ Fix ~~~ +- Fix symbols not found by the JIT runtime when heyoka + is ``dlopen()``-ed with ``RTLD_LOCAL`` + (`#460 `__). - Workaround for a clang 17 issue that would result in runtime exceptions during (de)serialisation (`#458 `__). diff --git a/include/heyoka/detail/get_dl_path.hpp b/include/heyoka/detail/get_dl_path.hpp new file mode 100644 index 000000000..1aa0e8d30 --- /dev/null +++ b/include/heyoka/detail/get_dl_path.hpp @@ -0,0 +1,27 @@ +// Copyright 2020, 2021, 2022, 2023, 2024 Francesco Biscani (bluescarni@gmail.com), Dario Izzo (dario.izzo@gmail.com) +// +// This file is part of the heyoka library. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef HEYOKA_DETAIL_GET_DL_PATH_HPP +#define HEYOKA_DETAIL_GET_DL_PATH_HPP + +#include + +#include + +HEYOKA_BEGIN_NAMESPACE + +namespace detail +{ + +const std::string &get_dl_path(); + +} + +HEYOKA_END_NAMESPACE + +#endif diff --git a/src/detail/get_dl_path.cpp b/src/detail/get_dl_path.cpp new file mode 100644 index 000000000..e8e088b14 --- /dev/null +++ b/src/detail/get_dl_path.cpp @@ -0,0 +1,96 @@ +// Copyright 2020, 2021, 2022, 2023, 2024 Francesco Biscani (bluescarni@gmail.com), Dario Izzo (dario.izzo@gmail.com) +// +// This file is part of the heyoka library. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include +#include +#include +#include + +#if __has_include() + +#define HEYOKA_DETAIL_GET_DL_PATH_DLFCN + +#include + +#endif + +#include +#include + +HEYOKA_BEGIN_NAMESPACE + +namespace detail +{ + +namespace +{ + +#if defined(HEYOKA_DETAIL_GET_DL_PATH_DLFCN) + +// NOTE: need a dummy variable to take the address of. +const int dummy = 42; + +// Implementation of make_dl_path() based on dladdr(), from the dlfcn.h header. +std::string make_dl_path_dlfcn() +{ + // Invoke dladdr(). + ::Dl_info dl_info; + const auto ret = ::dladdr(static_cast(&dummy), &dl_info); + + // NOTE: in case of any failure, we will fall through the + // "return {};" statement and produce an empty string. + if (ret != 0 && dl_info.dli_fname != nullptr) { + try { + return std::filesystem::canonical(std::filesystem::path(dl_info.dli_fname)).string(); + // LCOV_EXCL_START + } catch (const std::exception &e) { + std::cerr << "WARNING - exception raised while trying to determine the path of the heyoka library: " + << e.what() << std::endl; + } catch (...) { + std::cerr << "WARNING - exception raised while trying to determine the path of the heyoka library" + << std::endl; + } + } + + return {}; + // LCOV_EXCL_STOP +} + +#endif + +// Factory function for dl_path. +std::string make_dl_path() +{ +#if defined(HEYOKA_DETAIL_GET_DL_PATH_DLFCN) + + return make_dl_path_dlfcn(); + +#else + + return {}; + +#endif +} + +// The path to the heyoka shared library. +const std::string dl_path = make_dl_path(); + +} // namespace + +// This function is meant to return the full canonicalised path to the heyoka shared library. +// +// If, for any reason, the path cannot be determined, an empty +// string will be returned instead. +const std::string &get_dl_path() +{ + return detail::dl_path; +} + +} // namespace detail + +HEYOKA_END_NAMESPACE diff --git a/src/llvm_state.cpp b/src/llvm_state.cpp index 8d4f45ff8..1fa6d229d 100644 --- a/src/llvm_state.cpp +++ b/src/llvm_state.cpp @@ -102,6 +102,7 @@ #endif +#include #include #include #include @@ -720,6 +721,31 @@ struct llvm_state::jit { // LCOV_EXCL_STOP m_lljit->getMainJITDylib().addGenerator(std::move(*dlsg)); + // NOTE: we also want to manually inject the symbols from the heyoka shared library + // into the JIT runtime. + // + // The reason for this is that if heyoka is loaded with dlopen() and the RTLD_LOCAL flag, + // then the JIT runtime will not be able to resolve symbols defined either in heyoka or + // in its dependencies. This will happen for instance in heyoka.py. + // + // With the following contraption, as far as I have understood, we are manually injecting all the + // symbols from heyoka (and, transitively, its dependencies) into the JIT runtime, and the symbols + // are thus made available to JIT code despite the use of RTLD_LOCAL. + // + // This approach was suggested by lhames on the LLVM discord. + if (const auto &dl_path = detail::get_dl_path(); !dl_path.empty()) { + auto new_dlsg = llvm::orc::DynamicLibrarySearchGenerator::Load(dl_path.c_str(), + m_lljit->getDataLayout().getGlobalPrefix()); + if (new_dlsg) [[likely]] { + m_lljit->getMainJITDylib().addGenerator(std::move(*new_dlsg)); + } else { + // LCOV_EXCL_START + throw std::invalid_argument( + "Could not create the dynamic library search generator for the heyoka library"); + // LCOV_EXCL_STOP + } + } + // Keep a target machine around to fetch various // properties of the host CPU. auto tm = jtmb.createTargetMachine(); @@ -735,7 +761,7 @@ struct llvm_state::jit { // NOTE: by default, errors in the execution session are printed // to screen. A custom error reported can be specified, ideally - // we would like th throw here but I am not sure whether throwing + // we would like to throw here but I am not sure whether throwing // here would disrupt LLVM's cleanup actions? // https://llvm.org/doxygen/classllvm_1_1orc_1_1ExecutionSession.html @@ -1802,6 +1828,30 @@ multi_jit::multi_jit(unsigned n_modules, unsigned opt_level, code_model c_model, // LCOV_EXCL_STOP m_lljit->getMainJITDylib().addGenerator(std::move(*dlsg)); + // NOTE: we also want to manually inject the symbols from the heyoka shared library + // into the JIT runtime. + // + // The reason for this is that if heyoka is loaded with dlopen() and the RTLD_LOCAL flag, + // then the JIT runtime will not be able to resolve symbols defined either in heyoka or + // in its dependencies. This will happen for instance in heyoka.py. + // + // With the following contraption, as far as I have understood, we are manually injecting all the + // symbols from heyoka (and, transitively, its dependencies) into the JIT runtime, and the symbols + // are thus made available to JIT code despite the use of RTLD_LOCAL. + // + // This approach was suggested by lhames on the LLVM discord. + if (const auto &dl_path = detail::get_dl_path(); !dl_path.empty()) { + auto new_dlsg = llvm::orc::DynamicLibrarySearchGenerator::Load(dl_path.c_str(), + m_lljit->getDataLayout().getGlobalPrefix()); + if (new_dlsg) [[likely]] { + m_lljit->getMainJITDylib().addGenerator(std::move(*new_dlsg)); + } else { + // LCOV_EXCL_START + throw std::invalid_argument("Could not create the dynamic library search generator for the heyoka library"); + // LCOV_EXCL_STOP + } + } + // Create the master context. m_ctx = std::make_unique(std::make_unique()); diff --git a/tools/gha_osx_x86.sh b/tools/gha_osx_x86.sh index 9029937e3..e5067aaf2 100644 --- a/tools/gha_osx_x86.sh +++ b/tools/gha_osx_x86.sh @@ -13,7 +13,7 @@ export PATH="$HOME/miniconda/bin:$PATH" bash miniconda.sh -b -p $HOME/miniconda conda create -y -p $deps_dir c-compiler zlib cxx-compiler libcxx cmake ninja \ llvmdev tbb-devel tbb libboost-devel sleef xtensor xtensor-blas blas \ - blas-devel fmt spdlog 'mppp=1.*' + blas-devel fmt spdlog 'mppp=1.*' 'clang<19' 'clangxx<19' source activate $deps_dir # Create the build dir and cd into it. diff --git a/tools/gha_osx_x86_static.sh b/tools/gha_osx_x86_static.sh index 497b3bad9..411269992 100644 --- a/tools/gha_osx_x86_static.sh +++ b/tools/gha_osx_x86_static.sh @@ -13,7 +13,7 @@ export PATH="$HOME/miniconda/bin:$PATH" bash miniconda.sh -b -p $HOME/miniconda conda create -y -p $deps_dir c-compiler zlib cxx-compiler libcxx cmake ninja \ llvmdev tbb-devel tbb libboost-devel sleef xtensor xtensor-blas blas \ - blas-devel fmt spdlog 'mppp=1.*' + blas-devel fmt spdlog 'mppp=1.*' 'clang<19' 'clangxx<19' source activate $deps_dir # Create the build dir and cd into it.