From a547b9f733f4b60be266aa046dfaf43609b31db4 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 16:27:02 -0400 Subject: [PATCH 001/333] initial import of gasnet parcelport --- CMakeLists.txt | 8 + cmake/FindGasnet.cmake | 46 +++ cmake/HPX_SetupGASNET.cmake | 48 +++ libs/core/CMakeLists.txt | 1 + libs/core/gasnet_base/CMakeLists.txt | 40 +++ libs/core/gasnet_base/README.rst | 16 + libs/core/gasnet_base/docs/index.rst | 18 ++ libs/core/gasnet_base/examples/CMakeLists.txt | 16 + .../include/hpx/gasnet_base/gasnet.hpp | 28 ++ .../hpx/gasnet_base/gasnet_environment.hpp | 94 ++++++ .../gasnet_base/src/gasnet_environment.cpp | 300 ++++++++++++++++++ libs/core/gasnet_base/tests/CMakeLists.txt | 42 +++ .../tests/performance/CMakeLists.txt | 5 + .../tests/regressions/CMakeLists.txt | 5 + .../gasnet_base/tests/unit/CMakeLists.txt | 5 + libs/full/CMakeLists.txt | 1 + libs/full/parcelport_gasnet/CMakeLists.txt | 46 +++ libs/full/parcelport_gasnet/README.rst | 16 + libs/full/parcelport_gasnet/docs/index.rst | 18 ++ .../parcelport_gasnet/examples/CMakeLists.txt | 16 + .../include/hpx/parcelport_gasnet/header.hpp | 148 +++++++++ .../hpx/parcelport_gasnet/locality.hpp | 70 ++++ .../hpx/parcelport_gasnet/receiver.hpp | 222 +++++++++++++ .../include/hpx/parcelport_gasnet/sender.hpp | 103 ++++++ .../parcelport_gasnet/sender_connection.hpp | 221 +++++++++++++ libs/full/parcelport_gasnet/src/locality.cpp | 37 +++ .../src/parcelport_gasnet.cpp | 294 +++++++++++++++++ .../parcelport_gasnet/tests/CMakeLists.txt | 42 +++ .../tests/performance/CMakeLists.txt | 5 + .../tests/regressions/CMakeLists.txt | 5 + .../tests/unit/CMakeLists.txt | 5 + 31 files changed, 1921 insertions(+) create mode 100644 cmake/FindGasnet.cmake create mode 100644 cmake/HPX_SetupGASNET.cmake create mode 100644 libs/core/gasnet_base/CMakeLists.txt create mode 100644 libs/core/gasnet_base/README.rst create mode 100644 libs/core/gasnet_base/docs/index.rst create mode 100644 libs/core/gasnet_base/examples/CMakeLists.txt create mode 100644 libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp create mode 100644 libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp create mode 100644 libs/core/gasnet_base/src/gasnet_environment.cpp create mode 100644 libs/core/gasnet_base/tests/CMakeLists.txt create mode 100644 libs/core/gasnet_base/tests/performance/CMakeLists.txt create mode 100644 libs/core/gasnet_base/tests/regressions/CMakeLists.txt create mode 100644 libs/core/gasnet_base/tests/unit/CMakeLists.txt create mode 100644 libs/full/parcelport_gasnet/CMakeLists.txt create mode 100644 libs/full/parcelport_gasnet/README.rst create mode 100644 libs/full/parcelport_gasnet/docs/index.rst create mode 100644 libs/full/parcelport_gasnet/examples/CMakeLists.txt create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp create mode 100644 libs/full/parcelport_gasnet/src/locality.cpp create mode 100644 libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp create mode 100644 libs/full/parcelport_gasnet/tests/CMakeLists.txt create mode 100644 libs/full/parcelport_gasnet/tests/performance/CMakeLists.txt create mode 100644 libs/full/parcelport_gasnet/tests/regressions/CMakeLists.txt create mode 100644 libs/full/parcelport_gasnet/tests/unit/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 33596644a2c2..f0da9d3a886d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1143,6 +1143,14 @@ if(HPX_WITH_NETWORKING) ADVANCED ) + hpx_option( + HPX_WITH_PARCELPORT_GASNET BOOL "Enable the GASNET based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) + endif() + hpx_option( HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON CATEGORY "Parcelport" diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake new file mode 100644 index 000000000000..9a20e8b33a5a --- /dev/null +++ b/cmake/FindGasnet.cmake @@ -0,0 +1,46 @@ +# Copyright (c) 2017 Thomas Heller +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +macro(find_gasnet) +set(GASNET_MPI_FOUND FALSE) +set(GASNET_UDP_FOUND FALSE) +set(GASNET_SMP_FOUND FALSE) + +if(HPX_WITH_NETWORKING) + pkg_check_modules(GASNET QUIET gasnet-mpi-seq) + if(NOT GASNET_FOUND) + pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") + else() + set(GASNET_UDP_FOUND TRUE) + endif() + else() + set(GASNET_MPI_FOUND TRUE) + hpx_setup_mpi() + endif() +else() + pkg_check_modules(GASNET QUIET gasnet-smp-seq) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET please set the PKG_CONFIG_PATH environment variable") + else() + set(GASNET_SMP_FOUND TRUE) + endif() +endif() + +if(NOT GASNET_INCLUDE_DIRS OR NOT GASNET_LIBRARY_DIRS) + hpx_error("Could not find GASNET_INCLUDE_DIRS or GASNET_LIBRARY_DIRS please \ + set the PKG_CONFIG_PATH environment variable" + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Gasnet DEFAULT_MSG GASNET_LIBRARY_DIRS GASNET_INCLUDE_DIRS +) + +endmacro() diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake new file mode 100644 index 000000000000..37957d79ba46 --- /dev/null +++ b/cmake/HPX_SetupGASNET.cmake @@ -0,0 +1,48 @@ +# Copyright (c) 2019-2022 Ste||ar Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(FindGasnet) + +macro(hpx_setup_gasnet) + + if(NOT GASNET_FOUND) + + find_gasnet() + + set(GASNET_CXXFLAGS) + set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_LDFLAGS) + + foreach(TOKEN ${GASNET_CFLAGS}) + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endforeach() + + foreach(TOKEN ${GASNET_LDFLAGS_TMP}) + list(APPEND GASNET_LDFLAGS ${TOKEN}) + endforeach() + + add_library(gasnet INTERFACE IMPORTED) + target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) + target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) + target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) + + if(GASNET_MPI_FOUND) + # Setup PMI imported target + find_package(PMI) + if(PMI_FOUND) + hpx_add_config_define_namespace( + DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET + ) + add_library(Pmi::pmi INTERFACE IMPORTED) + target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) + target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) + endif() + endif() + + endif() +endmacro() diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index ac374840ecfc..aef704f82f90 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -55,6 +55,7 @@ set(_hpx_core_modules logging memory mpi_base + gasnet_base pack_traversal plugin prefix diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt new file mode 100644 index 000000000000..1649612a6947 --- /dev/null +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -0,0 +1,40 @@ +# Copyright (c) 2019-2023 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + # setup GASNET, if necessary + include(HPX_SetupGASNET) + hpx_setup_gasnet() +else() + return() +endif() + +# Default location is $HPX_ROOT/libs/gasnet_base/include +set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp hpx/gasnet_base/gasnet_environment.hpp) + +# Default location is $HPX_ROOT/libs/gasnet_base/include_compatibility +# cmake-format: off +set(gasnet_base_compat_headers + hpx/plugins/parcelport/gasnet/gasnet.hpp => hpx/modules/gasnet_base.hpp + hpx/plugins/parcelport/gasnet/gasnet_environment.hpp => hpx/modules/gasnet_base.hpp +) +# cmake-format: on + +set(gasnet_base_sources gasnet_environment.cpp) + +include(HPX_AddModule) +add_hpx_module( + core gasnet_base + GLOBAL_HEADER_GEN ON + SOURCES ${gasnet_base_sources} + HEADERS ${gasnet_base_headers} + COMPAT_HEADERS ${gasnet_base_compat_headers} + MODULE_DEPENDENCIES gasnet hpx_logging hpx_runtime_configuration hpx_util + DEPENDENCIES ${additional_dependencies} gasnet + CMAKE_SUBDIRS examples tests +) diff --git a/libs/core/gasnet_base/README.rst b/libs/core/gasnet_base/README.rst new file mode 100644 index 000000000000..1ae4ac39ce3b --- /dev/null +++ b/libs/core/gasnet_base/README.rst @@ -0,0 +1,16 @@ + +.. + Copyright (c) 2020 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +======== +mpi_base +======== + +This library is part of HPX. + +Documentation can be found `here +`__. diff --git a/libs/core/gasnet_base/docs/index.rst b/libs/core/gasnet_base/docs/index.rst new file mode 100644 index 000000000000..43dbdb7c2a03 --- /dev/null +++ b/libs/core/gasnet_base/docs/index.rst @@ -0,0 +1,18 @@ +.. + Copyright (c) 2020 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _modules_mpi_base: + +======== +mpi_base +======== + +This module provides helper functionality for detecting MPI environments. + +See the :ref:`API reference ` of this module for more +details. + diff --git a/libs/core/gasnet_base/examples/CMakeLists.txt b/libs/core/gasnet_base/examples/CMakeLists.txt new file mode 100644 index 000000000000..0d05c6cd0214 --- /dev/null +++ b/libs/core/gasnet_base/examples/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.mpi_base) + add_hpx_pseudo_dependencies(examples.modules examples.modules.mpi_base) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.mpi_base) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.mpi_base + ) + endif() +endif() diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp new file mode 100644 index 000000000000..23ab2a16eb38 --- /dev/null +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -0,0 +1,28 @@ +// Copyright (c) 2017 Mikael Simberg +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +#include + +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +#endif diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp new file mode 100644 index 000000000000..e577e7f78c9e --- /dev/null +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -0,0 +1,94 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2023 Tactical Computing Labs, LLC (Christopher Taylor) +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) + +#include +#include +#include + +#include +#include + +#include + +namespace hpx { namespace util { + struct HPX_CORE_EXPORT gasnet_environment + { + static bool check_gasnet_environment(runtime_configuration const& cfg); + + static int init(int* argc, char*** argv, const int minimal, + const int required, int& provided); + static void init(int* argc, char*** argv, runtime_configuration& cfg); + static void finalize(); + + static bool enabled(); + static bool multi_threaded(); + static bool has_called_init(); + + static int rank(); + static int size(); + + static std::string get_processor_name(); + + struct HPX_CORE_EXPORT scoped_lock + { + scoped_lock(); + scoped_lock(scoped_lock const&) = delete; + scoped_lock& operator=(scoped_lock const&) = delete; + ~scoped_lock(); + void unlock(); + }; + + struct HPX_CORE_EXPORT scoped_try_lock + { + scoped_try_lock(); + scoped_try_lock(scoped_try_lock const&) = delete; + scoped_try_lock& operator=(scoped_try_lock const&) = delete; + ~scoped_try_lock(); + void unlock(); + bool locked; + }; + + typedef hpx::spinlock mutex_type; + + private: + static mutex_type mtx_; + + static bool enabled_; + static bool has_called_init_; + static int provided_threading_flag_; + + static int is_initialized_; + + public: + static int init_val_; + }; +}} // namespace hpx::util + +#include + +#else + +#include + +#include + +namespace hpx { namespace util { + struct HPX_CORE_EXPORT gasnet_environment + { + static bool check_gasnet_environment(runtime_configuration const& cfg); + }; +}} // namespace hpx::util + +#include + +#endif diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp new file mode 100644 index 000000000000..b59d10d0cfb6 --- /dev/null +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -0,0 +1,300 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2020 Google +// Copyright (c) 2022 Patrick Diehl +// Copyright (c) 2023 Tactical Computing Labs, LLC (Christopher Taylor) +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +namespace hpx { namespace util { + + namespace detail { + + bool detect_gasnet_environment( + util::runtime_configuration const& cfg, char const* default_env) + { + std::string gasnet_environment_strings = + cfg.get_entry("hpx.parcel.gasnet.env", default_env); + + boost::char_separator sep(";,: "); + boost::tokenizer> tokens( + gasnet_environment_strings, sep); + for (auto const& tok : tokens) + { + char* env = std::getenv(tok.c_str()); + if (env) + { + LBT_(debug) + << "Found GASNET environment variable: " << tok << "=" + << std::string(env) << ", enabling GASNET support\n"; + return true; + } + } + + LBT_(info) << "No known GASNET environment variable found, disabling " + "GASNET support\n"; + return false; + } + } // namespace detail + + bool gasnet_environment::check_gasnet_environment( + util::runtime_configuration const& cfg) + { +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) + // We disable the GASNET parcelport if any of these hold: + // + // - The parcelport is explicitly disabled + // - The application is not run in an GASNET environment + // - The TCP parcelport is enabled and has higher priority + if (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) == 0 || + (get_entry_as(cfg, "hpx.parcel.udp.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.udp.priority", 1) > + get_entry_as(cfg, "hpx.parcel.gasnet.priority", 0))) || + (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.gasnet.priority", 1) > + get_entry_as(cfg, "hpx.parcel.gasnet.priority", 0)))) + { + LBT_(info) << "GASNET support disabled via configuration settings\n"; + return false; + } + + return true; +#elif defined(HPX_HAVE_MODULE_GASNET_BASE) + // if GASNET futures are enabled while networking is off we need to + // check whether we were run using gasnetrun + return detail::detect_gasnet_environment(cfg, HPX_HAVE_PARCELPORT_GASNET_ENV); +#else + return false; +#endif + } +}} // namespace hpx::util + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) + +namespace hpx { namespace util { + + gasnet_environment::mutex_type gasnet_environment::mtx_; + bool gasnet_environment::enabled_ = false; + bool gasnet_environment::has_called_init_ = false; + int gasnet_environment::provided_threading_flag_ = GASNET_SEQ; + int gasnet_environment::is_initialized_ = -1; + int gasnet_environment::init_val_ = GASNET_ERR_RESOURCE; + + /////////////////////////////////////////////////////////////////////////// + int gasnet_environment::init( + int* argc, char*** argv, const int minimal, const int required, int& provided) + { + if(!has_called_init_) { + gasnet_environment::init_val_ = gasnet_init(argc, argv); + has_called_init_ = true; + } + + if(gasnet_environment::init_val_ == GASNET_ERR_NOT_INIT) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET initialization error"); + } + else if(gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET resource error"); + } + else if(gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET bad argument error"); + } + else if(gasnet_environment::init_val_ == GASNET_ERR_NOT_READY) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET not ready error"); + } + + if (provided < minimal) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET doesn't provide minimal requested thread level"); + } + + return gasnet_environment::init_val_; + } + + /////////////////////////////////////////////////////////////////////////// + void gasnet_environment::init( + int* argc, char*** argv, util::runtime_configuration& rtcfg) + { + if (enabled_) + return; // don't call twice + + int this_rank = -1; + has_called_init_ = false; + + // We assume to use the GASNET parcelport if it is not explicitly disabled + enabled_ = check_gasnet_environment(rtcfg); + if (!enabled_) + { + rtcfg.add_entry("hpx.parcel.gasnet.enable", "0"); + return; + } + + rtcfg.add_entry("hpx.parcel.bootstrap", "gasnet"); + + int required = GASNET_SEQ; + int retval = + init(argc, argv, required, required, provided_threading_flag_); + if (GASNET_OK != retval) + { + // explicitly disable gasnet if not run by gasnetrun + rtcfg.add_entry("hpx.parcel.gasnet.enable", "0"); + + enabled_ = false; + + int msglen = 0; + char message[1024 + 1]; + sprintf(message, "%s\n", gasnet_ErrorDesc(retval)); + msglen = strnlen(message, 1025); + + std::string msg("gasnet_environment::init: gasnet_init failed: "); + msg = msg + message + "."; + throw std::runtime_error(msg.c_str()); + } + + if (provided_threading_flag_ != GASNET_SEQ) + { + // explicitly disable gasnet if not run by gasnetrun + rtcfg.add_entry("hpx.parcel.gasnet.multithreaded", "0"); + } + + this_rank = rank(); + +#if defined(HPX_HAVE_NETWORKING) + if (this_rank == 0) + { + rtcfg.mode_ = hpx::runtime_mode::console; + } + else + { + rtcfg.mode_ = hpx::runtime_mode::worker; + } +#elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) + rtcfg.mode_ = hpx::runtime_mode::console; +#else + rtcfg.mode_ = hpx::runtime_mode::local; +#endif + + rtcfg.add_entry("hpx.parcel.gasnet.rank", std::to_string(this_rank)); + rtcfg.add_entry("hpx.parcel.gasnet.processorname", get_processor_name()); + } + + std::string gasnet_environment::get_processor_name() + { + char name[1024 + 1] = {'\0'}; + const std::string rnkstr = std::to_string(rank()); + const int len = rnkstr.size(); + if(1025 < len) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::get_processor_name", + "GASNET processor name is larger than 1025"); + } + std::copy(std::begin(rnkstr), std::end(rnkstr), name); + return name; + } + + void gasnet_environment::finalize() + { + if (enabled() && has_called_init()) { + gasnet_exit(1); + } + } + + bool gasnet_environment::enabled() + { + return enabled_; + } + + bool gasnet_environment::multi_threaded() + { + return provided_threading_flag_ != GASNET_SEQ; + } + + bool gasnet_environment::has_called_init() + { + return has_called_init_; + } + + int gasnet_environment::size() + { + int res(-1); + if (enabled()) + res = static_cast(gasnet_nodes()); + return res; + } + + int gasnet_environment::rank() + { + int res(-1); + if (enabled()) + res = static_cast(gasnet_mynode()); + return res; + } + + gasnet_environment::scoped_lock::scoped_lock() + { + if (!multi_threaded()) + mtx_.lock(); + } + + gasnet_environment::scoped_lock::~scoped_lock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + void gasnet_environment::scoped_lock::unlock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + gasnet_environment::scoped_try_lock::scoped_try_lock() + : locked(true) + { + if (!multi_threaded()) + { + locked = mtx_.try_lock(); + } + } + + gasnet_environment::scoped_try_lock::~scoped_try_lock() + { + if (!multi_threaded() && locked) + mtx_.unlock(); + } + + void gasnet_environment::scoped_try_lock::unlock() + { + if (!multi_threaded() && locked) + { + locked = false; + mtx_.unlock(); + } + } +}} // namespace hpx::util + +#endif diff --git a/libs/core/gasnet_base/tests/CMakeLists.txt b/libs/core/gasnet_base/tests/CMakeLists.txt new file mode 100644 index 000000000000..3296f365ea31 --- /dev/null +++ b/libs/core/gasnet_base/tests/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(HPX_Option) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.mpi_base) + add_hpx_pseudo_dependencies(tests.unit.modules tests.unit.modules.mpi_base) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.mpi_base) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.mpi_base + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.mpi_base) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.mpi_base + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.mpi_base + HEADERS ${mpi_base_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + NOLIBS + DEPENDENCIES hpx_mpi_base + ) + endif() +endif() diff --git a/libs/core/gasnet_base/tests/performance/CMakeLists.txt b/libs/core/gasnet_base/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/gasnet_base/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/gasnet_base/tests/regressions/CMakeLists.txt b/libs/core/gasnet_base/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/gasnet_base/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/gasnet_base/tests/unit/CMakeLists.txt b/libs/core/gasnet_base/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/gasnet_base/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index d491915e7ba1..8e9a582b9e99 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -34,6 +34,7 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi + parcelport_gasnet parcelport_tcp parcelset parcelset_base diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt new file mode 100644 index 000000000000..67f8755cc7ea --- /dev/null +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2019-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET)) + return() +endif() + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +set(parcelport_gasnet_headers + hpx/parcelport_gasnet/header.hpp + hpx/parcelport_gasnet/locality.hpp + hpx/parcelport_gasnet/receiver.hpp + hpx/parcelport_gasnet/sender.hpp + hpx/parcelport_gasnet/sender_connection.hpp +) + +# cmake-format: off +set(parcelport_gasnet_compat_headers) +# cmake-format: on + +set(parcelport_gasnet_sources locality.cpp parcelport_gasnet.cpp) + +include(HPX_SetupGASNET) +hpx_setup_gasnet() + +include(HPX_AddModule) + +add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES gasnet hpx_core + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests +) + +set(HPX_STATIC_PARCELPORT_PLUGINS + ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_gasnet + CACHE INTERNAL "" FORCE +) diff --git a/libs/full/parcelport_gasnet/README.rst b/libs/full/parcelport_gasnet/README.rst new file mode 100644 index 000000000000..c0729a670a32 --- /dev/null +++ b/libs/full/parcelport_gasnet/README.rst @@ -0,0 +1,16 @@ + +.. + Copyright (c) 2020-2021 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +================= +parcelport_gasnet +================= + +This module is part of HPX. + +Documentation can be found `here +`__. diff --git a/libs/full/parcelport_gasnet/docs/index.rst b/libs/full/parcelport_gasnet/docs/index.rst new file mode 100644 index 000000000000..d61191bdd7d7 --- /dev/null +++ b/libs/full/parcelport_gasnet/docs/index.rst @@ -0,0 +1,18 @@ +.. + Copyright (c) 2020-2021 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _modules_parcelport_gasnet: + +============== +parcelport_gasnet +============== + +TODO: High-level description of the module. + +See the :ref:`API reference ` of this module for more +details. + diff --git a/libs/full/parcelport_gasnet/examples/CMakeLists.txt b/libs/full/parcelport_gasnet/examples/CMakeLists.txt new file mode 100644 index 000000000000..57dd90f4214e --- /dev/null +++ b/libs/full/parcelport_gasnet/examples/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.parcelport_gasnet) + add_hpx_pseudo_dependencies(examples.modules examples.modules.parcelport_gasnet) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.parcelport_gasnet) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.parcelport_gasnet + ) + endif() +endif() diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp new file mode 100644 index 000000000000..c336ac8defd0 --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -0,0 +1,148 @@ +// Copyright (c) 2013-2014 Hartmut Kaiser +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#include + +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::gasnet { + struct header + { + using value_type = int; + enum data_pos + { + pos_tag = 0 * sizeof(value_type), + pos_size = 1 * sizeof(value_type), + pos_numbytes = 2 * sizeof(value_type), + pos_numchunks_first = 3 * sizeof(value_type), + pos_numchunks_second = 4 * sizeof(value_type), + pos_piggy_back_flag = 5 * sizeof(value_type), + pos_piggy_back_data = 5 * sizeof(value_type) + 1 + }; + + static constexpr int data_size_ = 512; + + template + header(Buffer const& buffer, int tag) noexcept + { + std::int64_t size = static_cast(buffer.size_); + std::int64_t numbytes = + static_cast(buffer.data_size_); + + HPX_ASSERT(size <= (std::numeric_limits::max)()); + HPX_ASSERT(numbytes <= (std::numeric_limits::max)()); + + set(tag); + set(static_cast(size)); + set(static_cast(numbytes)); + set( + static_cast(buffer.num_chunks_.first)); + set( + static_cast(buffer.num_chunks_.second)); + + if (buffer.data_.size() <= (data_size_ - pos_piggy_back_data)) + { + data_[pos_piggy_back_flag] = 1; + std::memcpy(&data_[pos_piggy_back_data], &buffer.data_[0], + buffer.data_.size()); + } + else + { + data_[pos_piggy_back_flag] = 0; + } + } + + header() noexcept + { + reset(); + } + + void reset() noexcept + { + std::memset(&data_[0], -1, data_size_); + data_[pos_piggy_back_flag] = 1; + } + + bool valid() const noexcept + { + return data_[0] != -1; + } + + void assert_valid() const noexcept + { + HPX_ASSERT(tag() != -1); + HPX_ASSERT(size() != -1); + HPX_ASSERT(numbytes() != -1); + HPX_ASSERT(num_chunks().first != -1); + HPX_ASSERT(num_chunks().second != -1); + } + + constexpr char* data() noexcept + { + return &data_[0]; + } + + value_type tag() const noexcept + { + return get(); + } + + value_type size() const noexcept + { + return get(); + } + + value_type numbytes() const noexcept + { + return get(); + } + + std::pair num_chunks() const noexcept + { + return std::make_pair( + get(), get()); + } + + constexpr char* piggy_back() noexcept + { + if (data_[pos_piggy_back_flag]) + return &data_[pos_piggy_back_data]; + return nullptr; + } + + private: + std::array data_; + + template + void set(T const& t) noexcept + { + std::memcpy(&data_[Pos], &t, sizeof(t)); + } + + template + value_type get() const noexcept + { + value_type res; + std::memcpy(&res, &data_[Pos], sizeof(res)); + return res; + } + }; +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp new file mode 100644 index 000000000000..907412a5c91a --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -0,0 +1,70 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#include + +#include + +namespace hpx::parcelset::policies::gasnet { + + class locality + { + public: + constexpr locality() noexcept + : rank_(-1) + { + } + + explicit constexpr locality(std::int32_t rank) noexcept + : rank_(rank) + { + } + + constexpr std::int32_t rank() const noexcept + { + return rank_; + } + + static constexpr const char* type() noexcept + { + return "gasnet"; + } + + explicit constexpr operator bool() const noexcept + { + return rank_ != -1; + } + + HPX_EXPORT void save(serialization::output_archive& ar) const; + HPX_EXPORT void load(serialization::input_archive& ar); + + private: + friend bool operator==( + locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ == rhs.rank_; + } + + friend bool operator<(locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ < rhs.rank_; + } + + friend HPX_EXPORT std::ostream& operator<<( + std::ostream& os, locality const& loc) noexcept; + + std::int32_t rank_; + }; +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp new file mode 100644 index 000000000000..f93e37577dc0 --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -0,0 +1,222 @@ +// Copyright (c) 2007-2013 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::gasnet { + struct buffer_wrapper + { + struct fake_allocator + { + }; + using allocator_type = fake_allocator; + void* ptr; + size_t length; + buffer_wrapper() = default; + buffer_wrapper(const buffer_wrapper& wrapper) = default; + buffer_wrapper& operator=(const buffer_wrapper& wrapper) = default; + buffer_wrapper(const allocator_type& alloc) + { + HPX_UNUSED(alloc); + ptr = nullptr; + length = 0; + } + buffer_wrapper( + const buffer_wrapper& wrapper, const allocator_type& alloc) + { + HPX_UNUSED(alloc); + ptr = wrapper.ptr; + length = wrapper.length; + } + char& operator[](size_t i) const + { + HPX_ASSERT(i < length); + char* p = (char*) ptr; + return p[i]; + } + void* data() const + { + return ptr; + } + size_t size() const + { + return length; + } + }; + + struct request_wrapper_t + { + GASNET_request_t request; + request_wrapper_t() + { + request.flag = GASNET_ERR_RETRY; + } + ~request_wrapper_t() + { + if (request.flag == GASNET_OK) + { + if (request.type == GASNET_IOVEC) + { + for (int j = 0; j < request.data.iovec.count; ++j) + { + GASNET_lbuffer_free(request.data.iovec.lbuffers[j]); + } + free(request.data.iovec.lbuffers); + free(request.data.iovec.piggy_back.address); + } + else + { + HPX_ASSERT(request.type = GASNET_MEDIUM); + GASNET_mbuffer_free(request.data.mbuffer); + } + } + else + { + HPX_ASSERT(request.flag == GASNET_ERR_RETRY); + } + } + }; + + template + struct receiver + { + using buffer_type = parcel_buffer; + + explicit receiver(Parcelport& pp) noexcept + : pp_(pp) + { + } + + void run() noexcept {} + + bool background_work() noexcept + { + // We first try to accept a new connection + request_wrapper_t request; + GASNET_queue_pop(util::gasnet_environment::h_queue(), &request.request); + + if (request.request.flag == GASNET_OK) + { + buffer_type buffer = decode_request(request.request); + decode_parcels(pp_, HPX_MOVE(buffer), -1); + return true; + } + return false; + } + + buffer_type decode_request(GASNET_request_t request) + { + buffer_type buffer_; + header* header_; + // decode header + if (request.type == GASNET_MEDIUM) + { + // only header + header_ = (header*) (request.data.mbuffer.address); + header_->assert_valid(); + + HPX_ASSERT(header_->piggy_back()); + HPX_ASSERT(header_->num_chunks().first == 0); + } + else + { + // iovec + HPX_ASSERT(request.type == GASNET_IOVEC); + header_ = (header*) (request.data.iovec.piggy_back.address); + header_->assert_valid(); + } +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + hpx::chrono::high_resolution_timer timer_; + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds(); + data.bytes_ = static_cast(header_->numbytes()); +#endif + int i = 0; + // decode data + char* piggy_back = header_->piggy_back(); + if (piggy_back) + { + buffer_.data_.length = header_->size(); + buffer_.data_.ptr = piggy_back; + } + else + { + HPX_ASSERT(request.type == GASNET_IOVEC); + HPX_ASSERT((size_t) header_->size() == + request.data.iovec.lbuffers[i].length); + buffer_.data_.length = header_->size(); + buffer_.data_.ptr = request.data.iovec.lbuffers[i].address; + ++i; + } + // decode transmission chunk + buffer_.num_chunks_ = header_->num_chunks(); + int num_zero_copy_chunks = + static_cast(buffer_.num_chunks_.first); + int num_non_zero_copy_chunks = + static_cast(buffer_.num_chunks_.second); + auto& tchunks = buffer_.transmission_chunks_; + tchunks.resize(num_zero_copy_chunks + num_non_zero_copy_chunks); + if (num_zero_copy_chunks != 0) + { + HPX_ASSERT(request.type == GASNET_IOVEC); + buffer_.chunks_.resize(num_zero_copy_chunks); + int tchunks_length = static_cast(tchunks.size() * + sizeof(buffer_type::transmission_chunk_type)); + HPX_ASSERT((size_t) tchunks_length == + request.data.iovec.lbuffers[i].length); + std::memcpy((void*) tchunks.data(), + request.data.iovec.lbuffers[i].address, tchunks_length); + ++i; + // zero-copy chunks + for (int j = 0; j < num_zero_copy_chunks; ++j) + { + std::size_t chunk_size = + buffer_.transmission_chunks_[j].second; + HPX_ASSERT( + request.data.iovec.lbuffers[i].length == chunk_size); + buffer_wrapper& c = buffer_.chunks_[j]; + c.length = chunk_size; + c.ptr = request.data.iovec.lbuffers[i].address; + ++i; + } + } + HPX_ASSERT( + request.type == GASNET_MEDIUM || i == request.data.iovec.count); + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + data.time_ = timer_.elapsed_nanoseconds() - data.time_; +#endif + return buffer_; + } + + Parcelport& pp_; + }; + +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp new file mode 100644 index 000000000000..125da766998d --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -0,0 +1,103 @@ +// Copyright (c) 2007-2013 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::gasnet { + struct sender + { + using connection_type = sender_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + using mutex_type = hpx::spinlock; + + sender() noexcept {} + + void run() noexcept {} + + connection_ptr create_connection(int dest, parcelset::parcelport* pp) + { + return std::make_shared(this, dest, pp); + } + + void add(connection_ptr const& ptr) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(ptr); + } + + void send_messages(connection_ptr connection) + { + // Check if sending has been completed.... + if (connection->send()) + { + error_code ec(throwmode::lightweight); + hpx::move_only_function + postprocess_handler; + std::swap( + postprocess_handler, connection->postprocess_handler_); + postprocess_handler(ec, connection->destination(), connection); + } + else + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + bool background_work() noexcept + { + connection_ptr connection; + { + std::unique_lock l( + connections_mtx_, std::try_to_lock); + if (l.owns_lock() && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + bool has_work = false; + if (connection) + { + send_messages(HPX_MOVE(connection)); + has_work = true; + } + + return has_work; + } + + private: + + mutex_type connections_mtx_; + connection_list connections_; + }; + +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp new file mode 100644 index 000000000000..5f90a36568e0 --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -0,0 +1,221 @@ +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::gasnet { + struct sender; + struct sender_connection; + + void add_connection(sender*, std::shared_ptr const&); + + struct sender_connection + : parcelset::parcelport_connection> + { + private: + using sender_type = sender; + + using write_handler_type = + hpx::function; + + using data_type = std::vector; + + enum connection_state + { + initialized, + sent_header, + }; + + using base_type = + parcelset::parcelport_connection; + + public: + sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) + : state_(initialized) + , sender_(s) + , dst_rank(dst) + , pp_(pp) + , there_(parcelset::locality(locality(dst_rank))) + { + } + + parcelset::locality const& destination() const noexcept + { + return there_; + } + + constexpr void verify_( + parcelset::locality const& /* parcel_locality_id */) const noexcept + { + } + + template + void async_write( + Handler&& handler, ParcelPostprocess&& parcel_postprocess) + { + HPX_ASSERT(!handler_); + HPX_ASSERT(!postprocess_handler_); + HPX_ASSERT(!buffer_.data_.empty()); + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now(); +#endif + header_ = header(buffer_, 99); + header_.assert_valid(); + + // calculate how many long messages to send + int long_msg_num = 0; + // data + if (!header_.piggy_back()) + ++long_msg_num; + // transmission chunks + int num_zero_copy_chunks = + static_cast(buffer_.num_chunks_.first); + if (num_zero_copy_chunks != 0) + long_msg_num += num_zero_copy_chunks + 1; + + if(num_zero_copy_chunks) { + if(write_handles.size() != long_msg_num) { + write_handles.resize(long_msg_num); + } + + // transmission chunk + std::vector< + typename parcel_buffer_type::transmission_chunk_type>& + tchunks = buffer_.transmission_chunks_; + int tchunks_length = static_cast(tchunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type)); + write_handles[0] = gasnet_put(dst_rank, tchunks.data(), tchunks_length); + + std::size_t i = 1; + // zero-copy chunks + for (int j = 0; j < (int) buffer_.chunks_.size(); ++j) + { + serialization::serialization_chunk& c = + buffer_.chunks_[j]; + if (c.type_ == + serialization::chunk_type::chunk_type_pointer) + { + auto address = + const_cast(c.data_.cpos_); + auto length = c.size_; + write_handles[i] = gasnet_put(dst_rank, address, length); + ++i; + } + } + } + + state_ = initialized; + + handler_ = HPX_FORWARD(Handler, handler); + + if (!send()) + { + postprocess_handler_ = + HPX_FORWARD(ParcelPostprocess, parcel_postprocess); + add_connection(sender_, shared_from_this()); + } + else + { + HPX_ASSERT(!handler_); + error_code ec; + parcel_postprocess(ec, there_, shared_from_this()); + } + } + + bool send() + { + switch (state_) + { + case initialized: + return send_header(); + + case sent_header: + return done(); + + default: + HPX_ASSERT(false); + } + + return false; + } + + bool send_header() + { + if (gastnet_put(util::gasnet_environment::h_endpoint(), iovec, sync_, + dst_rank, 0, GASNET_DEFAULT_COMP_REMOTE, + nullptr) != GASNET_OK) + { + return false; + } + + state_ = sent_header; + return done(); + } + + bool done() + { + const int ret = gasnet_try_syncnb_all(write_handles.data(), write_handles.size()); + if(ret != GASNET_OK) + return false; + + error_code ec; + handler_(ec); + handler_.reset(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now() - + buffer_.data_point_.time_; + pp_->add_sent_data(buffer_.data_point_); +#endif + buffer_.clear(); + + state_ = initialized; + + return true; + } + + connection_state state_; + sender_type* sender_; + int dst_rank; + hpx::move_only_function handler_; + hpx::move_only_function)> + postprocess_handler_; + + header header_; + parcelset::parcelport* pp_; + + parcelset::locality there_; + std::vector write_handles; +(num_zero_copy_chunks); + }; +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/src/locality.cpp b/libs/full/parcelport_gasnet/src/locality.cpp new file mode 100644 index 000000000000..8af4b9a764aa --- /dev/null +++ b/libs/full/parcelport_gasnet/src/locality.cpp @@ -0,0 +1,37 @@ +// Copyright (c) 2013-2014 Thomas Heller +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#include +#include + +#include + +namespace hpx::parcelset::policies::gasnet { + + void locality::save(serialization::output_archive& ar) const + { + ar << rank_; + } + + void locality::load(serialization::input_archive& ar) + { + ar >> rank_; + } + + std::ostream& operator<<(std::ostream& os, locality const& loc) noexcept + { + hpx::util::ios_flags_saver ifs(os); + os << loc.rank_; + return os; + } +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp new file mode 100644 index 000000000000..90df494d1f9f --- /dev/null +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -0,0 +1,294 @@ +// Copyright (c) 2007-2013 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2020 Google +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace hpx::parcelset { + + namespace policies::gasnet { + class HPX_EXPORT parcelport; + } // namespace policies::gasnet + + template <> + struct connection_handler_traits + { + using connection_type = policies::gasnet::sender_connection; + using send_early_parcel = std::true_type; + using do_background_work = std::true_type; + using send_immediate_parcels = std::false_type; + + static constexpr const char* type() noexcept + { + return "gasnet"; + } + + static constexpr const char* pool_name() noexcept + { + return "parcel-pool-gasnet"; + } + + static constexpr const char* pool_name_postfix() noexcept + { + return "-gasnet"; + } + }; + + namespace policies::gasnet { + + void add_connection( + sender* s, std::shared_ptr const& ptr) + { + s->add(ptr); + } + + class HPX_EXPORT parcelport : public parcelport_impl + { + using base_type = parcelport_impl; + + static parcelset::locality here() + { + return parcelset::locality( + locality(util::gasnet_environment::enabled() ? + util::gasnet_environment::rank() : + -1)); + } + + static std::size_t max_connections( + util::runtime_configuration const& ini) + { + return hpx::util::get_entry_as(ini, + "hpx.parcel.gasnet.max_connections", + HPX_PARCEL_MAX_CONNECTIONS); + } + + public: + parcelport(util::runtime_configuration const& ini, + threads::policies::callback_notifier const& notifier) + : base_type(ini, here(), notifier) + , stopped_(false) + , receiver_(*this) + { + } + + ~parcelport() + { + util::gasnet_environment::finalize(); + } + + // Start the handling of connections. + bool do_run() + { + receiver_.run(); + sender_.run(); + for (std::size_t i = 0; i != io_service_pool_.size(); ++i) + { + io_service_pool_.get_io_service(int(i)).post( + hpx::bind(&parcelport::io_service_work, this)); + } + return true; + } + + // Stop the handling of connections. + void do_stop() + { + while (do_background_work(0, parcelport_background_mode_all)) + { + if (threads::get_self_ptr()) + hpx::this_thread::suspend( + hpx::threads::thread_schedule_state::pending, + "gasnet::parcelport::do_stop"); + } + stopped_ = true; + LCI_barrier(); + } + + /// Return the name of this locality + std::string get_locality_name() const override + { + // hostname-rank + return util::gasnet_environment::get_processor_name() + "-" + + std::to_string(util::gasnet_environment::rank()); + } + + std::shared_ptr create_connection( + parcelset::locality const& l, error_code&) + { + int dest_rank = l.get().rank(); + return sender_.create_connection(dest_rank, this); + } + + parcelset::locality agas_locality( + util::runtime_configuration const&) const override + { + return parcelset::locality( + locality(util::gasnet_environment::enabled() ? 0 : -1)); + } + + parcelset::locality create_locality() const override + { + return parcelset::locality(locality()); + } + + bool background_work( + std::size_t /* num_thread */, parcelport_background_mode mode) + { + if (stopped_) + return false; + + bool has_work = false; + if (mode & parcelport_background_mode_send) + { + has_work = sender_.background_work(); + } + if (mode & parcelport_background_mode_receive) + { + has_work = receiver_.background_work() || has_work; + } + return has_work; + } + + private: + using mutex_type = hpx::spinlock; + + std::atomic stopped_; + + sender sender_; + receiver receiver_; + + void io_service_work() + { + std::size_t k = 0; + // We only execute work on the IO service while HPX is starting + while (hpx::is_starting()) + { + bool has_work = sender_.background_work(); + has_work = receiver_.background_work() || has_work; + if (has_work) + { + k = 0; + } + else + { + ++k; + util::detail::yield_k(k, + "hpx::parcelset::policies::gasnet::parcelport::" + "io_service_work"); + } + } + } + + void early_write_handler(std::error_code const& ec, parcel const& p) + { + if (ec) + { + // all errors during early parcel handling are fatal + std::exception_ptr exception = hpx::detail::get_exception( + hpx::exception(ec), "gasnet::early_write_handler", + __FILE__, __LINE__, + "error while handling early parcel: " + ec.message() + + "(" + std::to_string(ec.value()) + ")" + + parcelset::dump_parcel(p)); + + hpx::report_error(exception); + } + } + }; + } // namespace policies::gasnet +} // namespace hpx::parcelset + +#include + +namespace hpx::traits { + // Inject additional configuration data into the factory registry for this + // type. This information ends up in the system wide configuration database + // under the plugin specific section: + // + // [hpx.parcel.gasnet] + // ... + // priority = 200 + // + template <> + struct plugin_config_data + { + static constexpr char const* priority() noexcept + { + return "50"; + } + + static void init( + int* argc, char*** argv, util::command_line_handling& cfg) + { + util::gasnet_environment::init(argc, argv, cfg.rtcfg_); + cfg.num_localities_ = + static_cast(util::gasnet_environment::size()); + cfg.node_ = static_cast(util::gasnet_environment::rank()); + } + + // TODO: implement creation of custom thread pool here + static void init(hpx::resource::partitioner&) noexcept {} + + static void destroy() + { + util::gasnet_environment::finalize(); + } + + static constexpr char const* call() noexcept + { + return + // TODO: change these for LCI +#if defined(HPX_HAVE_PARCELPORT_GASNET_ENV) + "env = " + "${HPX_HAVE_PARCELPORT_GASNET_ENV:" HPX_HAVE_PARCELPORT_GASNET_ENV + "}\n" +#else + "env = ${HPX_HAVE_PARCELPORT_GASNET_ENV:" + "MV2_COMM_WORLD_RANK,PMIX_RANK,PMI_RANK,GASNET_COMM_WORLD_SIZE," + "ALPS_APP_PE,PALS_NODEID" + "}\n" +#endif + "max_connections = " + "${HPX_HAVE_PARCELPORT_GASNET_MAX_CONNECTIONS:8192}\n"; + } + }; +} // namespace hpx::traits + +HPX_REGISTER_PARCELPORT(hpx::parcelset::policies::gasnet::parcelport, gasnet) + +#endif diff --git a/libs/full/parcelport_gasnet/tests/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/CMakeLists.txt new file mode 100644 index 000000000000..fed23bb2b5a5 --- /dev/null +++ b/libs/full/parcelport_gasnet/tests/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.parcelport_lci) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.parcelport_lci + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.parcelport_lci) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.parcelport_lci + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.parcelport_lci) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.parcelport_lci + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.parcelport_lci + HEADERS ${parcelport_lci_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + DEPENDENCIES hpx_parcelport_lci + ) + endif() +endif() diff --git a/libs/full/parcelport_gasnet/tests/performance/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_gasnet/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_gasnet/tests/regressions/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_gasnet/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_gasnet/tests/unit/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_gasnet/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From 3723fe7b0ed603fbc4bdc6eada472b6d3cfe12ca Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 16:48:36 -0400 Subject: [PATCH 002/333] updates --- cmake/FindGasnet.cmake | 85 +++- cmake/HPX_SetupGASNET.cmake | 69 ++- libs/core/gasnet_base/CMakeLists.txt | 69 ++- .../include/hpx/gasnet_base/gasnet.hpp | 1 + .../hpx/gasnet_base/gasnet_environment.hpp | 21 +- .../gasnet_base/src/gasnet_environment.cpp | 404 +++++++++++++++++- 6 files changed, 583 insertions(+), 66 deletions(-) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 9a20e8b33a5a..4ddf0e5baba1 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -10,37 +10,80 @@ set(GASNET_MPI_FOUND FALSE) set(GASNET_UDP_FOUND FALSE) set(GASNET_SMP_FOUND FALSE) -if(HPX_WITH_NETWORKING) - pkg_check_modules(GASNET QUIET gasnet-mpi-seq) +find_package(PkgConfig QUIET) + +if(HPX_WITH_PARCELPORT_GASNET_MPI) + pkg_check_modules(GASNET QUIET gasnet-mpi-par) if(NOT GASNET_FOUND) - pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") - else() - set(GASNET_UDP_FOUND TRUE) - endif() - else() - set(GASNET_MPI_FOUND TRUE) - hpx_setup_mpi() + hpx_error("Could not find GASNET MPI please set the PKG_CONFIG_PATH environment variable") endif() -else() - pkg_check_modules(GASNET QUIET gasnet-smp-seq) + + find_path( + GASNET_INCLUDE_DIR gasnet.h + HINTS ${GASNET_ROOT} ENV GASNETC_ROOT ${GASNET_DIR} ENV GASNET_DIR + PATH_SUFFIXES include + ) + + find_library( + GASNET_LIBRARY + NAMES gasnet-mpi-par + HINTS ${GASNET_ROOT} ENV GASNET_ROOT + PATH_SUFFIXES lib lib64 + ) + + set(GASNET_MPI_FOUND TRUE) + hpx_setup_mpi() +endif() + +if(HPX_WITH_PARCELPORT_GASNET_UDP) + pkg_check_modules(GASNET QUIET gasnet-udp-par) if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET please set the PKG_CONFIG_PATH environment variable") - else() - set(GASNET_SMP_FOUND TRUE) + hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") endif() + set(GASNET_UDP_FOUND TRUE) + + find_path( + GASNET_INCLUDE_DIR gasnet.h + HINTS ${GASNET_ROOT} ENV GASNETC_ROOT ${GASNET_DIR} ENV GASNET_DIR + PATH_SUFFIXES include + ) + + find_library( + GASNET_LIBRARY + NAMES gasnet-udp-par + HINTS ${GASNET_ROOT} ENV GASNET_ROOT + PATH_SUFFIXES lib lib64 + ) endif() -if(NOT GASNET_INCLUDE_DIRS OR NOT GASNET_LIBRARY_DIRS) - hpx_error("Could not find GASNET_INCLUDE_DIRS or GASNET_LIBRARY_DIRS please \ - set the PKG_CONFIG_PATH environment variable" - ) +if(HPX_WITH_PARCELPORT_GASNET_SMP) + pkg_check_modules(GASNET QUIET gasnet-smp-par) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET smp please set the PKG_CONFIG_PATH environment variable") + endif() + set(GASNET_SMP_FOUND TRUE) + + find_path( + GASNET_INCLUDE_DIR gasnet.h + HINTS ${GASNET_ROOT} ENV GASNETC_ROOT ${GASNET_DIR} ENV GASNET_DIR + PATH_SUFFIXES include + ) + + find_library( + GASNET_LIBRARY + NAMES gasnet-smp-par + HINTS ${GASNET_ROOT} ENV GASNET_ROOT + PATH_SUFFIXES lib lib64 + ) endif() +set(GASNET_LIBRARIES ${GASNET_LIBRARY}) +set(GASNET_INCLUDE_DIRS ${GASNET_INCLUDE_DIR}) + include(FindPackageHandleStandardArgs) find_package_handle_standard_args( - Gasnet DEFAULT_MSG GASNET_LIBRARY_DIRS GASNET_INCLUDE_DIRS + Gasnet DEFAULT_MSG GASNET_LIBRARY GASNET_INCLUDE_DIR ) +mark_as_advanced(GASNET_ROOT GASNET_LIBRARY GASNET_INCLUDE_DIR) endmacro() diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake index 37957d79ba46..3c9ab5b0f97b 100644 --- a/cmake/HPX_SetupGASNET.cmake +++ b/cmake/HPX_SetupGASNET.cmake @@ -14,22 +14,73 @@ macro(hpx_setup_gasnet) find_gasnet() set(GASNET_CXXFLAGS) - set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_CXXDEFS) + set(GASNET_INCLUDES) set(GASNET_LDFLAGS) + set(GASNET_LIBFLAGS) foreach(TOKEN ${GASNET_CFLAGS}) - list(APPEND GASNET_CXXFLAGS ${TOKEN}) + string(FIND "${TOKEN}" "-I" incpos) + string(FIND "${TOKEN}" "-l" lnkpos) + string(FIND "${TOKEN}" "-L" libpos) + string(FIND "${TOKEN}" "-D" defpos) + + if(NOT ${incpos} EQUAL -1) + string(REPLACE "-I" "" TOKEN "${TOKEN}") + list(APPEND GASNET_INCLUDES ${TOKEN}) + elseif(NOT ${lnkpos} EQUAL -1) + string(REPLACE "-l" "" TOKEN "${TOKEN}") + list(APPEND GASNET_LDFLAGS ${TOKEN}) + elseif(NOT ${libpos} EQUAL -1) + string(REPLACE "-L" "" TOKEN "${TOKEN}") + list(APPEND GASNET_LIBFLAGS ${TOKEN}) + elseif(NOT ${defpos} EQUAL -1) + string(REPLACE "-D" "" TOKEN "${TOKEN}") + list(APPEND GASNET_CXXDEFS ${TOKEN}) + else() + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endif() endforeach() - foreach(TOKEN ${GASNET_LDFLAGS_TMP}) - list(APPEND GASNET_LDFLAGS ${TOKEN}) + foreach(TOKEN ${GASNET_LDFLAGS}) + string(FIND "${TOKEN}" "-l" lnkpos) + if(NOT ${lnkpos} EQUAL -1) + list(APPEND GASNET_LIBFLAGS ${TOKEN}) + endif() endforeach() - add_library(gasnet INTERFACE IMPORTED) - target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) - target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) - target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) + list(APPEND GASNET_CXXDEFS "-DGASNET_PAR") + + #string (REPLACE ";" " " GASNET_CXXFLAGS "${GASNET_CXXFLAGS}") + #string (REPLACE ";" " " GASNET_CXXDEFS "${GASNET_CXXDEFS}") + #string (REPLACE ";" " " GASNET_INCLUDES "${GASNET_INCLUDES}") + #string (REPLACE ";" " " GASNET_LDFLAGS "${GASNET_LDFLAGS}") + #string (REPLACE ";" " " GASNET_LIBFLAGS "${GASNET_LIBFLAGS}") + + set(gasnet_libraries ${GASNET_LIBFLAGS}) + + if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) + add_library(gasnet-mpi-par INTERFACE IMPORTED) + target_compile_options(gasnet-mpi-par INTERFACE ${GASNET_CXXFLAGS}) + target_compile_definitions(gasnet-mpi-par INTERFACE ${GASNET_CXXDEFS}) + target_include_directories(gasnet-mpi-par SYSTEM INTERFACE ${GASNET_INCLUDES}) + target_link_options(gasnet-mpi-par INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet-mpi-par INTERFACE ${GASNET_LIBFLAGS}) + elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING) + add_library(gasnet-udp-par INTERFACE IMPORTED) + target_compile_options(gasnet-udp-par INTERFACE ${GASNET_CXXFLAGS}) + target_compile_definitions(gasnet-udp-par INTERFACE ${GASNET_CXXDEFS}) + target_include_directories(gasnet-udp-par SYSTEM INTERFACE${GASNET_INCLUDES}) + target_link_options(gasnet-udp-par INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet-udp-par INTERFACE ${GASNET_LIBFLAGS}) + else() + add_library(gasnet-smp-par INTERFACE IMPORTED) + target_compile_options(gasnet-smp-par INTERFACE ${GASNET_CXXFLAGS}) + target_compile_definitions(gasnet-smp-par INTERFACE ${GASNET_CXXDEFS}) + target_include_directories(gasnet-smp-par SYSTEM INTERFACE ${GASNET_INCLUDES}) + target_link_options(gasnet-smp-par INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet-smp-par INTERFACE ${GASNET_LIBFLAGS}) + endif() if(GASNET_MPI_FOUND) # Setup PMI imported target diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 1649612a6947..52e59d3b57b1 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -6,7 +6,13 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET_MPI) + # setup GASNET, if necessary + include(HPX_SetupGASNET) + hpx_setup_gasnet() +endif() + +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET_UDP) # setup GASNET, if necessary include(HPX_SetupGASNET) hpx_setup_gasnet() @@ -14,6 +20,16 @@ else() return() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET_SMP) + # setup GASNET, if necessary + include(HPX_SetupGASNET) + hpx_setup_gasnet() +else() + return() +endif() + +include(HPX_AddModule) + # Default location is $HPX_ROOT/libs/gasnet_base/include set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp hpx/gasnet_base/gasnet_environment.hpp) @@ -27,14 +43,43 @@ set(gasnet_base_compat_headers set(gasnet_base_sources gasnet_environment.cpp) -include(HPX_AddModule) -add_hpx_module( - core gasnet_base - GLOBAL_HEADER_GEN ON - SOURCES ${gasnet_base_sources} - HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} - MODULE_DEPENDENCIES gasnet hpx_logging hpx_runtime_configuration hpx_util - DEPENDENCIES ${additional_dependencies} gasnet - CMAKE_SUBDIRS examples tests -) +if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) + set(additional_dependencies Mpi::mpi gasnet-mpi-par) + + add_hpx_module( + core gasnet_base + GLOBAL_HEADER_GEN ON + SOURCES ${gasnet_base_sources} + HEADERS ${gasnet_base_headers} + COMPAT_HEADERS ${gasnet_base_compat_headers} + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util + DEPENDENCIES ${additional_dependencies} + CMAKE_SUBDIRS examples tests + ) +elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING) + set(additional_dependencies gasnet-udp-par) + + add_hpx_module( + core gasnet_base + GLOBAL_HEADER_GEN ON + SOURCES ${gasnet_base_sources} + HEADERS ${gasnet_base_headers} + COMPAT_HEADERS ${gasnet_base_compat_headers} + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util + DEPENDENCIES ${additional_dependencies} + CMAKE_SUBDIRS examples tests + ) +else() + set(additional_dependencies gasnet-smp-par) + + add_hpx_module( + core gasnet_base + GLOBAL_HEADER_GEN ON + SOURCES ${gasnet_base_sources} + HEADERS ${gasnet_base_headers} + COMPAT_HEADERS ${gasnet_base_compat_headers} + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util + DEPENDENCIES ${additional_dependencies} + CMAKE_SUBDIRS examples tests + ) +endif() diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp index 23ab2a16eb38..559f0cbf0346 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -17,6 +17,7 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif +#define GASNET_PAR 1 #include #if defined(__clang__) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp index e577e7f78c9e..3e9b40a8a8ff 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -9,11 +9,12 @@ #include -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) +#if (defined(HPX_HAVE_NETWORKING) && (defined(HPX_HAVE_PARCELPORT_GASNET_MPI) || defined(HPX_HAVE_PARCELPORT_GASNET_SMP))) #include #include #include +#include #include #include @@ -21,6 +22,7 @@ #include namespace hpx { namespace util { + struct HPX_CORE_EXPORT gasnet_environment { static bool check_gasnet_environment(runtime_configuration const& cfg); @@ -39,6 +41,14 @@ namespace hpx { namespace util { static std::string get_processor_name(); + static bool gettable(const int node, void* start, const size_t len); + + template + static void put(T * addr, const int rank, T * raddr, const std::size_t size); + + template + static void get(T * addr, const int rank, T * raddr, const std::size_t size); + struct HPX_CORE_EXPORT scoped_lock { scoped_lock(); @@ -60,8 +70,9 @@ namespace hpx { namespace util { typedef hpx::spinlock mutex_type; - private: - static mutex_type mtx_; + public: + static hpx::spinlock pollingLock; + static hpx::mutex mtx_; static bool enabled_; static bool has_called_init_; @@ -69,8 +80,10 @@ namespace hpx { namespace util { static int is_initialized_; - public: + static hpx::mutex dshm_mut; static int init_val_; + static hpx::mutex* segment_mutex; + static gasnet_seginfo_t* segments; }; }} // namespace hpx::util diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index b59d10d0cfb6..83ec81fdb774 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2013-2015 Thomas Heller // Copyright (c) 2020 Google // Copyright (c) 2022 Patrick Diehl -// Copyright (c) 2023 Tactical Computing Labs, LLC (Christopher Taylor) +// Copyright (c) 2023 Christopher Taylor // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -13,12 +13,189 @@ #include #include #include +#include #include #include #include #include +#include + +// +// AM functions +// +typedef enum { + SIGNAL = 136, // ack to a done_t via gasnet_AMReplyShortM() + SIGNAL_LONG, // ack to a done_t via gasnet_AMReplyLongM() + DO_REPLY_PUT = 143, // do a PUT here from another locale + DO_COPY_PAYLOAD // copy AM payload to another address +} AM_handler_function_idx_t; + +typedef struct { + void* ack; // acknowledgement object + void* tgt; // target memory address + void* src; // source memory address + size_t size; // number of bytes. +} xfer_info_t; + +// Gasnet AM handler arguments are only 32 bits, so here we have +// functions to get the 2 arguments for a 64-bit pointer, +// and a function to reconstitute the pointer from the 2 arguments. +// +static inline +gasnet_handlerarg_t get_arg_from_ptr0(uintptr_t addr) +{ + // This one returns the bottom 32 bits. + return ((gasnet_handlerarg_t) + ((((uint64_t) (addr)) << 32UL) >> 32UL)); +} +static inline +gasnet_handlerarg_t get_arg_from_ptr1(uintptr_t addr) +{ + // this one returns the top 32 bits. + return ((gasnet_handlerarg_t) + (((uint64_t) (addr)) >> 32UL)); +} +static inline +uintptr_t get_uintptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) +{ + return (uintptr_t) + (((uint64_t) (uint32_t) a0) + | (((uint64_t) (uint32_t) a1) << 32UL)); +} +static inline +void* get_ptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) +{ + return (void*) get_uintptr_from_args(a0, a1); +} + +// Build acknowledgement address arguments for gasnetAMRequest*() calls. +// +#define Arg0(addr) get_arg_from_ptr0((uintptr_t)addr) +#define Arg1(addr) get_arg_from_ptr1((uintptr_t)addr) + +// The following macro is from the GASNet test.h distribution +// +#define GASNET_Safe(fncall) do { \ + int _retval; \ + if ((_retval = fncall) != GASNET_OK) { \ + fprintf(stderr, "ERROR calling: %s\n" \ + " at: %s:%i\n" \ + " error: %s (%s)\n", \ + #fncall, __FILE__, __LINE__, \ + gasnet_ErrorName(_retval), gasnet_ErrorDesc(_retval)); \ + fflush(stderr); \ + gasnet_exit(_retval); \ + } \ + } while(0) + +// This is the type of object we use to manage GASNet acknowledgements. +// +// Initialize the count to 0, the target to the number of return signal +// events you expect, and the flag to 0. Fire the request, then do a +// BLOCKUNTIL(flag). When all the return signals have occurred, the AM +// handler will set the flag to 1 and your BLOCKUNTIL will complete. +// (Note that the GASNet documentation says that GASNet code assumes +// the condition for a BLOCKUNTIL can only be changed by the execution +// of an AM handler.) +// +typedef struct { + std::atomic count; + std::uint32_t target; + volatile int flag; +} done_t; + +static void AM_signal(gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { + done_t* done = (done_t*) get_ptr_from_args(a0, a1); + uint_least32_t prev; + prev = done->count.fetch_add(1, std::memory_order_seq_cst); + if (prev + 1 == done->target) + done->flag = 1; +} + +static void AM_signal_long(gasnet_token_t token, void *buf, size_t nbytes, + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { + done_t* done = (done_t*) get_ptr_from_args(a0, a1); + uint_least32_t prev; + prev = done->count.fetch_add(1, std::memory_order_seq_cst); + if (prev + 1 == done->target) + done->flag = 1; +} + +// Put from arg->src (which is local to the AM handler) back to +// arg->dst (which is local to the caller of this AM). +// nbytes is < gasnet_AMMaxLongReply here (see chpl_comm_get). +// +static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) { + xfer_info_t* x = static_cast(buf); + + assert(nbytes == sizeof(xfer_info_t)); + + GASNET_Safe(gasnet_AMReplyLong2(token, SIGNAL_LONG, + x->src, x->size, x->tgt, + Arg0(x->ack), Arg1(x->ack))); +} + +// Copy from the payload in this active message to dst. +// +static +void AM_copy_payload(gasnet_token_t token, void* buf, size_t nbytes, + gasnet_handlerarg_t ack0, gasnet_handlerarg_t ack1, + gasnet_handlerarg_t dst0, gasnet_handlerarg_t dst1) +{ + void* dst = get_ptr_from_args(dst0, dst1); + { + // would prefer to protect the memory segments + // associated with each node (n-node mutex) + // will require future work + // + std::lock_guard lk(hpx::util::gasnet_environment::dshm_mut); + std::memcpy(dst, buf, nbytes); + } + + GASNET_Safe(gasnet_AMReplyShort2(token, SIGNAL, ack0, ack1)); +} + +static gasnet_handlerentry_t ftable[] = { + {SIGNAL, (void (*)())&AM_signal}, + {SIGNAL_LONG, (void (*)())&AM_signal_long}, + {DO_REPLY_PUT, (void (*)())&AM_reply_put}, + {DO_COPY_PAYLOAD, (void (*)())&AM_copy_payload} +}; + +// +// Initialize one of the above. +// +static inline +void init_done_obj(done_t* done, int target) { + done->count.store(0, std::memory_order_seq_cst); + done->target = target; + done->flag = 0; +} + +static inline void am_poll_try() { + // Serialize polling for IBV, UCX, Aries, and OFI. Concurrent polling causes + // contention in these configurations. For other configurations that are + // AM-based (udp/amudp, mpi/ammpi) serializing can hurt performance. + // +#if defined(GASNET_CONDUIT_IBV) || defined(GASNET_CONDUIT_UCX) || defined(GASNET_CONDUIT_ARIES) || defined(GASNET_CONDUIT_OFI) + std::lock_guard lk(gasnet_environment::pollingLock); + (void) gasnet_AMPoll(); +#else + (void) gasnet_AMPoll(); +#endif +} + +static inline +void wait_done_obj(done_t* done, bool do_yield) +{ + while (!done->flag) { + am_poll_try(); + if (do_yield) + hpx::this_thread::yield(); + } +} /////////////////////////////////////////////////////////////////////////////// namespace hpx { namespace util { @@ -55,7 +232,8 @@ namespace hpx { namespace util { bool gasnet_environment::check_gasnet_environment( util::runtime_configuration const& cfg) { -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && ( defined(HPX_HAVE_PARCELPORT_GASNET_MPI) || \ + defined(HPX_HAVE_PARCELPORT_GASNET_SMP) ) // We disable the GASNET parcelport if any of these hold: // // - The parcelport is explicitly disabled @@ -74,27 +252,26 @@ namespace hpx { namespace util { } return true; -#elif defined(HPX_HAVE_MODULE_GASNET_BASE) - // if GASNET futures are enabled while networking is off we need to - // check whether we were run using gasnetrun - return detail::detect_gasnet_environment(cfg, HPX_HAVE_PARCELPORT_GASNET_ENV); -#else - return false; #endif } }} // namespace hpx::util -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && (defined(HPX_HAVE_PARCELPORT_GASNET_MPI) || \ + defined(HPX_HAVE_PARCELPORT_GASNET_SMP))) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) namespace hpx { namespace util { - gasnet_environment::mutex_type gasnet_environment::mtx_; + hpx::spinlock gasnet_environment::pollingLock{}; + hpx::mutex gasnet_environment::dshm_mut{}; + hpx::mutex gasnet_environment::mtx_{}; bool gasnet_environment::enabled_ = false; bool gasnet_environment::has_called_init_ = false; - int gasnet_environment::provided_threading_flag_ = GASNET_SEQ; + int gasnet_environment::provided_threading_flag_ = GASNET_PAR; int gasnet_environment::is_initialized_ = -1; int gasnet_environment::init_val_ = GASNET_ERR_RESOURCE; + hpx::mutex* gasnet_environment::segment_mutex = nullptr; + gasnet_seginfo_t* gasnet_environment::segments = nullptr; /////////////////////////////////////////////////////////////////////////// int gasnet_environment::init( @@ -132,6 +309,28 @@ namespace hpx { namespace util { "GASNET doesn't provide minimal requested thread level"); } + if(gasnet_attach(nullptr, 0, gasnet_getMaxLocalSegmentSize(), 0) != GASNET_OK) { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET failed to attach to memory"); + } + + // create a number of segments equal to the number of hardware + // threads per machine (locality) + // + gasnet_environment::segments = new gasnet_seginfo_t[size()]; + gasnet_environment::segment_mutex = new hpx::mutex[size()]; + + GASNET_Safe(gasnet_getSegmentInfo(segments, size())); + + int retval; + + gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); + while ((retval = gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS)) == GASNET_ERR_NOT_READY) {} + gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS); + + gasnet_set_waitmode(GASNET_WAIT_BLOCK); + return gasnet_environment::init_val_; } @@ -155,7 +354,7 @@ namespace hpx { namespace util { rtcfg.add_entry("hpx.parcel.bootstrap", "gasnet"); - int required = GASNET_SEQ; + int required = GASNET_PAR; int retval = init(argc, argv, required, required, provided_threading_flag_); if (GASNET_OK != retval) @@ -167,29 +366,35 @@ namespace hpx { namespace util { int msglen = 0; char message[1024 + 1]; - sprintf(message, "%s\n", gasnet_ErrorDesc(retval)); - msglen = strnlen(message, 1025); + std::snprintf(message, 1024+1, "%s\n", gasnet_ErrorDesc(retval)); + msglen = strnlen(message, 1025+1); std::string msg("gasnet_environment::init: gasnet_init failed: "); msg = msg + message + "."; throw std::runtime_error(msg.c_str()); } - if (provided_threading_flag_ != GASNET_SEQ) + if (provided_threading_flag_ != GASNET_PAR) { // explicitly disable gasnet if not run by gasnetrun rtcfg.add_entry("hpx.parcel.gasnet.multithreaded", "0"); } this_rank = rank(); + // need to figure this out to improve efficiency; create + // a number of segments equal to the number of hardware + // threads per machine (locality) + // + //segments.resize(hpx::threads::hardware_concurrency() * size()); + // + gasnet_environment::segments = new gasnet_seginfo_t[size()]; + gasnet_environment::segment_mutex = new hpx::mutex[size()]; #if defined(HPX_HAVE_NETWORKING) - if (this_rank == 0) - { + if (this_rank == 0) { rtcfg.mode_ = hpx::runtime_mode::console; } - else - { + else { rtcfg.mode_ = hpx::runtime_mode::worker; } #elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) @@ -216,6 +421,165 @@ namespace hpx { namespace util { return name; } + bool gettable(const int node, void* start, const size_t len) + { + const uintptr_t segstart = (uintptr_t) gasnet_environment::segments[node].addr; + const uintptr_t segend = segstart + gasnet_environment::segments[node].size; + const uintptr_t reqstart = (uintptr_t) start; + const uintptr_t reqend = reqstart + len; + + return ( segstart <= reqstart && + reqstart <= segend && + segstart <= reqend && + reqend <= segend); + } + + template + void gasnet_environment::put(T * addr, const int node, T * raddr, const std::size_t size) { + static_assert(std::is_integral::value, "Error! ***gasnet_environment::put***\tIntegral required."); + + const bool in_remote_seg = gettable(node, raddr, size); + if(in_remote_seg) { + const std::lock_guard lk(segment_mutex[node]); + gasnet_put(node, static_cast(raddr), static_cast(addr), size); + } + else { + // tell the remote node to copy the data being sent + // + size_t max_chunk = gasnet_AMMaxMedium(); + size_t start = 0; + + // AMRequestMedium will send put; the active message handler + // will memcpy on the remote host + // + for(start = 0; start < size; start += max_chunk) { + size_t this_size; + void* addr_chunk; + void* raddr_chunk; + done_t done; + + this_size = size - start; + if( this_size > max_chunk ) { + this_size = max_chunk; + } + + addr_chunk = ((char*) addr) + start; + raddr_chunk = ((char*) raddr) + start; + + init_done_obj(&done, 1); + + // Send an AM over to ask for a them to copy the data + // passed in the active message (addr_chunk) to raddr_chunk. + GASNET_Safe(gasnet_AMRequestMedium4(node, DO_COPY_PAYLOAD, + addr_chunk, this_size, + Arg0(&done), Arg1(&done), + Arg0(raddr_chunk), + Arg1(raddr_chunk))); + + // Wait for the PUT to complete. + wait_done_obj(&done, false); + } + } + } + + template + void gasnet_environment::get(T * addr, const int node, T * raddr, const std::size_t size) { + static_assert(std::is_integral::value, "Error! ***gasnet_environment::get***\tIntegral required."); + + bool remote_in_segment = false; + + if (rank() == node) { + std::memmove(addr, raddr, size); + } + else { + // Handle remote address not in remote segment. + // The GASNet Spec says: + // The source memory address for all gets and the target memory address + // for all puts must fall within the memory area registered for remote + // access by the remote node (see gasnet_attach()), or the results are + // undefined + // + // In other words, it is OK if the local side of a GET or PUT + // is not in the registered memory region. + // + remote_in_segment = gettable(node, raddr, size); + + if( remote_in_segment ) { + // If raddr is in the remote segment, do a normal gasnet_get. + // GASNet will handle the local portion not being in the segment. + // + gasnet_get(addr, node, raddr, size); // dest, node, src, size + } + else { + // If raddr is not in the remote segment, we need to send an + // active message; the other node will PUT back to us. + // The local side has to be in the registered memory segment. + // + bool local_in_segment = false; + void* local_buf = nullptr; + std::size_t max_chunk = gasnet_AMMaxLongReply(); + std::size_t start = 0; + + local_in_segment = gettable(rank(), addr, size); + + // If the local address isn't in a registered segment, + // do the GET into a temporary buffer instead, and then + // copy the result back. + // + if( ! local_in_segment ) { + size_t buf_sz = size; + if( buf_sz > max_chunk ) { + buf_sz = max_chunk; + } + + local_buf = calloc(1, buf_sz); + assert(gettable(chpl_nodeID, local_buf, buf_sz)); + } + + // do a PUT on the remote locale back to here. + // But do it in chunks of size gasnet_AMMaxLongReply() + // since we use gasnet_AMReplyLong to do the PUT. + for(start = 0; start < size; start += max_chunk) { + size_t this_size; + void* addr_chunk; + xfer_info_t info; + done_t done; + + this_size = size - start; + if( this_size > max_chunk ) { + this_size = max_chunk; + } + + addr_chunk = ((char*) addr) + start; + + init_done_obj(&done, 1); + + info.ack = &done; + info.tgt = local_buf?local_buf:addr_chunk; + info.src = ((char*) raddr) + start; + info.size = this_size; + + // Send an AM over to ask for a PUT back to us + GASNET_Safe(gasnet_AMRequestMedium0(node, DO_REPLY_PUT, + &info, sizeof(info))); + + // Wait for the PUT to complete. + wait_done_obj(&done, false); + + // Now copy from local_buf back to addr if necessary. + if( local_buf ) { + std::memcpy(addr_chunk, local_buf, this_size); + } + } + + // If we were using a temporary local buffer free it + if( local_buf ) { + free(local_buf); + } + } + } + } + void gasnet_environment::finalize() { if (enabled() && has_called_init()) { @@ -230,7 +594,7 @@ namespace hpx { namespace util { bool gasnet_environment::multi_threaded() { - return provided_threading_flag_ != GASNET_SEQ; + return provided_threading_flag_ != GASNET_PAR; } bool gasnet_environment::has_called_init() From b5f95ff8b869f34eee2878fc6a44df19ad6a99ac Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 19:49:22 -0400 Subject: [PATCH 003/333] updates --- libs/full/parcelport_gasnet/CMakeLists.txt | 54 +++- .../include/hpx/parcelport_gasnet/header.hpp | 8 +- .../hpx/parcelport_gasnet/locality.hpp | 8 +- .../hpx/parcelport_gasnet/receiver.hpp | 266 +++++++----------- .../parcelport_gasnet/receiver_connection.hpp | 251 +++++++++++++++++ .../include/hpx/parcelport_gasnet/sender.hpp | 95 +++++-- .../parcelport_gasnet/sender_connection.hpp | 209 ++++++++++---- .../hpx/parcelport_gasnet/tag_provider.hpp | 61 ++++ libs/full/parcelport_gasnet/src/locality.cpp | 6 +- .../src/parcelport_gasnet.cpp | 93 ++---- 10 files changed, 721 insertions(+), 330 deletions(-) create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp create mode 100644 libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 67f8755cc7ea..66070ae35e3e 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -4,9 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET)) - return() -endif() +if(HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_UDP)) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -29,18 +27,48 @@ hpx_setup_gasnet() include(HPX_AddModule) -add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES gasnet hpx_core - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests -) +if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + DEPENDENCIES hpx_core gasnet-mpi-par Mpi::mpi + CMAKE_SUBDIRS examples tests + ) +else() + if(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + DEPENDENCIES hpx_core gasnet-udp-par + CMAKE_SUBDIRS examples tests + ) + else() + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + DEPENDENCIES hpx_core gasnet-smp-par + CMAKE_SUBDIRS examples tests + ) + endif() +endif() set(HPX_STATIC_PARCELPORT_PLUGINS ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_gasnet CACHE INTERNAL "" FORCE ) + +else() + return() +endif() diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp index c336ac8defd0..707983ef2fbe 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -1,6 +1,6 @@ -// Copyright (c) 2013-2014 Hartmut Kaiser +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2013-2021 Hartmut Kaiser // Copyright (c) 2013-2015 Thomas Heller -// Copyright (c) 2023 Christopher Taylor // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -10,7 +10,7 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_PARCELPORT_GASNET) #include #include @@ -22,9 +22,11 @@ #include namespace hpx::parcelset::policies::gasnet { + struct header { using value_type = int; + enum data_pos { pos_tag = 0 * sizeof(value_type), diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index 907412a5c91a..4a03a45f2837 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -1,6 +1,6 @@ +// Copyright (c) 2023 Christopher Taylor // Copyright (c) 2007-2021 Hartmut Kaiser // Copyright (c) 2013-2014 Thomas Heller -// Copyright (c) 2023 Christopher Taylor // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -10,7 +10,7 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_PARCELPORT_GASNET) #include #include @@ -32,7 +32,7 @@ namespace hpx::parcelset::policies::gasnet { constexpr std::int32_t rank() const noexcept { - return rank_; + return static_cast(rank_); } static constexpr const char* type() noexcept @@ -63,7 +63,7 @@ namespace hpx::parcelset::policies::gasnet { friend HPX_EXPORT std::ostream& operator<<( std::ostream& os, locality const& loc) noexcept; - std::int32_t rank_; + gasnet_node_t rank_; }; } // namespace hpx::parcelset::policies::gasnet diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index f93e37577dc0..f66706e4e12c 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -1,6 +1,5 @@ -// Copyright (c) 2007-2013 Hartmut Kaiser +// Copyright (c) 2007-2021 Hartmut Kaiser // Copyright (c) 2014-2015 Thomas Heller -// Copyright (c) 2023 Christopher Taylor // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -10,15 +9,14 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) - +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_MPI) #include +#include + #include -#include +#include #include -#include -#include #include #include #include @@ -26,197 +24,139 @@ #include #include #include -#include +#include -namespace hpx::parcelset::policies::gasnet { - struct buffer_wrapper - { - struct fake_allocator - { - }; - using allocator_type = fake_allocator; - void* ptr; - size_t length; - buffer_wrapper() = default; - buffer_wrapper(const buffer_wrapper& wrapper) = default; - buffer_wrapper& operator=(const buffer_wrapper& wrapper) = default; - buffer_wrapper(const allocator_type& alloc) - { - HPX_UNUSED(alloc); - ptr = nullptr; - length = 0; - } - buffer_wrapper( - const buffer_wrapper& wrapper, const allocator_type& alloc) - { - HPX_UNUSED(alloc); - ptr = wrapper.ptr; - length = wrapper.length; - } - char& operator[](size_t i) const - { - HPX_ASSERT(i < length); - char* p = (char*) ptr; - return p[i]; - } - void* data() const - { - return ptr; - } - size_t size() const - { - return length; - } - }; +#include - struct request_wrapper_t - { - GASNET_request_t request; - request_wrapper_t() - { - request.flag = GASNET_ERR_RETRY; - } - ~request_wrapper_t() - { - if (request.flag == GASNET_OK) - { - if (request.type == GASNET_IOVEC) - { - for (int j = 0; j < request.data.iovec.count; ++j) - { - GASNET_lbuffer_free(request.data.iovec.lbuffers[j]); - } - free(request.data.iovec.lbuffers); - free(request.data.iovec.piggy_back.address); - } - else - { - HPX_ASSERT(request.type = GASNET_MEDIUM); - GASNET_mbuffer_free(request.data.mbuffer); - } - } - else - { - HPX_ASSERT(request.flag == GASNET_ERR_RETRY); - } - } - }; +namespace hpx::parcelset::policies::mpi { template struct receiver { - using buffer_type = parcel_buffer; + using header_list = std::list>; + using handles_header_type = std::set>; + using connection_type = receiver_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; - explicit receiver(Parcelport& pp) noexcept + explicit constexpr receiver(Parcelport& pp) noexcept : pp_(pp) + , hdr_request_(0) { } - void run() noexcept {} + void run() noexcept + { + util::gasnet_environment::scoped_lock l; + new_header(); + } bool background_work() noexcept { // We first try to accept a new connection - request_wrapper_t request; - GASNET_queue_pop(util::gasnet_environment::h_queue(), &request.request); + connection_ptr connection = accept(); - if (request.request.flag == GASNET_OK) + // If we don't have a new connection, try to handle one of the + // already accepted ones. + if (!connection) { - buffer_type buffer = decode_request(request.request); - decode_parcels(pp_, HPX_MOVE(buffer), -1); + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l.owns_lock() && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + if (connection) + { + receive_messages(HPX_MOVE(connection)); return true; } + return false; } - buffer_type decode_request(GASNET_request_t request) + void receive_messages(connection_ptr connection) noexcept { - buffer_type buffer_; - header* header_; - // decode header - if (request.type == GASNET_MEDIUM) - { - // only header - header_ = (header*) (request.data.mbuffer.address); - header_->assert_valid(); - - HPX_ASSERT(header_->piggy_back()); - HPX_ASSERT(header_->num_chunks().first == 0); - } - else + if (!connection->receive()) { - // iovec - HPX_ASSERT(request.type == GASNET_IOVEC); - header_ = (header*) (request.data.iovec.piggy_back.address); - header_->assert_valid(); + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); } -#if defined(HPX_HAVE_PARCELPORT_COUNTERS) - hpx::chrono::high_resolution_timer timer_; - parcelset::data_point& data = buffer_.data_point_; - data.time_ = timer_.elapsed_nanoseconds(); - data.bytes_ = static_cast(header_->numbytes()); -#endif - int i = 0; - // decode data - char* piggy_back = header_->piggy_back(); - if (piggy_back) - { - buffer_.data_.length = header_->size(); - buffer_.data_.ptr = piggy_back; - } - else + } + + connection_ptr accept() noexcept + { + std::unique_lock l(headers_mtx_, std::try_to_lock); + if (l.owns_lock()) { - HPX_ASSERT(request.type == GASNET_IOVEC); - HPX_ASSERT((size_t) header_->size() == - request.data.iovec.lbuffers[i].length); - buffer_.data_.length = header_->size(); - buffer_.data_.ptr = request.data.iovec.lbuffers[i].address; - ++i; + return accept_locked(l); } - // decode transmission chunk - buffer_.num_chunks_ = header_->num_chunks(); - int num_zero_copy_chunks = - static_cast(buffer_.num_chunks_.first); - int num_non_zero_copy_chunks = - static_cast(buffer_.num_chunks_.second); - auto& tchunks = buffer_.transmission_chunks_; - tchunks.resize(num_zero_copy_chunks + num_non_zero_copy_chunks); - if (num_zero_copy_chunks != 0) + return connection_ptr(); + } + + template + connection_ptr accept_locked(Lock& header_lock) noexcept + { + connection_ptr res; + util::mpi_environment::scoped_try_lock l; + + if (l.locked) { - HPX_ASSERT(request.type == GASNET_IOVEC); - buffer_.chunks_.resize(num_zero_copy_chunks); - int tchunks_length = static_cast(tchunks.size() * - sizeof(buffer_type::transmission_chunk_type)); - HPX_ASSERT((size_t) tchunks_length == - request.data.iovec.lbuffers[i].length); - std::memcpy((void*) tchunks.data(), - request.data.iovec.lbuffers[i].address, tchunks_length); - ++i; - // zero-copy chunks - for (int j = 0; j < num_zero_copy_chunks; ++j) + MPI_Status status; + if (request_done_locked(hdr_request_, &status)) { - std::size_t chunk_size = - buffer_.transmission_chunks_[j].second; - HPX_ASSERT( - request.data.iovec.lbuffers[i].length == chunk_size); - buffer_wrapper& c = buffer_.chunks_[j]; - c.length = chunk_size; - c.ptr = request.data.iovec.lbuffers[i].address; - ++i; + header h = new_header(); + l.unlock(); + header_lock.unlock(); + + res.reset(new connection_type(status.MPI_SOURCE, h, pp_)); + return res; } } - HPX_ASSERT( - request.type == GASNET_MEDIUM || i == request.data.iovec.count); + return res; + } + + header new_header() noexcept + { + header h = rcv_header_; + rcv_header_.reset(); + + ExpBackoff bo{0}; + + while(rcv_header_.data() == 0) { + gasnet_AMPoll(); + bo(); + } -#if defined(HPX_HAVE_PARCELPORT_COUNTERS) - data.time_ = timer_.elapsed_nanoseconds() - data.time_; -#endif - return buffer_; + return h; } Parcelport& pp_; - }; -} // namespace hpx::parcelset::policies::gasnet + hpx::spinlock headers_mtx_; + header rcv_header_; + + hpx::spinlock handles_header_mtx_; + handles_header_type handles_header_; + + hpx::spinlock connections_mtx_; + connection_list connections_; + + bool request_done_locked(MPI_Request& r, MPI_Status* status) noexcept + { + int completed = 0; + int ret = MPI_Test(&r, &completed, status); + HPX_ASSERT(ret == MPI_SUCCESS); + (void) ret; + if (completed) + { + return true; + } + return false; + } + }; +} // namespace hpx::parcelset::policies::mpi #endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp new file mode 100644 index 000000000000..9c874e003d64 --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -0,0 +1,251 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2007-2021 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_PARCELPORT_GASNET) +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace hpx::parcelset::policies::gasnet { + + struct ExpBackoff { + int numTries; + const static int maxRetries = 10; + + void operator()() { + if(numTries <= maxRetries) { + gasnet_AMPoll(); + hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); +  } + else { + numTries = 0; + } + } + }; + + template + struct receiver_connection + { + private: + enum connection_state + { + initialized, + rcvd_transmission_chunks, + rcvd_data, + rcvd_chunks, + sent_release_tag + }; + + using data_type = std::vector; + using buffer_type = parcel_buffer; + + public: + receiver_connection(int src, header h, Parcelport& pp) noexcept + : state_(initialized) + , src_(src) + , tag_(h.tag()) + , header_(h) + , request_(MPI_REQUEST_NULL) + , request_ptr_(false) + , chunks_idx_(0) + , pp_(pp) + { + header_.assert_valid(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds(); + data.bytes_ = static_cast(header_.numbytes()); +#endif + buffer_.data_.resize(static_cast(header_.size())); + buffer_.num_chunks_ = header_.num_chunks(); + } + + bool receive(std::size_t num_thread = -1) + { + switch (state_) + { + case initialized: + return receive_transmission_chunks(num_thread); + + case rcvd_transmission_chunks: + return receive_data(num_thread); + + case rcvd_data: + return receive_chunks(num_thread); + + case rcvd_chunks: + return send_release_tag(num_thread); + + case sent_release_tag: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool receive_transmission_chunks(std::size_t num_thread = -1) + { + auto self_ = util::gasnet_environment::rank(); + + // determine the size of the chunk buffer + std::size_t num_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.first)); + std::size_t num_non_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.second)); + buffer_.transmission_chunks_.resize( + num_zero_copy_chunks + num_non_zero_copy_chunks); + if (num_zero_copy_chunks != 0) + { + buffer_.chunks_.resize(num_zero_copy_chunks); + { + util::mpi_environment::scoped_lock l; + std::memcpy(buffer_.transmission_chunks_.data(), + util::gasnet_environment::gasnet_buffer[self_], + static_cast(buffer_.transmission_chunks_.size() * + sizeof(buffer_type::transmission_chunk_type))); + + request_ptr_ = true; + } + } + + state_ = rcvd_transmission_chunks; + + return receive_data(num_thread); + } + + bool receive_data(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } + + char* piggy_back = header_.piggy_back(); + if (piggy_back) + { + std::memcpy( + &buffer_.data_[0], piggy_back, buffer_.data_.size()); + } + else + { + auto self_ = util::gasnet_environment::rank(); + util::mpi_environment::scoped_lock l; + std::memcopy(buffer_.data_.data(), util::gasnet_environment::gasnet_buffer[self_], buffer_.data_.size()); + request_ptr_ = true; + } + + state_ = rcvd_data; + + return receive_chunks(num_thread); + } + + bool receive_chunks(std::size_t num_thread = -1) + { + + while (chunks_idx_ < buffer_.chunks_.size()) + { + if (!request_done()) + { + return false; + } + + std::size_t idx = chunks_idx_++; + std::size_t chunk_size = + buffer_.transmission_chunks_[idx].second; + + data_type& c = buffer_.chunks_[idx]; + c.resize(chunk_size); + { + auto self_ = util::gasnet_environment::rank(); + util::mpi_environment::scoped_lock l; + std::memcopy(c.data(), data.util::gasnet_environment::gasnet_buffer[self_], c.size()); + request_ptr_ = true; + } + } + + state_ = rcvd_chunks; + + return send_release_tag(num_thread); + } + + bool send_release_tag(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds() - data.time_; +#endif + { + auto self_ = util::gasnet_environment::rank(); + util::gasnet_environment::scoped_lock l; + std::memcopy(&tag_[src_], data.util::gasnet_environment::gasnet_buffer[self_], sizeof(int)); + request_ptr_ = true; + } + + decode_parcels(pp_, HPX_MOVE(buffer_), num_thread); + + state_ = sent_release_tag; + + return done(); + } + + bool done() noexcept + { + return request_done(); + } + + bool request_done() noexcept + { + util::gasnet_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return request_ptr_; + } + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + hpx::chrono::high_resolution_timer timer_; +#endif + connection_state state_; + + int src_; + int tag_; + header header_; + buffer_type buffer_; + + bool request_ptr_; + std::size_t chunks_idx_; + + Parcelport& pp_; + }; +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index 125da766998d..1acc76e1d4b8 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -1,6 +1,5 @@ -// Copyright (c) 2007-2013 Hartmut Kaiser +// Copyright (c) 2007-2021 Hartmut Kaiser // Copyright (c) 2014-2015 Thomas Heller -// Copyright (c) 2023 Christopher Taylor // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -10,13 +9,14 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) - +#if defined(HPX_HAVE_PARCELPORT_GASNET) #include -#include +#include +#include +#include -#include #include +#include #include #include @@ -25,18 +25,29 @@ #include #include +#include + namespace hpx::parcelset::policies::gasnet { + struct sender { using connection_type = sender_connection; using connection_ptr = std::shared_ptr; using connection_list = std::deque; - using mutex_type = hpx::spinlock; - - sender() noexcept {} + // different versions of clang-format disagree + // clang-format off + sender() noexcept + : next_free_tag_request_((MPI_Request) (-1)) + , next_free_tag_(-1) + { + } + // clang-format on - void run() noexcept {} + void run() noexcept + { + get_next_free_tag(); + } connection_ptr create_connection(int dest, parcelset::parcelport* pp) { @@ -45,10 +56,15 @@ namespace hpx::parcelset::policies::gasnet { void add(connection_ptr const& ptr) { - std::unique_lock l(connections_mtx_); + std::unique_lock l(connections_mtx_); connections_.push_back(ptr); } + int acquire_tag() noexcept + { + return tag_provider_.acquire(); + } + void send_messages(connection_ptr connection) { // Check if sending has been completed.... @@ -64,7 +80,7 @@ namespace hpx::parcelset::policies::gasnet { } else { - std::unique_lock l(connections_mtx_); + std::unique_lock l(connections_mtx_); connections_.push_back(HPX_MOVE(connection)); } } @@ -73,9 +89,8 @@ namespace hpx::parcelset::policies::gasnet { { connection_ptr connection; { - std::unique_lock l( - connections_mtx_, std::try_to_lock); - if (l.owns_lock() && !connections_.empty()) + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l && !connections_.empty()) { connection = HPX_MOVE(connections_.front()); connections_.pop_front(); @@ -88,16 +103,60 @@ namespace hpx::parcelset::policies::gasnet { send_messages(HPX_MOVE(connection)); has_work = true; } - + next_free_tag(); return has_work; } private: + tag_provider tag_provider_; + + void next_free_tag() noexcept + { + int next_free = -1; + { + std::unique_lock l(next_free_tag_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + next_free = next_free_tag_locked(); + } + } + + if (next_free != -1) + { + HPX_ASSERT(next_free > 1); + tag_provider_.release(next_free); + } + } + + int next_free_tag_locked() noexcept + { + util::gasnet_environment::scoped_try_lock l; + if (l.locked) + { + return get_next_free_tag(); + } + return -1; + } + + int get_next_free_tag() noexcept + { + int next_free = next_free_tag_; + + util::mpi_environment::scoped_lock l; + std::memcpy(&next_free, + util::gasnet_environment::gasnet_buffer[self_], + sizeof(int)); + + return next_free; + } - mutex_type connections_mtx_; + hpx::spinlock connections_mtx_; connection_list connections_; - }; + hpx::spinlock next_free_tag_mtx_; + MPI_Request next_free_tag_request_; + int next_free_tag_; + }; } // namespace hpx::parcelset::policies::gasnet #endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index 5f90a36568e0..0af40e7f6b89 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -1,3 +1,4 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser // Copyright (c) 2014-2015 Thomas Heller // Copyright (c) 2023 Christopher Taylor // @@ -12,6 +13,7 @@ #if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include +#include #include #include @@ -20,18 +22,20 @@ #include #include #include +#include #include -#include #include #include #include #include namespace hpx::parcelset::policies::gasnet { + struct sender; struct sender_connection; + int acquire_tag(sender*) noexcept; void add_connection(sender*, std::shared_ptr const&); struct sender_connection @@ -49,6 +53,9 @@ namespace hpx::parcelset::policies::gasnet { { initialized, sent_header, + sent_transmission_chunks, + sent_data, + sent_chunks }; using base_type = @@ -58,10 +65,17 @@ namespace hpx::parcelset::policies::gasnet { sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) : state_(initialized) , sender_(s) - , dst_rank(dst) + , tag_(-1) + , dst_(dst) + , chunks_idx_(0) + , ack_(0) , pp_(pp) - , there_(parcelset::locality(locality(dst_rank))) + , there_(parcelset::locality(locality(dst_))) { + gasnet_buffer.resize(hpx::util::gasnet_environment::size()); + for(std::size_t i = 0; i < gasnet_buffer.size(); ++i) { + gasnet_buffer[i] = static_cast(hpx::util::gasnet_environment::segments[i].addr); + } } parcelset::locality const& destination() const noexcept @@ -86,51 +100,11 @@ namespace hpx::parcelset::policies::gasnet { buffer_.data_point_.time_ = hpx::chrono::high_resolution_clock::now(); #endif - header_ = header(buffer_, 99); + chunks_idx_ = 0; + tag_ = acquire_tag(sender_); + header_ = header(buffer_, tag_); header_.assert_valid(); - // calculate how many long messages to send - int long_msg_num = 0; - // data - if (!header_.piggy_back()) - ++long_msg_num; - // transmission chunks - int num_zero_copy_chunks = - static_cast(buffer_.num_chunks_.first); - if (num_zero_copy_chunks != 0) - long_msg_num += num_zero_copy_chunks + 1; - - if(num_zero_copy_chunks) { - if(write_handles.size() != long_msg_num) { - write_handles.resize(long_msg_num); - } - - // transmission chunk - std::vector< - typename parcel_buffer_type::transmission_chunk_type>& - tchunks = buffer_.transmission_chunks_; - int tchunks_length = static_cast(tchunks.size() * - sizeof(parcel_buffer_type::transmission_chunk_type)); - write_handles[0] = gasnet_put(dst_rank, tchunks.data(), tchunks_length); - - std::size_t i = 1; - // zero-copy chunks - for (int j = 0; j < (int) buffer_.chunks_.size(); ++j) - { - serialization::serialization_chunk& c = - buffer_.chunks_[j]; - if (c.type_ == - serialization::chunk_type::chunk_type_pointer) - { - auto address = - const_cast(c.data_.cpos_); - auto length = c.size_; - write_handles[i] = gasnet_put(dst_rank, address, length); - ++i; - } - } - } - state_ = initialized; handler_ = HPX_FORWARD(Handler, handler); @@ -157,35 +131,139 @@ namespace hpx::parcelset::policies::gasnet { return send_header(); case sent_header: + return send_transmission_chunks(); + + case sent_transmission_chunks: + return send_data(); + + case sent_data: + return send_chunks(); + + case sent_chunks: return done(); default: HPX_ASSERT(false); } - return false; } bool send_header() { - if (gastnet_put(util::gasnet_environment::h_endpoint(), iovec, sync_, - dst_rank, 0, GASNET_DEFAULT_COMP_REMOTE, - nullptr) != GASNET_OK) { - return false; + util::gasnet_environment::scoped_lock l; + HPX_ASSERT(state_ == initialized); + + // compute + send the number of GASNET_PAGEs to send and the remainder number of bytes to a GASNET_PAGE + // + const std::size_t chunks[] = { header_.data_size_ / GASNET_PAGESIZE, header_.data_size_ % GASNET_PAGESIZE }; + std::copy(chunks, sizeof(chunks), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + + hpx::util::gasnet_environment::put( + gasnet_buffer[hpx::util::gasnet_environment::rank()], + dst_, + gasnet_buffer[dst_], + sizeof(chunks) + ); } state_ = sent_header; + return send_transmission_chunks(); + } + + bool send_transmission_chunks() + { + HPX_ASSERT(state_ == sent_header); + if (!request_done()) + { + return false; + } + + HPX_ASSERT(request_ptr_ == nullptr); + + std::vector& + chunks = buffer_.transmission_chunks_; + if (!chunks.empty()) + { + util::gasnet_environment::scoped_lock l; + std::copy_n(chunks.data(), chunks.size(), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + gasnet_put_bulk(dst_, gasnet_buffer[dst_], gasnet_buffer[hpx::util::gasnet_environment::rank()], + static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)) + ); + } + + state_ = sent_transmission_chunks; + return send_data(); + } + + bool send_data() + { + HPX_ASSERT(state_ == sent_transmission_chunks); + if (!request_done()) + { + return false; + } + + if (!header_.piggy_back()) + { + util::gasnet_environment::scoped_lock l; + std::copy_n(buffer_.data_.data(), buffer_.data_.size(), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + + hpx::util::gasnet_environment::put( + gasnet_buffer[hpx::util::gasnet_environment::rank()], + dst_, + gasnet_buffer[dst_], + sizeof(chunks) + ); + } + state_ = sent_data; + + return send_chunks(); + } + + bool send_chunks() + { + HPX_ASSERT(state_ == sent_data); + + while (chunks_idx_ < buffer_.chunks_.size()) + { + serialization::serialization_chunk& c = + buffer_.chunks_[chunks_idx_]; + if (c.type_ == serialization::chunk_type::chunk_type_pointer) + { + if (!request_done()) + { + return false; + } + + util::gasnet_environment::scoped_lock l; + + std::copy_n(const_cast(c.data_.c_pos_), static_cast(c.size_), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + + hpx::util::gasnet_environment::put( + gasnet_buffer[hpx::util::gasnet_environment::rank()], + dst_, + gasnet_buffer[dst_], + static_cast(c.size_) + ); + } + + chunks_idx_++; + } + + state_ = sent_chunks; + return done(); } bool done() { - const int ret = gasnet_try_syncnb_all(write_handles.data(), write_handles.size()); - if(ret != GASNET_OK) - return false; + if (!request_done()) + { + return false; + } - error_code ec; + error_code ec(throwmode::lightweight); handler_(ec); handler_.reset(); #if defined(HPX_HAVE_PARCELPORT_COUNTERS) @@ -201,20 +279,35 @@ namespace hpx::parcelset::policies::gasnet { return true; } + bool request_done() + { + util::gasnet_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return true; + } + connection_state state_; sender_type* sender_; - int dst_rank; + int tag_; + int dst_; hpx::move_only_function handler_; hpx::move_only_function)> postprocess_handler_; header header_; + std::vector gasnet_buffer; + + std::size_t chunks_idx_; + char ack_; + parcelset::parcelport* pp_; parcelset::locality there_; - std::vector write_handles; -(num_zero_copy_chunks); }; } // namespace hpx::parcelset::policies::gasnet diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp new file mode 100644 index 000000000000..45ba94918f65 --- /dev/null +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp @@ -0,0 +1,61 @@ +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_PARCELPORT_GASNET) +#include +#include + +#include +#include +#include + +namespace hpx::parcelset::policies::gasnet { + + struct tag_provider + { + tag_provider() noexcept + : next_tag_(2) + { + } + + int acquire() noexcept + { + int tag = -1; + std::lock_guard l(mtx_); + if (free_tags_.empty()) + { + HPX_ASSERT(next_tag_ < (std::numeric_limits::max)()); + tag = next_tag_++; + } + else + { + tag = free_tags_.front(); + free_tags_.pop_front(); + } + HPX_ASSERT(tag > 1); + return tag; + } + + void release(int tag) + { + HPX_ASSERT(tag > 1); + std::lock_guard l(mtx_); + HPX_ASSERT(tag < next_tag_); + + free_tags_.push_back(tag); + } + + hpx::spinlock mtx_; + int next_tag_; + std::deque free_tags_; + }; +} // namespace hpx::parcelset::policies::gasnet + +#endif diff --git a/libs/full/parcelport_gasnet/src/locality.cpp b/libs/full/parcelport_gasnet/src/locality.cpp index 8af4b9a764aa..553a1e661608 100644 --- a/libs/full/parcelport_gasnet/src/locality.cpp +++ b/libs/full/parcelport_gasnet/src/locality.cpp @@ -1,6 +1,6 @@ -// Copyright (c) 2013-2014 Thomas Heller +// Copyright (c) 2023 Christopher Taylor // Copyright (c) 2007-2021 Hartmut Kaiser -// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2013-2014 Thomas Heller // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -8,7 +8,7 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_PARCELPORT_GASNET) #include #include diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 90df494d1f9f..9314da7f1434 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -1,7 +1,7 @@ -// Copyright (c) 2007-2013 Hartmut Kaiser +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2022 Hartmut Kaiser // Copyright (c) 2014-2015 Thomas Heller // Copyright (c) 2020 Google -// Copyright (c) 2023 Christopher Taylor // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -9,10 +9,11 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_PARCELPORT_GASNET) #include #include #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include -#include #include #include #include @@ -72,6 +72,11 @@ namespace hpx::parcelset { namespace policies::gasnet { + int acquire_tag(sender* s) noexcept + { + return s->acquire_tag(); + } + void add_connection( sender* s, std::shared_ptr const& ptr) { @@ -117,6 +122,7 @@ namespace hpx::parcelset { { receiver_.run(); sender_.run(); + for (std::size_t i = 0; i != io_service_pool_.size(); ++i) { io_service_pool_.get_io_service(int(i)).post( @@ -128,6 +134,7 @@ namespace hpx::parcelset { // Stop the handling of connections. void do_stop() { + int retval; while (do_background_work(0, parcelport_background_mode_all)) { if (threads::get_self_ptr()) @@ -135,16 +142,23 @@ namespace hpx::parcelset { hpx::threads::thread_schedule_state::pending, "gasnet::parcelport::do_stop"); } + stopped_ = true; - LCI_barrier(); + gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); + if((retval = gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) { + // throw exception + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET failed ", std::to_string(gasnetErrorName(retval)), " ", + std::to_string(gasnetErrorDesc(retval))); + gasnet_exit(retval); + } } /// Return the name of this locality std::string get_locality_name() const override { - // hostname-rank - return util::gasnet_environment::get_processor_name() + "-" + - std::to_string(util::gasnet_environment::rank()); + return util::gasnet_environment::get_processor_name(); } std::shared_ptr create_connection( @@ -170,7 +184,9 @@ namespace hpx::parcelset { std::size_t /* num_thread */, parcelport_background_mode mode) { if (stopped_) + { return false; + } bool has_work = false; if (mode & parcelport_background_mode_send) @@ -185,8 +201,6 @@ namespace hpx::parcelset { } private: - using mutex_type = hpx::spinlock; - std::atomic stopped_; sender sender_; @@ -195,6 +209,7 @@ namespace hpx::parcelset { void io_service_work() { std::size_t k = 0; + // We only execute work on the IO service while HPX is starting while (hpx::is_starting()) { @@ -233,62 +248,4 @@ namespace hpx::parcelset { } // namespace policies::gasnet } // namespace hpx::parcelset -#include - -namespace hpx::traits { - // Inject additional configuration data into the factory registry for this - // type. This information ends up in the system wide configuration database - // under the plugin specific section: - // - // [hpx.parcel.gasnet] - // ... - // priority = 200 - // - template <> - struct plugin_config_data - { - static constexpr char const* priority() noexcept - { - return "50"; - } - - static void init( - int* argc, char*** argv, util::command_line_handling& cfg) - { - util::gasnet_environment::init(argc, argv, cfg.rtcfg_); - cfg.num_localities_ = - static_cast(util::gasnet_environment::size()); - cfg.node_ = static_cast(util::gasnet_environment::rank()); - } - - // TODO: implement creation of custom thread pool here - static void init(hpx::resource::partitioner&) noexcept {} - - static void destroy() - { - util::gasnet_environment::finalize(); - } - - static constexpr char const* call() noexcept - { - return - // TODO: change these for LCI -#if defined(HPX_HAVE_PARCELPORT_GASNET_ENV) - "env = " - "${HPX_HAVE_PARCELPORT_GASNET_ENV:" HPX_HAVE_PARCELPORT_GASNET_ENV - "}\n" -#else - "env = ${HPX_HAVE_PARCELPORT_GASNET_ENV:" - "MV2_COMM_WORLD_RANK,PMIX_RANK,PMI_RANK,GASNET_COMM_WORLD_SIZE," - "ALPS_APP_PE,PALS_NODEID" - "}\n" -#endif - "max_connections = " - "${HPX_HAVE_PARCELPORT_GASNET_MAX_CONNECTIONS:8192}\n"; - } - }; -} // namespace hpx::traits - -HPX_REGISTER_PARCELPORT(hpx::parcelset::policies::gasnet::parcelport, gasnet) - #endif From e1cbe9e2767c2ca52397f5cc8a6df2838cd3f83b Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 20:22:34 -0400 Subject: [PATCH 004/333] fixed some misplaced mpi references --- .../include/hpx/parcelport_gasnet/locality.hpp | 1 + .../include/hpx/parcelport_gasnet/receiver.hpp | 17 +++++++---------- .../parcelport_gasnet/receiver_connection.hpp | 6 +++--- .../include/hpx/parcelport_gasnet/sender.hpp | 6 ++---- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index 4a03a45f2837..fe1749e44de7 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -12,6 +12,7 @@ #if defined(HPX_HAVE_PARCELPORT_GASNET) #include +#include #include diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index f66706e4e12c..5defa4393111 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -9,9 +9,9 @@ #include -#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_MPI) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include -#include +#include #include #include @@ -26,9 +26,7 @@ #include #include -#include - -namespace hpx::parcelset::policies::mpi { +namespace hpx::parcelset::policies::gasnet { template struct receiver @@ -100,18 +98,17 @@ namespace hpx::parcelset::policies::mpi { connection_ptr accept_locked(Lock& header_lock) noexcept { connection_ptr res; - util::mpi_environment::scoped_try_lock l; + util::gasnet_environment::scoped_try_lock l; if (l.locked) { - MPI_Status status; - if (request_done_locked(hdr_request_, &status)) + if (request_done_locked(hdr_request_)) { header h = new_header(); l.unlock(); header_lock.unlock(); - res.reset(new connection_type(status.MPI_SOURCE, h, pp_)); + res.reset(new connection_type(h, pp_)); return res; } } @@ -157,6 +154,6 @@ namespace hpx::parcelset::policies::mpi { return false; } }; -} // namespace hpx::parcelset::policies::mpi +} // namespace hpx::parcelset::policies::gasnet #endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index 9c874e003d64..9d1209f5c0fd 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -121,7 +121,7 @@ namespace hpx::parcelset::policies::gasnet { { buffer_.chunks_.resize(num_zero_copy_chunks); { - util::mpi_environment::scoped_lock l; + util::gasnet_environment::scoped_lock l; std::memcpy(buffer_.transmission_chunks_.data(), util::gasnet_environment::gasnet_buffer[self_], static_cast(buffer_.transmission_chunks_.size() * @@ -152,7 +152,7 @@ namespace hpx::parcelset::policies::gasnet { else { auto self_ = util::gasnet_environment::rank(); - util::mpi_environment::scoped_lock l; + util::gasnet_environment::scoped_lock l; std::memcopy(buffer_.data_.data(), util::gasnet_environment::gasnet_buffer[self_], buffer_.data_.size()); request_ptr_ = true; } @@ -180,7 +180,7 @@ namespace hpx::parcelset::policies::gasnet { c.resize(chunk_size); { auto self_ = util::gasnet_environment::rank(); - util::mpi_environment::scoped_lock l; + util::gasnet_environment::scoped_lock l; std::memcopy(c.data(), data.util::gasnet_environment::gasnet_buffer[self_], c.size()); request_ptr_ = true; } diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index 1acc76e1d4b8..b66a6e9a96f2 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -12,7 +12,7 @@ #if defined(HPX_HAVE_PARCELPORT_GASNET) #include #include -#include +#include #include #include @@ -25,8 +25,6 @@ #include #include -#include - namespace hpx::parcelset::policies::gasnet { struct sender @@ -142,7 +140,7 @@ namespace hpx::parcelset::policies::gasnet { { int next_free = next_free_tag_; - util::mpi_environment::scoped_lock l; + util::gasnet_environment::scoped_lock l; std::memcpy(&next_free, util::gasnet_environment::gasnet_buffer[self_], sizeof(int)); From 964b06b93214da965b431648689d13a58baa2e61 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 21:03:30 -0400 Subject: [PATCH 005/333] fixed issues in receiver --- .../hpx/parcelport_gasnet/receiver.hpp | 47 ++++++++++--------- .../parcelport_gasnet/receiver_connection.hpp | 15 ------ 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index 5defa4393111..897eb6356b9b 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -28,6 +28,21 @@ namespace hpx::parcelset::policies::gasnet { + struct exp_backoff { + int numTries; + const static int maxRetries = 10; + + void operator()() { + if(numTries <= maxRetries) { + gasnet_AMPoll(); + hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); +  } + else { + numTries = 0; + } + } + }; + template struct receiver { @@ -102,15 +117,14 @@ namespace hpx::parcelset::policies::gasnet { if (l.locked) { - if (request_done_locked(hdr_request_)) - { - header h = new_header(); - l.unlock(); - header_lock.unlock(); - - res.reset(new connection_type(h, pp_)); - return res; - } + header h = new_header(); + l.unlock(); + header_lock.unlock(); + + // remote localities 'put' into the gasnet shared memory segment on this machine + // + res.reset(new connection_type(hpx::util::gasnet_environment::rank(), h, pp_)); + return res; } return res; } @@ -120,7 +134,7 @@ namespace hpx::parcelset::policies::gasnet { header h = rcv_header_; rcv_header_.reset(); - ExpBackoff bo{0}; + exp_backoff bo{0}; while(rcv_header_.data() == 0) { gasnet_AMPoll(); @@ -140,19 +154,6 @@ namespace hpx::parcelset::policies::gasnet { hpx::spinlock connections_mtx_; connection_list connections_; - - bool request_done_locked(MPI_Request& r, MPI_Status* status) noexcept - { - int completed = 0; - int ret = MPI_Test(&r, &completed, status); - HPX_ASSERT(ret == MPI_SUCCESS); - (void) ret; - if (completed) - { - return true; - } - return false; - } }; } // namespace hpx::parcelset::policies::gasnet diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index 9d1209f5c0fd..d2e015381f6f 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -29,21 +29,6 @@ namespace hpx::parcelset::policies::gasnet { - struct ExpBackoff { - int numTries; - const static int maxRetries = 10; - - void operator()() { - if(numTries <= maxRetries) { - gasnet_AMPoll(); - hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); -  } - else { - numTries = 0; - } - } - }; - template struct receiver_connection { From b7c4bb3b605beaa0f9f7414ae47b4d665a786e5d Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 21:05:00 -0400 Subject: [PATCH 006/333] documented sender --- .../include/hpx/parcelport_gasnet/sender_connection.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index 0af40e7f6b89..f5f6c16673a3 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -159,6 +159,9 @@ namespace hpx::parcelset::policies::gasnet { const std::size_t chunks[] = { header_.data_size_ / GASNET_PAGESIZE, header_.data_size_ % GASNET_PAGESIZE }; std::copy(chunks, sizeof(chunks), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + // put from this localities gasnet shared memory segment + // into the remote locality (dst_)'s shared memory segment + // hpx::util::gasnet_environment::put( gasnet_buffer[hpx::util::gasnet_environment::rank()], dst_, From b27e8ed5e2b16ea57112690105aa1b5e712923b4 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 21:42:33 -0400 Subject: [PATCH 007/333] rm'd compat script --- libs/core/gasnet_base/CMakeLists.txt | 8 -------- .../include/hpx/parcelport_gasnet/locality.hpp | 3 ++- libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp | 3 ++- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 52e59d3b57b1..a7e66517d2eb 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -33,14 +33,6 @@ include(HPX_AddModule) # Default location is $HPX_ROOT/libs/gasnet_base/include set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp hpx/gasnet_base/gasnet_environment.hpp) -# Default location is $HPX_ROOT/libs/gasnet_base/include_compatibility -# cmake-format: off -set(gasnet_base_compat_headers - hpx/plugins/parcelport/gasnet/gasnet.hpp => hpx/modules/gasnet_base.hpp - hpx/plugins/parcelport/gasnet/gasnet_environment.hpp => hpx/modules/gasnet_base.hpp -) -# cmake-format: on - set(gasnet_base_sources gasnet_environment.cpp) if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index fe1749e44de7..08be691e150b 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -12,7 +12,8 @@ #if defined(HPX_HAVE_PARCELPORT_GASNET) #include -#include +#include +#include #include diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 9314da7f1434..5863dfefcd3e 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -26,6 +25,8 @@ #include #include #include +#include +#include #include #include #include From 075dded363bc7fe8ea342bbf5a56b2e1e0c0f144 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Fri, 5 May 2023 01:00:07 -0400 Subject: [PATCH 008/333] updates to gasnet draft PR --- CMakeLists.txt | 28 +++++- cmake/FindGasnet.cmake | 89 ++++++++++++----- cmake/HPX_SetupGASNET.cmake | 95 ++----------------- cmake/templates/HPXConfig.cmake.in | 6 ++ libs/core/gasnet_base/CMakeLists.txt | 46 ++------- libs/core/gasnet_base/examples/CMakeLists.txt | 8 +- .../include/hpx/gasnet_base/gasnet.hpp | 3 + .../hpx/gasnet_base/gasnet_environment.hpp | 5 +- .../gasnet_base/src/gasnet_environment.cpp | 7 +- libs/core/gasnet_base/tests/CMakeLists.txt | 18 ++-- libs/core/version/src/version.cpp | 18 ++++ .../full/command_line_handling/CMakeLists.txt | 12 +++ .../src/command_line_handling.cpp | 3 + libs/full/parcelport_gasnet/CMakeLists.txt | 56 +++++------ .../hpx/parcelport_gasnet/locality.hpp | 4 +- .../src/parcelport_gasnet.cpp | 44 +++++++++ 16 files changed, 241 insertions(+), 201 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f0da9d3a886d..288d36748a6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1151,6 +1151,30 @@ if(HPX_WITH_NETWORKING) hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) endif() + hpx_option( + HPX_WITH_PARCELPORT_GASNET_MPI BOOL "Enable the GASNET-MPI based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET_MPI) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_MPI) + endif() + + hpx_option( + HPX_WITH_PARCELPORT_GASNET_SMP BOOL "Enable the GASNET-SMP based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET_SMP) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_SMP) + endif() + + hpx_option( + HPX_WITH_PARCELPORT_GASNET_UDP BOOL "Enable the GASNET-UDP based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET_UDP) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) + endif() + hpx_option( HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON CATEGORY "Parcelport" @@ -1188,7 +1212,7 @@ else(HPX_WITH_NETWORKING) ) endif(HPX_WITH_NETWORKING) -if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI) +if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI OR HPX_WITH_PARCELPORT_GASNET_MPI) # mpi parcelport settings hpx_option( HPX_WITH_PARCELPORT_MPI_ENV @@ -2133,7 +2157,7 @@ add_hpx_library_headers_noglob(hpx_external) # Setup plugins (set here cause if we include it inside plugins, it will not be # defined in src/CMakeLists.txt where we call add_static_parcelports) include(HPX_SetupMPI) # must come before APEX -if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI) +if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI OR HPX_WITH_PARCELPORT_GASNET_MPI) hpx_setup_mpi() endif() include(HPX_SetupLCI) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 4ddf0e5baba1..1013c81cdc3e 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -6,6 +6,7 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) macro(find_gasnet) + set(GASNET_MPI_FOUND FALSE) set(GASNET_UDP_FOUND FALSE) set(GASNET_SMP_FOUND FALSE) @@ -13,21 +14,20 @@ set(GASNET_SMP_FOUND FALSE) find_package(PkgConfig QUIET) if(HPX_WITH_PARCELPORT_GASNET_MPI) - pkg_check_modules(GASNET QUIET gasnet-mpi-par) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET MPI please set the PKG_CONFIG_PATH environment variable") - endif() + pkg_check_modules(PC_GASNET QUIET gasnet-mpi-par) find_path( GASNET_INCLUDE_DIR gasnet.h - HINTS ${GASNET_ROOT} ENV GASNETC_ROOT ${GASNET_DIR} ENV GASNET_DIR + HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_INCLUDEDIR} + ${PC_GASNET_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( GASNET_LIBRARY NAMES gasnet-mpi-par - HINTS ${GASNET_ROOT} ENV GASNET_ROOT + HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_LIBDIR} + ${PC_GASNET_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) @@ -37,53 +37,98 @@ endif() if(HPX_WITH_PARCELPORT_GASNET_UDP) pkg_check_modules(GASNET QUIET gasnet-udp-par) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") - endif() - set(GASNET_UDP_FOUND TRUE) find_path( GASNET_INCLUDE_DIR gasnet.h - HINTS ${GASNET_ROOT} ENV GASNETC_ROOT ${GASNET_DIR} ENV GASNET_DIR + HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_INCLUDEDIR} + ${PC_GASNET_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( GASNET_LIBRARY NAMES gasnet-udp-par - HINTS ${GASNET_ROOT} ENV GASNET_ROOT + HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_LIBDIR} + ${PC_GASNET_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) + + set(GASNET_UDP_FOUND TRUE) endif() if(HPX_WITH_PARCELPORT_GASNET_SMP) pkg_check_modules(GASNET QUIET gasnet-smp-par) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET smp please set the PKG_CONFIG_PATH environment variable") - endif() - set(GASNET_SMP_FOUND TRUE) find_path( GASNET_INCLUDE_DIR gasnet.h - HINTS ${GASNET_ROOT} ENV GASNETC_ROOT ${GASNET_DIR} ENV GASNET_DIR + HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_INCLUDEDIR} + ${PC_GASNET_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( GASNET_LIBRARY NAMES gasnet-smp-par - HINTS ${GASNET_ROOT} ENV GASNET_ROOT + HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_LIBDIR} + ${PC_GASNET_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) + + set(GASNET_SMP_FOUND TRUE) +endif() + + +# Set GASNET_ROOT in case the other hints are used +if(NOT GASNET_ROOT AND "$ENV{GASNET_ROOT}") + set(GASNET_ROOT $ENV{GASNET_ROOT}) +elseif(NOT GASNET_ROOT) + string(REPLACE "/include" "" GASNET_ROOT "${GASNET_INCLUDE_DIR}") +endif() + +# Set GASNET_ROOT in case the other hints are used +if(GASNET_ROOT) + # The call to file is for compatibility with windows paths + file(TO_CMAKE_PATH ${GASNET_ROOT} GASNET_ROOT) +elseif("$ENV{GASNET_ROOT}") + file(TO_CMAKE_PATH $ENV{GASNET_ROOT} GASNET_ROOT) +else() + file(TO_CMAKE_PATH "${GASNET_INCLUDE_DIR}" GASNET_INCLUDE_DIR) + string(REPLACE "/include" "" GASNET_ROOT "${GASNET_INCLUDE_DIR}") endif() set(GASNET_LIBRARIES ${GASNET_LIBRARY}) -set(GASNET_INCLUDE_DIRS ${GASNET_INCLUDE_DIR}) -include(FindPackageHandleStandardArgs) +if(HPX_WITH_PARCELPORT_GASNET_MPI) + set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/mpi-conduit") +elseif(HPX_WITH_PARCELPORT_GASNET_UDP) + set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/udp-conduit") +elseif(HPX_WITH_PARCELPORT_GASNET_SMP) + set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/smp-conduit") +endif() + +set(GASNET_INCLUDE_DIRS ${GASNET_INCLUDE_DIR} ${GASNET_CONDUIT_INCLUDE_DIR}) + find_package_handle_standard_args( - Gasnet DEFAULT_MSG GASNET_LIBRARY GASNET_INCLUDE_DIR + GASNET DEFAULT_MSG GASNET_LIBRARY GASNET_INCLUDE_DIR ) -mark_as_advanced(GASNET_ROOT GASNET_LIBRARY GASNET_INCLUDE_DIR) +get_property( + _type + CACHE GASNET_ROOT + PROPERTY TYPE +) + +if(_type) + set_property(CACHE GASNET_ROOT PROPERTY ADVANCED 1) + if("x${_type}" STREQUAL "xUNINITIALIZED") + set_property(CACHE GASNET_ROOT PROPERTY TYPE PATH) + endif() +endif() + +mark_as_advanced(GASNET_ROOT GASNET_LIBRARY GASNET_INCLUDE_DIR GASNET_CONDUIT_INCLUDE_DIR) + +add_library(Gasnet::gasnet INTERFACE IMPORTED) +target_include_directories(Gasnet::gasnet SYSTEM INTERFACE "${GASNET_INCLUDE_DIR}" "${GASNET_CONDUIT_INCLUDE_DIR}") +target_link_libraries(Gasnet::gasnet INTERFACE ${GASNET_LIBRARY}) + endmacro() diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake index 3c9ab5b0f97b..295a3d42562e 100644 --- a/cmake/HPX_SetupGASNET.cmake +++ b/cmake/HPX_SetupGASNET.cmake @@ -4,96 +4,17 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -include(HPX_Message) -include(FindGasnet) - macro(hpx_setup_gasnet) - if(NOT GASNET_FOUND) - - find_gasnet() - - set(GASNET_CXXFLAGS) - set(GASNET_CXXDEFS) - set(GASNET_INCLUDES) - set(GASNET_LDFLAGS) - set(GASNET_LIBFLAGS) - - foreach(TOKEN ${GASNET_CFLAGS}) - string(FIND "${TOKEN}" "-I" incpos) - string(FIND "${TOKEN}" "-l" lnkpos) - string(FIND "${TOKEN}" "-L" libpos) - string(FIND "${TOKEN}" "-D" defpos) - - if(NOT ${incpos} EQUAL -1) - string(REPLACE "-I" "" TOKEN "${TOKEN}") - list(APPEND GASNET_INCLUDES ${TOKEN}) - elseif(NOT ${lnkpos} EQUAL -1) - string(REPLACE "-l" "" TOKEN "${TOKEN}") - list(APPEND GASNET_LDFLAGS ${TOKEN}) - elseif(NOT ${libpos} EQUAL -1) - string(REPLACE "-L" "" TOKEN "${TOKEN}") - list(APPEND GASNET_LIBFLAGS ${TOKEN}) - elseif(NOT ${defpos} EQUAL -1) - string(REPLACE "-D" "" TOKEN "${TOKEN}") - list(APPEND GASNET_CXXDEFS ${TOKEN}) - else() - list(APPEND GASNET_CXXFLAGS ${TOKEN}) - endif() - endforeach() - - foreach(TOKEN ${GASNET_LDFLAGS}) - string(FIND "${TOKEN}" "-l" lnkpos) - if(NOT ${lnkpos} EQUAL -1) - list(APPEND GASNET_LIBFLAGS ${TOKEN}) - endif() - endforeach() - - list(APPEND GASNET_CXXDEFS "-DGASNET_PAR") - - #string (REPLACE ";" " " GASNET_CXXFLAGS "${GASNET_CXXFLAGS}") - #string (REPLACE ";" " " GASNET_CXXDEFS "${GASNET_CXXDEFS}") - #string (REPLACE ";" " " GASNET_INCLUDES "${GASNET_INCLUDES}") - #string (REPLACE ";" " " GASNET_LDFLAGS "${GASNET_LDFLAGS}") - #string (REPLACE ";" " " GASNET_LIBFLAGS "${GASNET_LIBFLAGS}") - - set(gasnet_libraries ${GASNET_LIBFLAGS}) +include(HPX_Message) +include(FindGasnet) - if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) - add_library(gasnet-mpi-par INTERFACE IMPORTED) - target_compile_options(gasnet-mpi-par INTERFACE ${GASNET_CXXFLAGS}) - target_compile_definitions(gasnet-mpi-par INTERFACE ${GASNET_CXXDEFS}) - target_include_directories(gasnet-mpi-par SYSTEM INTERFACE ${GASNET_INCLUDES}) - target_link_options(gasnet-mpi-par INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet-mpi-par INTERFACE ${GASNET_LIBFLAGS}) - elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING) - add_library(gasnet-udp-par INTERFACE IMPORTED) - target_compile_options(gasnet-udp-par INTERFACE ${GASNET_CXXFLAGS}) - target_compile_definitions(gasnet-udp-par INTERFACE ${GASNET_CXXDEFS}) - target_include_directories(gasnet-udp-par SYSTEM INTERFACE${GASNET_INCLUDES}) - target_link_options(gasnet-udp-par INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet-udp-par INTERFACE ${GASNET_LIBFLAGS}) - else() - add_library(gasnet-smp-par INTERFACE IMPORTED) - target_compile_options(gasnet-smp-par INTERFACE ${GASNET_CXXFLAGS}) - target_compile_definitions(gasnet-smp-par INTERFACE ${GASNET_CXXDEFS}) - target_include_directories(gasnet-smp-par SYSTEM INTERFACE ${GASNET_INCLUDES}) - target_link_options(gasnet-smp-par INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet-smp-par INTERFACE ${GASNET_LIBFLAGS}) - endif() +find_gasnet() - if(GASNET_MPI_FOUND) - # Setup PMI imported target - find_package(PMI) - if(PMI_FOUND) - hpx_add_config_define_namespace( - DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET - ) - add_library(Pmi::pmi INTERFACE IMPORTED) - target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) - target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) - endif() - endif() +if(NOT GASNET_FOUND) + hpx_error("GASNET could not be found, please specify \ + GASNET_ROOT to point to the root of your GASNET installation" +) +endif() - endif() endmacro() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 0996804504b9..5c2b04da73e9 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,6 +52,12 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + set(GASNET_ROOT "@GASNET_ROOT@") + include(HPX_SetupGASNET) + hpx_setup_gasnet() +endif() + # Eve can be installed by HPX or externally installed. In the first case we use # exported targets, in the second we find Eve again using find_package. if("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index a7e66517d2eb..40963536e1e2 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -6,27 +6,14 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET_MPI) - # setup GASNET, if necessary - include(HPX_SetupGASNET) - hpx_setup_gasnet() -endif() - -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET_UDP) - # setup GASNET, if necessary - include(HPX_SetupGASNET) - hpx_setup_gasnet() -else() +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_GASNET_UDP))) + message(STATUS "*** PARCELPORT_GASNET not set") return() endif() -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET_SMP) - # setup GASNET, if necessary - include(HPX_SetupGASNET) - hpx_setup_gasnet() -else() - return() -endif() +# setup GASNET, if necessary +include(HPX_SetupGASNET) +hpx_setup_gasnet() include(HPX_AddModule) @@ -35,43 +22,26 @@ set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp hpx/gasnet_base/gasnet_enviro set(gasnet_base_sources gasnet_environment.cpp) -if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) - set(additional_dependencies Mpi::mpi gasnet-mpi-par) - - add_hpx_module( - core gasnet_base - GLOBAL_HEADER_GEN ON - SOURCES ${gasnet_base_sources} - HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} - MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - DEPENDENCIES ${additional_dependencies} - CMAKE_SUBDIRS examples tests - ) -elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING) - set(additional_dependencies gasnet-udp-par) +if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) + set(additional_dependencies Mpi::mpi Gasnet::gasnet) add_hpx_module( core gasnet_base GLOBAL_HEADER_GEN ON SOURCES ${gasnet_base_sources} HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util DEPENDENCIES ${additional_dependencies} CMAKE_SUBDIRS examples tests ) else() - set(additional_dependencies gasnet-smp-par) - add_hpx_module( core gasnet_base GLOBAL_HEADER_GEN ON SOURCES ${gasnet_base_sources} HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - DEPENDENCIES ${additional_dependencies} + DEPENDENCIES Gasnet::gasnet CMAKE_SUBDIRS examples tests ) endif() diff --git a/libs/core/gasnet_base/examples/CMakeLists.txt b/libs/core/gasnet_base/examples/CMakeLists.txt index 0d05c6cd0214..a224dcc5f277 100644 --- a/libs/core/gasnet_base/examples/CMakeLists.txt +++ b/libs/core/gasnet_base/examples/CMakeLists.txt @@ -5,12 +5,12 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(HPX_WITH_EXAMPLES) - add_hpx_pseudo_target(examples.modules.mpi_base) - add_hpx_pseudo_dependencies(examples.modules examples.modules.mpi_base) + add_hpx_pseudo_target(examples.modules.gasnet_base) + add_hpx_pseudo_dependencies(examples.modules examples.modules.gasnet_base) if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) - add_hpx_pseudo_target(tests.examples.modules.mpi_base) + add_hpx_pseudo_target(tests.examples.modules.gasnet_base) add_hpx_pseudo_dependencies( - tests.examples.modules tests.examples.modules.mpi_base + tests.examples.modules tests.examples.modules.gasnet_base ) endif() endif() diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp index 559f0cbf0346..e76245428664 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -17,7 +17,10 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif +#ifndef GASNET_PAR #define GASNET_PAR 1 +#endif + #include #if defined(__clang__) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp index 3e9b40a8a8ff..2ecdb8db9412 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -9,10 +9,11 @@ #include -#if (defined(HPX_HAVE_NETWORKING) && (defined(HPX_HAVE_PARCELPORT_GASNET_MPI) || defined(HPX_HAVE_PARCELPORT_GASNET_SMP))) +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) -#include #include +#include #include #include diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 83ec81fdb774..d46aaa7b12d8 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -232,8 +232,7 @@ namespace hpx { namespace util { bool gasnet_environment::check_gasnet_environment( util::runtime_configuration const& cfg) { -#if defined(HPX_HAVE_NETWORKING) && ( defined(HPX_HAVE_PARCELPORT_GASNET_MPI) || \ - defined(HPX_HAVE_PARCELPORT_GASNET_SMP) ) +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE)) // We disable the GASNET parcelport if any of these hold: // // - The parcelport is explicitly disabled @@ -256,9 +255,7 @@ namespace hpx { namespace util { } }} // namespace hpx::util -#if (defined(HPX_HAVE_NETWORKING) && (defined(HPX_HAVE_PARCELPORT_GASNET_MPI) || \ - defined(HPX_HAVE_PARCELPORT_GASNET_SMP))) || \ - defined(HPX_HAVE_MODULE_GASNET_BASE) +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE)) namespace hpx { namespace util { diff --git a/libs/core/gasnet_base/tests/CMakeLists.txt b/libs/core/gasnet_base/tests/CMakeLists.txt index 3296f365ea31..e6593cda9ac9 100644 --- a/libs/core/gasnet_base/tests/CMakeLists.txt +++ b/libs/core/gasnet_base/tests/CMakeLists.txt @@ -9,34 +9,34 @@ include(HPX_Option) if(HPX_WITH_TESTS) if(HPX_WITH_TESTS_UNIT) - add_hpx_pseudo_target(tests.unit.modules.mpi_base) - add_hpx_pseudo_dependencies(tests.unit.modules tests.unit.modules.mpi_base) + add_hpx_pseudo_target(tests.unit.modules.gasnet_base) + add_hpx_pseudo_dependencies(tests.unit.modules tests.unit.modules.gasnet_base) add_subdirectory(unit) endif() if(HPX_WITH_TESTS_REGRESSIONS) - add_hpx_pseudo_target(tests.regressions.modules.mpi_base) + add_hpx_pseudo_target(tests.regressions.modules.gasnet_base) add_hpx_pseudo_dependencies( - tests.regressions.modules tests.regressions.modules.mpi_base + tests.regressions.modules tests.regressions.modules.gasnet_base ) add_subdirectory(regressions) endif() if(HPX_WITH_TESTS_BENCHMARKS) - add_hpx_pseudo_target(tests.performance.modules.mpi_base) + add_hpx_pseudo_target(tests.performance.modules.gasnet_base) add_hpx_pseudo_dependencies( - tests.performance.modules tests.performance.modules.mpi_base + tests.performance.modules tests.performance.modules.gasnet_base ) add_subdirectory(performance) endif() if(HPX_WITH_TESTS_HEADERS) add_hpx_header_tests( - modules.mpi_base - HEADERS ${mpi_base_headers} + modules.gasnet_base + HEADERS ${gasnet_base_headers} HEADER_ROOT ${PROJECT_SOURCE_DIR}/include NOLIBS - DEPENDENCIES hpx_mpi_base + DEPENDENCIES hpx_gasnet_base ) endif() endif() diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index a7c72bc8ac09..445af6cb6f5c 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -131,6 +131,16 @@ namespace hpx { } #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) + std::string gasnet_version() + { + std::ostringstream strm; + strm << "GASNET"; + return strm.str(); + } +#endif + std::string copyright() { char const* const copyright = @@ -294,6 +304,10 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) " LCI: {}\n" #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) + " GASNET: {}\n" +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_MPI)) || \ defined(HPX_HAVE_MODULE_MPI_BASE) " MPI: {}\n" @@ -310,6 +324,10 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) lci_version(), #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) + gasnet_version(), +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_MPI)) || \ defined(HPX_HAVE_MODULE_MPI_BASE) mpi_version(), diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index b9f91bd04344..de97cc98ffca 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -34,6 +34,18 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGASNET) + hpx_setup_gasnet() + + if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) + set(additional_dependencies Mpi::mpi Papi::papi) + elseif(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UDP) + set(additional_dependencies Papi::papi) + elseif(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_SMP) + set(additional_dependencies Papi::papi) + endif() +endif() include(HPX_AddModule) add_hpx_module( diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index a04e3073bd8d..5c5f98339c61 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -21,6 +21,9 @@ #if defined(HPX_HAVE_MODULE_LCI_BASE) #include #endif +#if defined(HPX_HAVE_MODULE_GASNET_BASE) +#include +#endif #include #include #include diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 66070ae35e3e..92a3e9796668 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -4,7 +4,9 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_UDP)) +if(NOT (HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_UDP))) + return() +endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -27,7 +29,7 @@ hpx_setup_gasnet() include(HPX_AddModule) -if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) +if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) add_hpx_module( full parcelport_gasnet GLOBAL_HEADER_GEN ON @@ -38,37 +40,31 @@ if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING) DEPENDENCIES hpx_core gasnet-mpi-par Mpi::mpi CMAKE_SUBDIRS examples tests ) -else() - if(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - DEPENDENCIES hpx_core gasnet-udp-par - CMAKE_SUBDIRS examples tests - ) - else() - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - DEPENDENCIES hpx_core gasnet-smp-par - CMAKE_SUBDIRS examples tests - ) - endif() +elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UDP) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + DEPENDENCIES hpx_core Gasnet::gasnet + CMAKE_SUBDIRS examples tests + ) +elseif(GASNET_SMP_FOUND) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + DEPENDENCIES hpx_core Gasnet::gasnet + CMAKE_SUBDIRS examples tests + ) endif() set(HPX_STATIC_PARCELPORT_PLUGINS ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_gasnet CACHE INTERNAL "" FORCE ) - -else() - return() -endif() diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index 08be691e150b..5fddd1ef3ffa 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -12,8 +12,8 @@ #if defined(HPX_HAVE_PARCELPORT_GASNET) #include -#include -#include +#include +#include #include diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 5863dfefcd3e..1e97faeb2321 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -249,4 +249,48 @@ namespace hpx::parcelset { } // namespace policies::gasnet } // namespace hpx::parcelset +namespace hpx::traits { + + // Inject additional configuration data into the factory registry for this + // type. This information ends up in the system wide configuration database + // under the plugin specific section: + // + // [hpx.parcel.gasnet] + // ... + // priority = 1 + // + template <> + struct plugin_config_data + { + static constexpr char const* priority() noexcept + { + return "1"; + } + + static void init(int* argc, char*** argv, util::command_line_handling& cfg) + { + util::gasnet_environment::init(argc, argv, cfg.rtcfg_); + cfg.num_localities_ = + static_cast(util::gasnet_environment::size()); + cfg.node_ = static_cast(util::gasnet_environment::rank()); + } + + // by default no additional initialization using the resource + // partitioner is required + static constexpr void init(hpx::resource::partitioner&) noexcept {} + + static void destroy() noexcept + { + util::gasnet_environment::finalize(); + } + + static constexpr char const* call() noexcept + { + return ""; + } + }; +} // namespace hpx::traits + +HPX_REGISTER_PARCELPORT(hpx::parcelset::policies::gasnet::connection_handler, gasnet) + #endif From e204eeffcceedf29ad24cbc814bcfd95c56c2cf7 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 7 May 2023 22:26:51 -0400 Subject: [PATCH 009/333] updated --- cmake/FindGasnet.cmake | 26 +++++++++++++--------- libs/full/parcelport_gasnet/CMakeLists.txt | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 1013c81cdc3e..285ee3b97720 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -14,7 +14,7 @@ set(GASNET_SMP_FOUND FALSE) find_package(PkgConfig QUIET) if(HPX_WITH_PARCELPORT_GASNET_MPI) - pkg_check_modules(PC_GASNET QUIET gasnet-mpi-par) + pkg_check_modules(GASNET QUIET IMPORTED_TARGET gasnet-mpi-par) find_path( GASNET_INCLUDE_DIR gasnet.h @@ -36,7 +36,7 @@ if(HPX_WITH_PARCELPORT_GASNET_MPI) endif() if(HPX_WITH_PARCELPORT_GASNET_UDP) - pkg_check_modules(GASNET QUIET gasnet-udp-par) + pkg_check_modules(GASNET QUIET IMPORTED_TARGET gasnet-udp-par) find_path( GASNET_INCLUDE_DIR gasnet.h @@ -57,7 +57,7 @@ if(HPX_WITH_PARCELPORT_GASNET_UDP) endif() if(HPX_WITH_PARCELPORT_GASNET_SMP) - pkg_check_modules(GASNET QUIET gasnet-smp-par) + pkg_check_modules(GASNET QUIET IMPORTED_TARGET gasnet-smp-par) find_path( GASNET_INCLUDE_DIR gasnet.h @@ -77,7 +77,6 @@ if(HPX_WITH_PARCELPORT_GASNET_SMP) set(GASNET_SMP_FOUND TRUE) endif() - # Set GASNET_ROOT in case the other hints are used if(NOT GASNET_ROOT AND "$ENV{GASNET_ROOT}") set(GASNET_ROOT $ENV{GASNET_ROOT}) @@ -96,20 +95,24 @@ else() string(REPLACE "/include" "" GASNET_ROOT "${GASNET_INCLUDE_DIR}") endif() -set(GASNET_LIBRARIES ${GASNET_LIBRARY}) +set(GASNET_LIBRARIES ${GASNET_LINK_LIBRARIES}) if(HPX_WITH_PARCELPORT_GASNET_MPI) set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/mpi-conduit") + add_library(Gasnetc::conduit INTERFACE IMPORTED) + target_include_directories(Gasnetc::conduit INTERFACE ${GASNET_CONDUIT_INCLUDE_DIR}) elseif(HPX_WITH_PARCELPORT_GASNET_UDP) set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/udp-conduit") + add_library(Gasnetc::conduit INTERFACE IMPORTED) + target_include_directories(Gasnetc::conduit INTERFACE ${GASNET_CONDUIT_INCLUDE_DIR}) elseif(HPX_WITH_PARCELPORT_GASNET_SMP) set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/smp-conduit") + add_library(Gasnetc::conduit INTERFACE IMPORTED) + target_include_directories(Gasnetc::conduit SYSTEM INTERFACE ${GASNET_CONDUIT_INCLUDE_DIR}) endif() -set(GASNET_INCLUDE_DIRS ${GASNET_INCLUDE_DIR} ${GASNET_CONDUIT_INCLUDE_DIR}) - find_package_handle_standard_args( - GASNET DEFAULT_MSG GASNET_LIBRARY GASNET_INCLUDE_DIR + GASNET DEFAULT_MSG GASNET_LIBRARY GASNET_INCLUDE_DIR GASNET_CONDUIT_INCLUDE_DIR ) get_property( @@ -125,10 +128,13 @@ if(_type) endif() endif() -mark_as_advanced(GASNET_ROOT GASNET_LIBRARY GASNET_INCLUDE_DIR GASNET_CONDUIT_INCLUDE_DIR) +mark_as_advanced( + GASNET_ROOT GASNET_LIBRARY GASNET_INCLUDE_DIR GASNET_CONDUIT_INCLUDE_DIR +) add_library(Gasnet::gasnet INTERFACE IMPORTED) -target_include_directories(Gasnet::gasnet SYSTEM INTERFACE "${GASNET_INCLUDE_DIR}" "${GASNET_CONDUIT_INCLUDE_DIR}") +target_include_directories(Gasnet::gasnet SYSTEM INTERFACE ${GASNET_INCLUDE_DIR}) target_link_libraries(Gasnet::gasnet INTERFACE ${GASNET_LIBRARY}) +add_dependencies(Gasnet::gasnet Gasnetc::conduit) endmacro() diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 92a3e9796668..bb096732b90c 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -37,7 +37,7 @@ if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) HEADERS ${parcelport_gasnet_headers} COMPAT_HEADERS ${parcelport_gasnet_compat_headers} MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - DEPENDENCIES hpx_core gasnet-mpi-par Mpi::mpi + DEPENDENCIES hpx_core Mpi::mpi Gasnet::gasnet CMAKE_SUBDIRS examples tests ) elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UDP) From a29e4861a513264e4405c155cc91aeb58c5af157 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 24 Jun 2023 21:50:25 -0400 Subject: [PATCH 010/333] updated parcelport module has linking issue with hpxcore --- CMakeLists.txt | 1 + cmake/FindGasnet.cmake | 116 +----------------- cmake/HPX_AddModule.cmake | 1 + cmake/HPX_SetupGASNET.cmake | 20 --- libs/core/gasnet_base/CMakeLists.txt | 31 +++-- libs/core/modules.rst | 1 + .../full/command_line_handling/CMakeLists.txt | 17 +-- libs/full/parcelport_gasnet/CMakeLists.txt | 21 ++-- .../include/hpx/parcelport_gasnet/header.hpp | 2 +- .../hpx/parcelport_gasnet/locality.hpp | 7 +- .../hpx/parcelport_gasnet/receiver.hpp | 37 +++--- .../parcelport_gasnet/receiver_connection.hpp | 35 +++--- .../include/hpx/parcelport_gasnet/sender.hpp | 13 +- .../parcelport_gasnet/sender_connection.hpp | 90 +++++++++----- .../hpx/parcelport_gasnet/tag_provider.hpp | 2 +- libs/full/parcelport_gasnet/src/locality.cpp | 3 +- .../src/parcelport_gasnet.cpp | 56 ++++++--- .../parcelport_gasnet/tests/CMakeLists.txt | 18 +-- 18 files changed, 207 insertions(+), 264 deletions(-) delete mode 100644 cmake/HPX_SetupGASNET.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 288d36748a6c..56c2540cedde 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,6 +588,7 @@ if(APPLE OR HPX_WITH_CUDA OR HPX_WITH_HIP OR HPX_WITH_PARCELPORT_LCI + OR HPX_WITH_PARCELPORT_GASNET ) set(HPX_WITH_PKGCONFIG_DEFAULT OFF) endif() diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 285ee3b97720..3c87c9a39100 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -11,130 +11,24 @@ set(GASNET_MPI_FOUND FALSE) set(GASNET_UDP_FOUND FALSE) set(GASNET_SMP_FOUND FALSE) -find_package(PkgConfig QUIET) +find_package(PkgConfig REQUIRED QUIET COMPONENTS) if(HPX_WITH_PARCELPORT_GASNET_MPI) - pkg_check_modules(GASNET QUIET IMPORTED_TARGET gasnet-mpi-par) - - find_path( - GASNET_INCLUDE_DIR gasnet.h - HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_INCLUDEDIR} - ${PC_GASNET_INCLUDE_DIRS} - PATH_SUFFIXES include - ) - - find_library( - GASNET_LIBRARY - NAMES gasnet-mpi-par - HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_LIBDIR} - ${PC_GASNET_LIBRARY_DIRS} - PATH_SUFFIXES lib lib64 - ) - + pkg_check_modules(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-mpi-par) set(GASNET_MPI_FOUND TRUE) hpx_setup_mpi() endif() if(HPX_WITH_PARCELPORT_GASNET_UDP) - pkg_check_modules(GASNET QUIET IMPORTED_TARGET gasnet-udp-par) - - find_path( - GASNET_INCLUDE_DIR gasnet.h - HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_INCLUDEDIR} - ${PC_GASNET_INCLUDE_DIRS} - PATH_SUFFIXES include - ) - - find_library( - GASNET_LIBRARY - NAMES gasnet-udp-par - HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_LIBDIR} - ${PC_GASNET_LIBRARY_DIRS} - PATH_SUFFIXES lib lib64 - ) - + pkg_check_modules(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-udp-par) set(GASNET_UDP_FOUND TRUE) endif() if(HPX_WITH_PARCELPORT_GASNET_SMP) - pkg_check_modules(GASNET QUIET IMPORTED_TARGET gasnet-smp-par) - - find_path( - GASNET_INCLUDE_DIR gasnet.h - HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_INCLUDEDIR} - ${PC_GASNET_INCLUDE_DIRS} - PATH_SUFFIXES include - ) - - find_library( - GASNET_LIBRARY - NAMES gasnet-smp-par - HINTS ${GASNET_ROOT} ENV GASNET_ROOT ${PC_GASNET_LIBDIR} - ${PC_GASNET_LIBRARY_DIRS} - PATH_SUFFIXES lib lib64 - ) - + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-smp-par) set(GASNET_SMP_FOUND TRUE) endif() -# Set GASNET_ROOT in case the other hints are used -if(NOT GASNET_ROOT AND "$ENV{GASNET_ROOT}") - set(GASNET_ROOT $ENV{GASNET_ROOT}) -elseif(NOT GASNET_ROOT) - string(REPLACE "/include" "" GASNET_ROOT "${GASNET_INCLUDE_DIR}") -endif() - -# Set GASNET_ROOT in case the other hints are used -if(GASNET_ROOT) - # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${GASNET_ROOT} GASNET_ROOT) -elseif("$ENV{GASNET_ROOT}") - file(TO_CMAKE_PATH $ENV{GASNET_ROOT} GASNET_ROOT) -else() - file(TO_CMAKE_PATH "${GASNET_INCLUDE_DIR}" GASNET_INCLUDE_DIR) - string(REPLACE "/include" "" GASNET_ROOT "${GASNET_INCLUDE_DIR}") -endif() - -set(GASNET_LIBRARIES ${GASNET_LINK_LIBRARIES}) - -if(HPX_WITH_PARCELPORT_GASNET_MPI) - set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/mpi-conduit") - add_library(Gasnetc::conduit INTERFACE IMPORTED) - target_include_directories(Gasnetc::conduit INTERFACE ${GASNET_CONDUIT_INCLUDE_DIR}) -elseif(HPX_WITH_PARCELPORT_GASNET_UDP) - set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/udp-conduit") - add_library(Gasnetc::conduit INTERFACE IMPORTED) - target_include_directories(Gasnetc::conduit INTERFACE ${GASNET_CONDUIT_INCLUDE_DIR}) -elseif(HPX_WITH_PARCELPORT_GASNET_SMP) - set(GASNET_CONDUIT_INCLUDE_DIR "${GASNET_INCLUDE_DIR}/smp-conduit") - add_library(Gasnetc::conduit INTERFACE IMPORTED) - target_include_directories(Gasnetc::conduit SYSTEM INTERFACE ${GASNET_CONDUIT_INCLUDE_DIR}) -endif() - -find_package_handle_standard_args( - GASNET DEFAULT_MSG GASNET_LIBRARY GASNET_INCLUDE_DIR GASNET_CONDUIT_INCLUDE_DIR -) - -get_property( - _type - CACHE GASNET_ROOT - PROPERTY TYPE -) - -if(_type) - set_property(CACHE GASNET_ROOT PROPERTY ADVANCED 1) - if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE GASNET_ROOT PROPERTY TYPE PATH) - endif() -endif() - -mark_as_advanced( - GASNET_ROOT GASNET_LIBRARY GASNET_INCLUDE_DIR GASNET_CONDUIT_INCLUDE_DIR -) - -add_library(Gasnet::gasnet INTERFACE IMPORTED) -target_include_directories(Gasnet::gasnet SYSTEM INTERFACE ${GASNET_INCLUDE_DIR}) -target_link_libraries(Gasnet::gasnet INTERFACE ${GASNET_LIBRARY}) -add_dependencies(Gasnet::gasnet Gasnetc::conduit) +target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) endmacro() diff --git a/cmake/HPX_AddModule.cmake b/cmake/HPX_AddModule.cmake index 3efec41ec1f1..52df66677975 100644 --- a/cmake/HPX_AddModule.cmake +++ b/cmake/HPX_AddModule.cmake @@ -23,6 +23,7 @@ function(add_hpx_module libname modulename) EXCLUDE_FROM_GLOBAL_HEADER ADD_TO_GLOBAL_HEADER ) + cmake_parse_arguments( ${modulename} "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN} diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake deleted file mode 100644 index 295a3d42562e..000000000000 --- a/cmake/HPX_SetupGASNET.cmake +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2019-2022 Ste||ar Group -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -macro(hpx_setup_gasnet) - -include(HPX_Message) -include(FindGasnet) - -find_gasnet() - -if(NOT GASNET_FOUND) - hpx_error("GASNET could not be found, please specify \ - GASNET_ROOT to point to the root of your GASNET installation" -) -endif() - -endmacro() diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 40963536e1e2..de76d7607687 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -4,44 +4,53 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_GASNET_UDP))) message(STATUS "*** PARCELPORT_GASNET not set") return() endif() -# setup GASNET, if necessary -include(HPX_SetupGASNET) -hpx_setup_gasnet() +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include(FindGasnet) +find_gasnet() -include(HPX_AddModule) +# Default location is $HPX_ROOT/libs/gasnet_base/include +set(gasnet_base_headers + hpx/gasnet_base/gasnet.hpp + hpx/gasnet_base/gasnet_environment.hpp +) # Default location is $HPX_ROOT/libs/gasnet_base/include -set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp hpx/gasnet_base/gasnet_environment.hpp) +# cmake-format: off +set(gasnet_base_compat_headers + hpx/plugins/parcelport/gasnet/gasnet.hpp => hpx/modules/gasnet_base.hpp + hpx/plugins/parcelport/gasnet/gasnet_environment.hpp => hpx/modules/gasnet_base.hpp +) +# cmake-format: on set(gasnet_base_sources gasnet_environment.cpp) if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) - set(additional_dependencies Mpi::mpi Gasnet::gasnet) - + include(HPX_AddModule) add_hpx_module( core gasnet_base GLOBAL_HEADER_GEN ON SOURCES ${gasnet_base_sources} HEADERS ${gasnet_base_headers} + COMPAT_HEADERS ${gasnet_base_compat_headers} + DEPENDENCIES PkgConfig::GASNET Mpi::mpi MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - DEPENDENCIES ${additional_dependencies} CMAKE_SUBDIRS examples tests ) else() + include(HPX_AddModule) add_hpx_module( core gasnet_base GLOBAL_HEADER_GEN ON SOURCES ${gasnet_base_sources} HEADERS ${gasnet_base_headers} + COMPAT_HEADERS ${gasnet_base_compat_headers} + DEPENDENCIES PkgConfig::GASNET MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - DEPENDENCIES Gasnet::gasnet CMAKE_SUBDIRS examples tests ) endif() diff --git a/libs/core/modules.rst b/libs/core/modules.rst index 3bb9bf20d3ef..46dbbe21dec1 100644 --- a/libs/core/modules.rst +++ b/libs/core/modules.rst @@ -54,6 +54,7 @@ Core modules /libs/core/logging/docs/index.rst /libs/core/memory/docs/index.rst /libs/core/mpi_base/docs/index.rst + /libs/core/gasnet_base/docs/index.rst /libs/core/pack_traversal/docs/index.rst /libs/core/plugin/docs/index.rst /libs/core/prefix/docs/index.rst diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index de97cc98ffca..9733990f4291 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -34,17 +34,12 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) - include(HPX_SetupGASNET) - hpx_setup_gasnet() - - if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) - set(additional_dependencies Mpi::mpi Papi::papi) - elseif(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UDP) - set(additional_dependencies Papi::papi) - elseif(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_SMP) - set(additional_dependencies Papi::papi) - endif() +if(HPX_WITH_PARCELPORT_GASNET_MPI) + set(additional_dependencies ${additional_dependencies} Mpi::mpi PkgConfig::GASNET) +elseif(HPX_WITH_PARCELPORT_GASNET_UDP) + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +elseif(HPX_WITH_PARCELPORT_GASNET_SMP) + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) endif() include(HPX_AddModule) diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index bb096732b90c..04d17ba12609 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -9,6 +9,8 @@ if(NOT (HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARC endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") +include(FindGasnet) +find_gasnet() set(parcelport_gasnet_headers hpx/parcelport_gasnet/header.hpp @@ -24,42 +26,45 @@ set(parcelport_gasnet_compat_headers) set(parcelport_gasnet_sources locality.cpp parcelport_gasnet.cpp) -include(HPX_SetupGASNET) -hpx_setup_gasnet() - -include(HPX_AddModule) - if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) +link_directories(${GASNET_LIBRARY_DIRS}) + include(HPX_AddModule) add_hpx_module( full parcelport_gasnet GLOBAL_HEADER_GEN ON SOURCES ${parcelport_gasnet_sources} HEADERS ${parcelport_gasnet_headers} COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET Mpi::mpi MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - DEPENDENCIES hpx_core Mpi::mpi Gasnet::gasnet CMAKE_SUBDIRS examples tests ) + elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UDP) +link_directories(${GASNET_LIBRARY_DIRS}) + include(HPX_AddModule) add_hpx_module( full parcelport_gasnet GLOBAL_HEADER_GEN ON SOURCES ${parcelport_gasnet_sources} HEADERS ${parcelport_gasnet_headers} COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - DEPENDENCIES hpx_core Gasnet::gasnet CMAKE_SUBDIRS examples tests ) + elseif(GASNET_SMP_FOUND) +link_directories(${GASNET_LIBRARY_DIRS}) + include(HPX_AddModule) add_hpx_module( full parcelport_gasnet GLOBAL_HEADER_GEN ON SOURCES ${parcelport_gasnet_sources} HEADERS ${parcelport_gasnet_headers} COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - DEPENDENCIES hpx_core Gasnet::gasnet CMAKE_SUBDIRS examples tests ) endif() diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp index 707983ef2fbe..d0282a2b0ed7 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -10,7 +10,7 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index 5fddd1ef3ffa..82c9b6c97892 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -10,10 +10,9 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include -#include -#include +#include #include @@ -44,7 +43,7 @@ namespace hpx::parcelset::policies::gasnet { explicit constexpr operator bool() const noexcept { - return rank_ != -1; + return rank_ != ((unsigned int)-1); } HPX_EXPORT void save(serialization::output_archive& ar) const; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index 897eb6356b9b..6797de62e7bb 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -28,21 +28,6 @@ namespace hpx::parcelset::policies::gasnet { - struct exp_backoff { - int numTries; - const static int maxRetries = 10; - - void operator()() { - if(numTries <= maxRetries) { - gasnet_AMPoll(); - hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); -  } - else { - numTries = 0; - } - } - }; - template struct receiver { @@ -52,9 +37,23 @@ namespace hpx::parcelset::policies::gasnet { using connection_ptr = std::shared_ptr; using connection_list = std::deque; + struct exp_backoff { + int numTries; + const static int maxRetries = 10; + + void operator()() { + if(numTries <= maxRetries) { + gasnet_AMPoll(); + hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); + } + else { + numTries = 0; + } + } + }; + explicit constexpr receiver(Parcelport& pp) noexcept - : pp_(pp) - , hdr_request_(0) + : pp_(pp), bo() { } @@ -134,10 +133,7 @@ namespace hpx::parcelset::policies::gasnet { header h = rcv_header_; rcv_header_.reset(); - exp_backoff bo{0}; - while(rcv_header_.data() == 0) { - gasnet_AMPoll(); bo(); } @@ -154,6 +150,7 @@ namespace hpx::parcelset::policies::gasnet { hpx::spinlock connections_mtx_; connection_list connections_; + exp_backoff bo; }; } // namespace hpx::parcelset::policies::gasnet diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index d2e015381f6f..edc74460fb17 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -10,7 +10,7 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include #include @@ -51,7 +51,6 @@ namespace hpx::parcelset::policies::gasnet { , src_(src) , tag_(h.tag()) , header_(h) - , request_(MPI_REQUEST_NULL) , request_ptr_(false) , chunks_idx_(0) , pp_(pp) @@ -93,7 +92,7 @@ namespace hpx::parcelset::policies::gasnet { bool receive_transmission_chunks(std::size_t num_thread = -1) { - auto self_ = util::gasnet_environment::rank(); + auto self_ = hpx::util::gasnet_environment::rank(); // determine the size of the chunk buffer std::size_t num_zero_copy_chunks = static_cast( @@ -106,9 +105,9 @@ namespace hpx::parcelset::policies::gasnet { { buffer_.chunks_.resize(num_zero_copy_chunks); { - util::gasnet_environment::scoped_lock l; + hpx::util::gasnet_environment::scoped_lock l; std::memcpy(buffer_.transmission_chunks_.data(), - util::gasnet_environment::gasnet_buffer[self_], + hpx::util::gasnet_environment::segments[self_].addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); @@ -136,9 +135,11 @@ namespace hpx::parcelset::policies::gasnet { } else { - auto self_ = util::gasnet_environment::rank(); - util::gasnet_environment::scoped_lock l; - std::memcopy(buffer_.data_.data(), util::gasnet_environment::gasnet_buffer[self_], buffer_.data_.size()); + auto self_ = hpx::util::gasnet_environment::rank(); + hpx::util::gasnet_environment::scoped_lock l; + std::memcpy(buffer_.data_.data(), + hpx::util::gasnet_environment::segments[self_].addr, + buffer_.data_.size()); request_ptr_ = true; } @@ -164,9 +165,11 @@ namespace hpx::parcelset::policies::gasnet { data_type& c = buffer_.chunks_[idx]; c.resize(chunk_size); { - auto self_ = util::gasnet_environment::rank(); - util::gasnet_environment::scoped_lock l; - std::memcopy(c.data(), data.util::gasnet_environment::gasnet_buffer[self_], c.size()); + auto self_ = hpx::util::gasnet_environment::rank(); + hpx::util::gasnet_environment::scoped_lock l; + std::memcpy(c.data(), + hpx::util::gasnet_environment::segments[self_].addr, + c.size()); request_ptr_ = true; } } @@ -187,9 +190,11 @@ namespace hpx::parcelset::policies::gasnet { data.time_ = timer_.elapsed_nanoseconds() - data.time_; #endif { - auto self_ = util::gasnet_environment::rank(); - util::gasnet_environment::scoped_lock l; - std::memcopy(&tag_[src_], data.util::gasnet_environment::gasnet_buffer[self_], sizeof(int)); + auto self_ = hpx::util::gasnet_environment::rank(); + hpx::util::gasnet_environment::scoped_lock l; + std::memcpy(&tag_, + hpx::util::gasnet_environment::segments[self_].addr, + sizeof(int)); request_ptr_ = true; } @@ -207,7 +212,7 @@ namespace hpx::parcelset::policies::gasnet { bool request_done() noexcept { - util::gasnet_environment::scoped_try_lock l; + hpx::util::gasnet_environment::scoped_try_lock l; if (!l.locked) { return false; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index b66a6e9a96f2..744ac3a6ab95 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -9,7 +9,7 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include #include @@ -36,7 +36,7 @@ namespace hpx::parcelset::policies::gasnet { // different versions of clang-format disagree // clang-format off sender() noexcept - : next_free_tag_request_((MPI_Request) (-1)) + : next_free_tag_request_(-1) , next_free_tag_(-1) { } @@ -44,6 +44,7 @@ namespace hpx::parcelset::policies::gasnet { void run() noexcept { + hpx::util::gasnet_environment::scoped_lock l; get_next_free_tag(); } @@ -128,7 +129,7 @@ namespace hpx::parcelset::policies::gasnet { int next_free_tag_locked() noexcept { - util::gasnet_environment::scoped_try_lock l; + hpx::util::gasnet_environment::scoped_try_lock l; if (l.locked) { return get_next_free_tag(); @@ -140,9 +141,9 @@ namespace hpx::parcelset::policies::gasnet { { int next_free = next_free_tag_; - util::gasnet_environment::scoped_lock l; + hpx::util::gasnet_environment::scoped_lock l; std::memcpy(&next_free, - util::gasnet_environment::gasnet_buffer[self_], + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, sizeof(int)); return next_free; @@ -152,7 +153,7 @@ namespace hpx::parcelset::policies::gasnet { connection_list connections_; hpx::spinlock next_free_tag_mtx_; - MPI_Request next_free_tag_request_; + int next_free_tag_request_; int next_free_tag_; }; } // namespace hpx::parcelset::policies::gasnet diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index f5f6c16673a3..406834dc0340 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -72,10 +72,6 @@ namespace hpx::parcelset::policies::gasnet { , pp_(pp) , there_(parcelset::locality(locality(dst_))) { - gasnet_buffer.resize(hpx::util::gasnet_environment::size()); - for(std::size_t i = 0; i < gasnet_buffer.size(); ++i) { - gasnet_buffer[i] = static_cast(hpx::util::gasnet_environment::segments[i].addr); - } } parcelset::locality const& destination() const noexcept @@ -151,22 +147,29 @@ namespace hpx::parcelset::policies::gasnet { bool send_header() { { - util::gasnet_environment::scoped_lock l; + hpx::util::gasnet_environment::scoped_lock l; HPX_ASSERT(state_ == initialized); // compute + send the number of GASNET_PAGEs to send and the remainder number of bytes to a GASNET_PAGE // - const std::size_t chunks[] = { header_.data_size_ / GASNET_PAGESIZE, header_.data_size_ % GASNET_PAGESIZE }; - std::copy(chunks, sizeof(chunks), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + const std::size_t chunks[] = { static_cast(header_.data_size_ / GASNET_PAGESIZE), static_cast(header_.data_size_ % GASNET_PAGESIZE) }; + const std::size_t sizeof_chunks = sizeof(chunks); + std::memcpy( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + chunks, sizeof_chunks); // put from this localities gasnet shared memory segment // into the remote locality (dst_)'s shared memory segment // hpx::util::gasnet_environment::put( - gasnet_buffer[hpx::util::gasnet_environment::rank()], + static_cast( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr + ), dst_, - gasnet_buffer[dst_], - sizeof(chunks) + static_cast( + hpx::util::gasnet_environment::segments[dst_].addr + ), + sizeof_chunks ); } @@ -188,9 +191,21 @@ namespace hpx::parcelset::policies::gasnet { chunks = buffer_.transmission_chunks_; if (!chunks.empty()) { - util::gasnet_environment::scoped_lock l; - std::copy_n(chunks.data(), chunks.size(), gasnet_buffer[hpx::util::gasnet_environment::rank()]); - gasnet_put_bulk(dst_, gasnet_buffer[dst_], gasnet_buffer[hpx::util::gasnet_environment::rank()], + hpx::util::gasnet_environment::scoped_lock l; + std::memcpy( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + chunks.data(), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type)) + ); + + gasnet_put_bulk(dst_, + static_cast( + hpx::util::gasnet_environment::segments[dst_].addr + ), + static_cast( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr + ), static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)) ); } @@ -209,14 +224,20 @@ namespace hpx::parcelset::policies::gasnet { if (!header_.piggy_back()) { - util::gasnet_environment::scoped_lock l; - std::copy_n(buffer_.data_.data(), buffer_.data_.size(), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + hpx::util::gasnet_environment::scoped_lock l; + std::memcpy( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + buffer_.data_.data(), buffer_.data_.size()); hpx::util::gasnet_environment::put( - gasnet_buffer[hpx::util::gasnet_environment::rank()], + static_cast( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr + ), dst_, - gasnet_buffer[dst_], - sizeof(chunks) + static_cast( + hpx::util::gasnet_environment::segments[dst_].addr + ), + buffer_.data_.size() ); } state_ = sent_data; @@ -239,19 +260,27 @@ namespace hpx::parcelset::policies::gasnet { return false; } - util::gasnet_environment::scoped_lock l; + hpx::util::gasnet_environment::scoped_lock l; - std::copy_n(const_cast(c.data_.c_pos_), static_cast(c.size_), gasnet_buffer[hpx::util::gasnet_environment::rank()]); + std::memcpy( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + c.data_.cpos_, + static_cast(c.size_) + ); hpx::util::gasnet_environment::put( - gasnet_buffer[hpx::util::gasnet_environment::rank()], + static_cast( + hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr + ), dst_, - gasnet_buffer[dst_], + static_cast( + hpx::util::gasnet_environment::segments[dst_].addr + ), static_cast(c.size_) ); } - chunks_idx_++; + ++chunks_idx_; } state_ = sent_chunks; @@ -284,7 +313,7 @@ namespace hpx::parcelset::policies::gasnet { bool request_done() { - util::gasnet_environment::scoped_try_lock l; + hpx::util::gasnet_environment::scoped_try_lock l; if (!l.locked) { return false; @@ -297,13 +326,16 @@ namespace hpx::parcelset::policies::gasnet { sender_type* sender_; int tag_; int dst_; - hpx::move_only_function handler_; - hpx::move_only_function)> - postprocess_handler_; + + using handler_type = hpx::move_only_function; + handler_type handler_; + + using post_handler_type = hpx::move_only_function)>; + post_handler_type postprocess_handler_; header header_; - std::vector gasnet_buffer; std::size_t chunks_idx_; char ack_; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp index 45ba94918f65..fbb1459ca424 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/tag_provider.hpp @@ -8,7 +8,7 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include diff --git a/libs/full/parcelport_gasnet/src/locality.cpp b/libs/full/parcelport_gasnet/src/locality.cpp index 553a1e661608..150ce6e82834 100644 --- a/libs/full/parcelport_gasnet/src/locality.cpp +++ b/libs/full/parcelport_gasnet/src/locality.cpp @@ -8,10 +8,9 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include - #include namespace hpx::parcelset::policies::gasnet { diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 1e97faeb2321..728d107f6258 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -9,10 +9,11 @@ #include -#if defined(HPX_HAVE_PARCELPORT_GASNET) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) #include #include #include +#include #include #include #include @@ -21,12 +22,9 @@ #include #include -#include #include #include #include -#include -#include #include #include #include @@ -54,6 +52,7 @@ namespace hpx::parcelset { using send_early_parcel = std::true_type; using do_background_work = std::true_type; using send_immediate_parcels = std::false_type; + using is_connectionless = std::false_type; static constexpr const char* type() noexcept { @@ -104,15 +103,32 @@ namespace hpx::parcelset { HPX_PARCEL_MAX_CONNECTIONS); } + static std::size_t background_threads( + util::runtime_configuration const& ini) + { +/* + return hpx::util::get_entry_as(ini, + "hpx.parcel.gasnet.background_threads", + HPX_HAVE_PARCELPORT_GASNET_BACKGROUND_THREADS); +*/ + return 1UL; + } + public: parcelport(util::runtime_configuration const& ini, threads::policies::callback_notifier const& notifier) : base_type(ini, here(), notifier) , stopped_(false) , receiver_(*this) + , background_threads_(background_threads(ini)) { } + parcelport(parcelport const&) = delete; + parcelport(parcelport&&) = delete; + parcelport& operator=(parcelport const&) = delete; + parcelport& operator=(parcelport&&) = delete; + ~parcelport() { util::gasnet_environment::finalize(); @@ -144,15 +160,20 @@ namespace hpx::parcelset { "gasnet::parcelport::do_stop"); } - stopped_ = true; - gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); - if((retval = gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) { - // throw exception - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET failed ", std::to_string(gasnetErrorName(retval)), " ", - std::to_string(gasnetErrorDesc(retval))); - gasnet_exit(retval); + + bool expected = false; + if (stopped_.compare_exchange_strong(expected, true)) + { + stopped_ = true; + gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); + if((retval = gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) { + // throw exception + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET failed ", std::string{gasnet_ErrorName(retval)}, " ", + std::string{gasnet_ErrorDesc(retval)}); + gasnet_exit(retval); + } } } @@ -182,9 +203,10 @@ namespace hpx::parcelset { } bool background_work( - std::size_t /* num_thread */, parcelport_background_mode mode) + std::size_t num_thread, parcelport_background_mode mode) { - if (stopped_) + if (stopped_.load(std::memory_order_acquire) || + num_thread >= background_threads_) { return false; } @@ -230,6 +252,8 @@ namespace hpx::parcelset { } } + std::size_t background_threads_; + void early_write_handler(std::error_code const& ec, parcel const& p) { if (ec) @@ -291,6 +315,6 @@ namespace hpx::traits { }; } // namespace hpx::traits -HPX_REGISTER_PARCELPORT(hpx::parcelset::policies::gasnet::connection_handler, gasnet) +HPX_REGISTER_PARCELPORT(hpx::parcelset::policies::gasnet::parcelport, gasnet) #endif diff --git a/libs/full/parcelport_gasnet/tests/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/CMakeLists.txt index fed23bb2b5a5..3f1060f293c3 100644 --- a/libs/full/parcelport_gasnet/tests/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/tests/CMakeLists.txt @@ -8,35 +8,35 @@ include(HPX_Message) if(HPX_WITH_TESTS) if(HPX_WITH_TESTS_UNIT) - add_hpx_pseudo_target(tests.unit.modules.parcelport_lci) + add_hpx_pseudo_target(tests.unit.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( - tests.unit.modules tests.unit.modules.parcelport_lci + tests.unit.modules tests.unit.modules.parcelport_gasnet ) add_subdirectory(unit) endif() if(HPX_WITH_TESTS_REGRESSIONS) - add_hpx_pseudo_target(tests.regressions.modules.parcelport_lci) + add_hpx_pseudo_target(tests.regressions.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( - tests.regressions.modules tests.regressions.modules.parcelport_lci + tests.regressions.modules tests.regressions.modules.parcelport_gasnet ) add_subdirectory(regressions) endif() if(HPX_WITH_TESTS_BENCHMARKS) - add_hpx_pseudo_target(tests.performance.modules.parcelport_lci) + add_hpx_pseudo_target(tests.performance.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( - tests.performance.modules tests.performance.modules.parcelport_lci + tests.performance.modules tests.performance.modules.parcelport_gasnet ) add_subdirectory(performance) endif() if(HPX_WITH_TESTS_HEADERS) add_hpx_header_tests( - modules.parcelport_lci - HEADERS ${parcelport_lci_headers} + modules.parcelport_gasnet + HEADERS ${parcelport_gasnet_headers} HEADER_ROOT ${PROJECT_SOURCE_DIR}/include - DEPENDENCIES hpx_parcelport_lci + DEPENDENCIES hpx_parcelport_gasnet ) endif() endif() From 4cc6875860308796d7f693302799ab454aa30dcf Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 24 Jun 2023 22:39:57 -0400 Subject: [PATCH 011/333] clean compile --- libs/core/CMakeLists.txt | 2 +- .../include/hpx/gasnet_base/gasnet_environment.hpp | 6 ++---- libs/core/gasnet_base/src/gasnet_environment.cpp | 12 +++--------- libs/full/CMakeLists.txt | 2 +- .../hpx/parcelport_gasnet/sender_connection.hpp | 6 +++--- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index aef704f82f90..ddd39a0a0baa 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -41,6 +41,7 @@ set(_hpx_core_modules format functional futures + gasnet_base hardware hashing include_local @@ -55,7 +56,6 @@ set(_hpx_core_modules logging memory mpi_base - gasnet_base pack_traversal plugin prefix diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp index 2ecdb8db9412..98c7310ab124 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -44,11 +44,9 @@ namespace hpx { namespace util { static bool gettable(const int node, void* start, const size_t len); - template - static void put(T * addr, const int rank, T * raddr, const std::size_t size); + static void put(std::uint8_t * addr, const int rank, std::uint8_t * raddr, const std::size_t size); - template - static void get(T * addr, const int rank, T * raddr, const std::size_t size); + static void get(std::uint8_t * addr, const int rank, std::uint8_t * raddr, const std::size_t size); struct HPX_CORE_EXPORT scoped_lock { diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index d46aaa7b12d8..e24eb9a7d0bb 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -418,7 +418,7 @@ namespace hpx { namespace util { return name; } - bool gettable(const int node, void* start, const size_t len) + bool gasnet_environment::gettable(const int node, void* start, const size_t len) { const uintptr_t segstart = (uintptr_t) gasnet_environment::segments[node].addr; const uintptr_t segend = segstart + gasnet_environment::segments[node].size; @@ -431,10 +431,7 @@ namespace hpx { namespace util { reqend <= segend); } - template - void gasnet_environment::put(T * addr, const int node, T * raddr, const std::size_t size) { - static_assert(std::is_integral::value, "Error! ***gasnet_environment::put***\tIntegral required."); - + void gasnet_environment::put(std::uint8_t * addr, const int node, std::uint8_t * raddr, const std::size_t size) { const bool in_remote_seg = gettable(node, raddr, size); if(in_remote_seg) { const std::lock_guard lk(segment_mutex[node]); @@ -479,10 +476,7 @@ namespace hpx { namespace util { } } - template - void gasnet_environment::get(T * addr, const int node, T * raddr, const std::size_t size) { - static_assert(std::is_integral::value, "Error! ***gasnet_environment::get***\tIntegral required."); - + void gasnet_environment::get(std::uint8_t * addr, const int node, std::uint8_t * raddr, const std::size_t size) { bool remote_in_segment = false; if (rank() == node) { diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index 8e9a582b9e99..c63bbce9c8f0 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -31,10 +31,10 @@ set(_hpx_full_modules lcos_distributed naming naming_base + parcelport_gasnet parcelport_lci parcelport_libfabric parcelport_mpi - parcelport_gasnet parcelport_tcp parcelset parcelset_base diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index 406834dc0340..396b7e556227 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -161,7 +161,7 @@ namespace hpx::parcelset::policies::gasnet { // put from this localities gasnet shared memory segment // into the remote locality (dst_)'s shared memory segment // - hpx::util::gasnet_environment::put( + hpx::util::gasnet_environment::put( static_cast( hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr ), @@ -229,7 +229,7 @@ namespace hpx::parcelset::policies::gasnet { hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, buffer_.data_.data(), buffer_.data_.size()); - hpx::util::gasnet_environment::put( + hpx::util::gasnet_environment::put( static_cast( hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr ), @@ -268,7 +268,7 @@ namespace hpx::parcelset::policies::gasnet { static_cast(c.size_) ); - hpx::util::gasnet_environment::put( + hpx::util::gasnet_environment::put( static_cast( hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr ), From 8bbaacb415375f76c3c17d61dc98668cc8666e17 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 24 Jun 2023 23:00:53 -0400 Subject: [PATCH 012/333] fixed debug build issues --- libs/core/gasnet_base/src/gasnet_environment.cpp | 2 +- .../include/hpx/parcelport_gasnet/sender_connection.hpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index e24eb9a7d0bb..32574862d90e 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -524,7 +524,7 @@ namespace hpx { namespace util { } local_buf = calloc(1, buf_sz); - assert(gettable(chpl_nodeID, local_buf, buf_sz)); + assert(gettable(node, local_buf, buf_sz)); } // do a PUT on the remote locale back to here. diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index 396b7e556227..c7b7b3d84b80 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -185,8 +185,6 @@ namespace hpx::parcelset::policies::gasnet { return false; } - HPX_ASSERT(request_ptr_ == nullptr); - std::vector& chunks = buffer_.transmission_chunks_; if (!chunks.empty()) From fa43d6daac3e137163fdcdb33dd060aba1520c09 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 24 Jun 2023 23:53:30 -0400 Subject: [PATCH 013/333] updated template, rm'd hpx_setup_gasnet --- cmake/templates/HPXConfig.cmake.in | 2 -- libs/core/gasnet_base/CMakeLists.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 5c2b04da73e9..7bb69ee4c570 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -54,8 +54,6 @@ endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) set(GASNET_ROOT "@GASNET_ROOT@") - include(HPX_SetupGASNET) - hpx_setup_gasnet() endif() # Eve can be installed by HPX or externally installed. In the first case we use diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index de76d7607687..346ace51b80b 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -10,6 +10,7 @@ if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND (HPX_WITH_PARCELP endif() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + include(FindGasnet) find_gasnet() From 408d2d1b6a80aaf496f046b2267e36b6fa5b7c2a Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 25 Jun 2023 00:15:07 -0400 Subject: [PATCH 014/333] added gasnet support --- .../src/command_line_handling.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index 5c5f98339c61..a2946563f961 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -990,6 +990,18 @@ namespace hpx::util { node_ = static_cast(util::lci_environment::rank()); } #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) + // better to put GASNET init after MPI init, since GASNET will also + // initialize MPI if MPI is not already initialized. + if (util::gasnet_environment::check_gasnet_environment(rtcfg_)) + { + util::gasnet_environment::init(&argc, &argv, rtcfg_); + num_localities_ = + static_cast(util::gasnet_environment::size()); + node_ = static_cast(util::gasnet_environment::rank()); + } +#endif // load plugin modules (after first pass of command line handling, so // that settings given on command line could propagate to modules) From ccc968d5d26a41fb11877afb3b07a9841cf3332c Mon Sep 17 00:00:00 2001 From: Chris Taylor Date: Fri, 7 Jul 2023 12:19:39 +0000 Subject: [PATCH 015/333] updated cmake script --- cmake/FindGasnet.cmake | 482 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 479 insertions(+), 3 deletions(-) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 3c87c9a39100..1976599be29a 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - +# macro(find_gasnet) set(GASNET_MPI_FOUND FALSE) @@ -14,21 +14,497 @@ set(GASNET_SMP_FOUND FALSE) find_package(PkgConfig REQUIRED QUIET COMPONENTS) if(HPX_WITH_PARCELPORT_GASNET_MPI) - pkg_check_modules(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-mpi-par) + message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-mpi-par) set(GASNET_MPI_FOUND TRUE) hpx_setup_mpi() endif() if(HPX_WITH_PARCELPORT_GASNET_UDP) - pkg_check_modules(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-udp-par) + message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-udp-par) set(GASNET_UDP_FOUND TRUE) endif() if(HPX_WITH_PARCELPORT_GASNET_SMP) + message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-smp-par) set(GASNET_SMP_FOUND TRUE) endif() target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) +if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() +endif() + +if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() +endif() + +if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library(FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} + NO_CACHE + NO_CMAKE_FIND_ROOT_PATH + NO_DEFAULT_PATH) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() +endif() + +if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library(FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} + NO_CACHE + NO_CMAKE_FIND_ROOT_PATH + NO_DEFAULT_PATH) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + +endif() + +if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() +endif() + +if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() +endif() + +if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library(FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} + NO_CACHE + NO_CMAKE_FIND_ROOT_PATH + NO_DEFAULT_PATH) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + endif() +endif() + +if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library(FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} + NO_CACHE + NO_CMAKE_FIND_ROOT_PATH + NO_DEFAULT_PATH) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() +endif() + +set_target_properties(PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}") +set_target_properties(PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}") + endmacro() From 8e6b0a176ac0f33a4a7dfec22b17b6336a08a37a Mon Sep 17 00:00:00 2001 From: gonidelis Date: Wed, 22 Feb 2023 19:28:50 -0600 Subject: [PATCH 016/333] Add 'v' prefix before release tagging to conform to GitHub's default convention --- docs/sphinx/releases/whats_new_1_9_0.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/releases/whats_new_1_9_0.rst b/docs/sphinx/releases/whats_new_1_9_0.rst index d97d7b70ce33..805c3ec2a28f 100644 --- a/docs/sphinx/releases/whats_new_1_9_0.rst +++ b/docs/sphinx/releases/whats_new_1_9_0.rst @@ -169,7 +169,7 @@ Closed pull requests * :hpx-pr:`6117` - Initialize buffer_allocate_time_ to 0 * :hpx-pr:`6116` - Add new command line argument --hpx:loopback_network * :hpx-pr:`6115` - Modernize modules of levels 14, 15, and 16 -* :hpx-pr:`6114` - Enhance the formatting of the documentation +* :hpx-pr:`6114` - Enhance the formatting of the documentation * :hpx-pr:`6113` - Modernize modules in module level 11, 12, and 13 * :hpx-pr:`6112` - Modernize modules from levels 9 and 10 * :hpx-pr:`6111` - Modernize all modules from module level 8 @@ -247,7 +247,7 @@ Closed pull requests * :hpx-pr:`6030` - Adding parcelport initialization hook for resource partitioner operation * :hpx-pr:`6029` - Simplify startup code * :hpx-pr:`6027` - Add/Fix documentation in Public API page -* :hpx-pr:`6026` - add option hpx:force_ipv4 to force resolving hostnames to ipv4 adresses +* :hpx-pr:`6026` - add option hpx:force_ipv4 to force resolving hostnames to ipv4 addresses * :hpx-pr:`6025` - build(docs): remove leftover sections * :hpx-pr:`6023` - Minor fixes on "How to build on Windows" * :hpx-pr:`6022` - build(doxy): don't extract private members @@ -273,11 +273,11 @@ Closed pull requests * :hpx-pr:`5990` - P2300 enhancements * :hpx-pr:`5989` - Fix missing documentation in Public API page * :hpx-pr:`5987` - Attempting to fix timed executor API -* :hpx-pr:`5986` - Fix warnings when building docs +* :hpx-pr:`5986` - Fix warnings when building docs * :hpx-pr:`5985` - Re-add deprecated tag_policy_tag et.al. types that were removed in V1.8.1 * :hpx-pr:`5981` - docs: add docs for condition_variable.hpp * :hpx-pr:`5980` - More work on execution::read -* :hpx-pr:`5979` - Unsupport clang-v8 and clang-v9, switch LSU clang-v13 to C++17 +* :hpx-pr:`5979` - Unsupported clang-v8 and clang-v9, switch LSU clang-v13 to C++17 * :hpx-pr:`5977` - fix: Compilation errors for -std=c++17 builders * :hpx-pr:`5975` - docs: fix & improve parallel algorithms documentation 5 * :hpx-pr:`5974` - [P2300] Adapt get completion signatures for awaitable senders From 4da4c35871b988c63f75b4bf2d3672a11fa09fa7 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 27 Mar 2023 17:43:28 +0300 Subject: [PATCH 017/333] Initial relocation trait definitions --- .../include/hpx/algorithms/traits/pointer_category.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp index 0cc691dad638..fcb782fdb3fb 100644 --- a/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp +++ b/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp @@ -105,6 +105,9 @@ namespace hpx::traits { template > + // Why do we need to check if the iterators are contiguous in the trivially + // copyable case? Is this category refering to buffers of + // the underlying types or the individual objects? struct pointer_move_category { using type = std::conditional_t< From 27acd6cc46df70578ae243e74916a61aaee763eb Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Tue, 28 Mar 2023 17:33:44 +0300 Subject: [PATCH 018/333] Split into files --- .../hpx/algorithms/traits/is_relocatable.hpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp new file mode 100644 index 000000000000..cbade6058a3b --- /dev/null +++ b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp @@ -0,0 +1,23 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +namespace hpx::traits { + + template + struct is_relocatable + { + static constexpr bool value = + std::is_move_constructible_v && std::is_destructible_v; + }; + + template + inline constexpr bool is_relocatable_v = is_relocatable::value; +} // namespace hpx::traits + From c5ec3584e41f02464367009d6057ed8402f0a51d Mon Sep 17 00:00:00 2001 From: Isidoros Date: Tue, 28 Mar 2023 19:01:11 +0300 Subject: [PATCH 019/333] include type_traits Co-authored-by: Hartmut Kaiser --- .../algorithms/include/hpx/algorithms/traits/is_relocatable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp index cbade6058a3b..f407ec4c46e4 100644 --- a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp +++ b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp @@ -6,7 +6,7 @@ #pragma once -#include +#include namespace hpx::traits { From 11326c89c3a4fc0710daad3a0301d8c3dad32cdf Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Tue, 28 Mar 2023 19:02:41 +0300 Subject: [PATCH 020/333] change copyright name --- .../algorithms/include/hpx/algorithms/traits/is_relocatable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp index f407ec4c46e4..b6acee25d53e 100644 --- a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp +++ b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Hartmut Kaiser +// Copyright (c) 2023 Isidoros Tsaousis-Seiras // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying From adde370a4185a57dec09931956226300e07fa04b Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Tue, 28 Mar 2023 21:42:31 +0300 Subject: [PATCH 021/333] clang-format + cmake additions --- libs/core/algorithms/CMakeLists.txt | 2 ++ .../hpx/algorithms/traits/is_relocatable.hpp | 17 ++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libs/core/algorithms/CMakeLists.txt b/libs/core/algorithms/CMakeLists.txt index c2fac77c66cf..2f0abc0024c8 100644 --- a/libs/core/algorithms/CMakeLists.txt +++ b/libs/core/algorithms/CMakeLists.txt @@ -6,6 +6,7 @@ set(algorithms_headers hpx/algorithms/traits/is_pair.hpp + hpx/algorithms/traits/is_relocatable.hpp hpx/algorithms/traits/is_value_proxy.hpp hpx/algorithms/traits/pointer_category.hpp hpx/algorithms/traits/projected.hpp @@ -225,6 +226,7 @@ set(algorithms_compat_headers hpx/algorithms.hpp => hpx/algorithm.hpp hpx/parallel/traits/projected.hpp => hpx/algorithms/traits/projected.hpp hpx/parallel/traits/projected_range.hpp => hpx/algorithms/traits/projected_range.hpp + hpx/traits/is_relocatable.hpp => hpx/algorithms/traits/is_relocatable.hpp hpx/traits/is_value_proxy.hpp => hpx/algorithms/traits/is_value_proxy.hpp hpx/traits/pointer_category.hpp => hpx/algorithms/traits/pointer_category.hpp hpx/traits/segmented_iterator_traits.hpp => hpx/algorithms/traits/segmented_iterator_traits.hpp diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp index b6acee25d53e..925c4c31d0f8 100644 --- a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp +++ b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp @@ -10,14 +10,13 @@ namespace hpx::traits { - template - struct is_relocatable - { - static constexpr bool value = - std::is_move_constructible_v && std::is_destructible_v; - }; + template + struct is_relocatable + { + static constexpr bool value = + std::is_move_constructible_v && std::is_destructible_v; + }; - template - inline constexpr bool is_relocatable_v = is_relocatable::value; + template + inline constexpr bool is_relocatable_v = is_relocatable::value; } // namespace hpx::traits - From ed0dbaf164aa28828e03074f8fc861f1d34b5d4c Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 29 Mar 2023 01:09:55 +0300 Subject: [PATCH 022/333] Remove from algorithms_compat_headers --- libs/core/algorithms/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/algorithms/CMakeLists.txt b/libs/core/algorithms/CMakeLists.txt index 2f0abc0024c8..8f37b9277a31 100644 --- a/libs/core/algorithms/CMakeLists.txt +++ b/libs/core/algorithms/CMakeLists.txt @@ -226,7 +226,6 @@ set(algorithms_compat_headers hpx/algorithms.hpp => hpx/algorithm.hpp hpx/parallel/traits/projected.hpp => hpx/algorithms/traits/projected.hpp hpx/parallel/traits/projected_range.hpp => hpx/algorithms/traits/projected_range.hpp - hpx/traits/is_relocatable.hpp => hpx/algorithms/traits/is_relocatable.hpp hpx/traits/is_value_proxy.hpp => hpx/algorithms/traits/is_value_proxy.hpp hpx/traits/pointer_category.hpp => hpx/algorithms/traits/pointer_category.hpp hpx/traits/segmented_iterator_traits.hpp => hpx/algorithms/traits/segmented_iterator_traits.hpp From 984cac9f2b7595f5a4b7e540cbbf82f5cafae68d Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 18:32:09 +0300 Subject: [PATCH 023/333] Added static assertion tests --- .../algorithms/util/test_is_relocatable.cpp | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp new file mode 100644 index 000000000000..6f393afc6d98 --- /dev/null +++ b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2017 Francisco Jose Tapia +// Copyright (c) 2020 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include + +#include + +#include + +// Integral types are relocatable +static_assert(hpx::relocatable); +static_assert(hpx::relocatable); +static_assert(hpx::relocatable); +static_assert(hpx::relocatable); + +// Array types are not move-constructible and thus not relocatable +static_assert(!hpx::relocatable); +static_assert(!hpx::relocatable); +static_assert(!hpx::relocatable); +static_assert(!hpx::relocatable); + +// Function types are not move-constructible and thus not relocatable +static_assert(!hpx::relocatable); + +// Void types are not move-constructible and thus not relocatable +static_assert(!hpx::relocatable); +static_assert(!hpx::relocatable); + +// std::mutex is not relocatable +static_assert(!hpx::relocatable); + +struct NotDestructible { + NotDestructible(const NotDestructible&); + NotDestructible(NotDestructible&&); + ~NotDestructible() = delete; +}; + +static_assert(!hpx::relocatable); + +struct NotMoveConstructible { + NotMoveConstructible(const NotMoveConstructible&); + NotMoveConstructible(NotMoveConstructible&&) = delete; +}; + +static_assert(!hpx::relocatable); + +struct NotCopyConstructible { + NotCopyConstructible(const NotCopyConstructible&) = delete; + NotCopyConstructible(NotCopyConstructible&&); +}; + +static_assert(hpx::relocatable); + +// reference types are relocatable +static_assert(hpx::relocatable); +static_assert(hpx::relocatable); +static_assert(hpx::relocatable); +static_assert(hpx::relocatable); +static_assert(std::relocatable); +static_assert(std::relocatable); +static_assert(std::relocatable); + +int main(int, char*[]) +{ +} From 592f15483fb0573d3a97f96ff64bef4883648932 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 20:01:35 +0300 Subject: [PATCH 024/333] clang-format --- .../algorithms/util/test_is_relocatable.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp index 6f393afc6d98..8e174fc57e09 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp +++ b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp @@ -16,7 +16,7 @@ static_assert(hpx::relocatable); static_assert(hpx::relocatable); static_assert(hpx::relocatable); -static_assert(hpx::relocatable); +static_assert(hpx::relocatable); // Array types are not move-constructible and thus not relocatable static_assert(!hpx::relocatable); @@ -34,7 +34,8 @@ static_assert(!hpx::relocatable); // std::mutex is not relocatable static_assert(!hpx::relocatable); -struct NotDestructible { +struct NotDestructible +{ NotDestructible(const NotDestructible&); NotDestructible(NotDestructible&&); ~NotDestructible() = delete; @@ -42,14 +43,16 @@ struct NotDestructible { static_assert(!hpx::relocatable); -struct NotMoveConstructible { +struct NotMoveConstructible +{ NotMoveConstructible(const NotMoveConstructible&); NotMoveConstructible(NotMoveConstructible&&) = delete; }; static_assert(!hpx::relocatable); -struct NotCopyConstructible { +struct NotCopyConstructible +{ NotCopyConstructible(const NotCopyConstructible&) = delete; NotCopyConstructible(NotCopyConstructible&&); }; @@ -59,12 +62,10 @@ static_assert(hpx::relocatable); // reference types are relocatable static_assert(hpx::relocatable); static_assert(hpx::relocatable); -static_assert(hpx::relocatable); +static_assert(hpx::relocatable); static_assert(hpx::relocatable); static_assert(std::relocatable); static_assert(std::relocatable); static_assert(std::relocatable); -int main(int, char*[]) -{ -} +int main(int, char*[]) {} From 1b69d6d643c87f02d934d70187c6d2fb997c6ccf Mon Sep 17 00:00:00 2001 From: Isidoros Date: Mon, 10 Apr 2023 18:34:35 +0300 Subject: [PATCH 025/333] Corrected copyrights name --- .../tests/unit/algorithms/util/test_is_relocatable.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp index 8e174fc57e09..9865f8648096 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp +++ b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp @@ -1,5 +1,4 @@ -// Copyright (c) 2015-2017 Francisco Jose Tapia -// Copyright (c) 2020 Hartmut Kaiser +// Copyright (c) 2023 Isidoros Tsaousis Seiras // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying From 4779ceb5cc8bff019753b0ee72468b1e052c9cd5 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 20:10:30 +0300 Subject: [PATCH 026/333] add relocatable tests to cmake --- .../core/algorithms/tests/unit/algorithms/util/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt index 95c4813c0c3d..c803032cd49a 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt +++ b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt @@ -4,8 +4,8 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -set(tests test_low_level test_merge_four test_merge_vector test_nbits - test_range +set(tests test_is_relocatable test_low_level test_merge_four test_merge_vector + test_nbits test_range ) foreach(test ${tests}) From 760a8b3933cc986857ae1f8dd22cfb78934c5de4 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 21:09:40 +0300 Subject: [PATCH 027/333] cmake-format --- libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt index c803032cd49a..25e81f7364a1 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt +++ b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -set(tests test_is_relocatable test_low_level test_merge_four test_merge_vector +set(tests test_is_relocatable test_low_level test_merge_four test_merge_vector test_nbits test_range ) From fcb523f48907bb7c2213778cd96b4e4ba43e9761 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 21:11:33 +0300 Subject: [PATCH 028/333] Add copyright --- libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt index 25e81f7364a1..426c259849fc 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt +++ b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright (c) 2020 Hartmut Kaiser +# Copyright (c) 2023 Isidoros Tsaousis Seiras # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying From 110a7ca63ef6483eb1dd414b876b9c597ad80ba4 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 23:28:09 +0300 Subject: [PATCH 029/333] typo in assertions --- .../algorithms/util/test_is_relocatable.cpp | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp index 9865f8648096..53e9391e3cae 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp +++ b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp @@ -12,26 +12,26 @@ #include // Integral types are relocatable -static_assert(hpx::relocatable); -static_assert(hpx::relocatable); -static_assert(hpx::relocatable); -static_assert(hpx::relocatable); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); // Array types are not move-constructible and thus not relocatable -static_assert(!hpx::relocatable); -static_assert(!hpx::relocatable); -static_assert(!hpx::relocatable); -static_assert(!hpx::relocatable); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); // Function types are not move-constructible and thus not relocatable -static_assert(!hpx::relocatable); +static_assert(!hpx::is_relocatable_v); // Void types are not move-constructible and thus not relocatable -static_assert(!hpx::relocatable); -static_assert(!hpx::relocatable); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); // std::mutex is not relocatable -static_assert(!hpx::relocatable); +static_assert(!hpx::is_relocatable_v); struct NotDestructible { @@ -40,7 +40,7 @@ struct NotDestructible ~NotDestructible() = delete; }; -static_assert(!hpx::relocatable); +static_assert(!hpx::is_relocatable_v); struct NotMoveConstructible { @@ -48,7 +48,7 @@ struct NotMoveConstructible NotMoveConstructible(NotMoveConstructible&&) = delete; }; -static_assert(!hpx::relocatable); +static_assert(!hpx::is_relocatable_v); struct NotCopyConstructible { @@ -56,13 +56,13 @@ struct NotCopyConstructible NotCopyConstructible(NotCopyConstructible&&); }; -static_assert(hpx::relocatable); +static_assert(hpx::is_relocatable_v); // reference types are relocatable -static_assert(hpx::relocatable); -static_assert(hpx::relocatable); -static_assert(hpx::relocatable); -static_assert(hpx::relocatable); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); static_assert(std::relocatable); static_assert(std::relocatable); static_assert(std::relocatable); From 6a26ab567e05795715ca0d9d55c4e67d9acec236 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Mon, 10 Apr 2023 23:30:19 +0300 Subject: [PATCH 030/333] fix second typo --- .../tests/unit/algorithms/util/test_is_relocatable.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp index 53e9391e3cae..1ee1dcf1ac80 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp +++ b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp @@ -63,8 +63,8 @@ static_assert(hpx::is_relocatable_v); static_assert(hpx::is_relocatable_v); static_assert(hpx::is_relocatable_v); static_assert(hpx::is_relocatable_v); -static_assert(std::relocatable); -static_assert(std::relocatable); -static_assert(std::relocatable); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); int main(int, char*[]) {} From ce8abb255e96533b2d315bd7a0fe022309cf521f Mon Sep 17 00:00:00 2001 From: Isidoros Date: Tue, 25 Apr 2023 18:38:12 +0300 Subject: [PATCH 031/333] remove name from cr --- libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt index 426c259849fc..25e81f7364a1 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt +++ b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt @@ -1,5 +1,4 @@ # Copyright (c) 2020 Hartmut Kaiser -# Copyright (c) 2023 Isidoros Tsaousis Seiras # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying From dcf8087bcc00d547f7023d549e52a24add81dfdb Mon Sep 17 00:00:00 2001 From: Isidoros Date: Tue, 25 Apr 2023 18:39:07 +0300 Subject: [PATCH 032/333] Corrected spelling of my name (added the dash between last names) --- .../tests/unit/algorithms/util/test_is_relocatable.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp index 1ee1dcf1ac80..af73f07695a5 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp +++ b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Isidoros Tsaousis Seiras +// Copyright (c) 2023 Isidoros Tsaousis-Seiras // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying From 6b17697b17ba0d7c4e0ba720fa5e56ddd4223157 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 5 May 2023 09:39:15 -0500 Subject: [PATCH 033/333] Moving is_relocatable to namespace hpx --- .../include/hpx/algorithms/traits/is_relocatable.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp index 925c4c31d0f8..5d983c3ec168 100644 --- a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp +++ b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp @@ -8,7 +8,7 @@ #include -namespace hpx::traits { +namespace hpx { template struct is_relocatable @@ -19,4 +19,4 @@ namespace hpx::traits { template inline constexpr bool is_relocatable_v = is_relocatable::value; -} // namespace hpx::traits +} // namespace hpx From 3ee0da3c6804e37dd2b35fe13cb164b3fad83b57 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 30 Apr 2023 16:21:31 -0500 Subject: [PATCH 034/333] Adding zero-copy support on the receiving end of the TCP parcel port - flyby: cleaning up and modernizing TCP parcel port --- libs/full/parcelset/src/parcel.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/full/parcelset/src/parcel.cpp b/libs/full/parcelset/src/parcel.cpp index 5dff726b9cca..8d385d23e0f5 100644 --- a/libs/full/parcelset/src/parcel.cpp +++ b/libs/full/parcelset/src/parcel.cpp @@ -480,7 +480,6 @@ namespace hpx::parcelset::detail { reinterpret_cast( action_->get_parent_thread_id().get())); #endif - return false; } From 8fefa9dbaf7992a9ec2442cdd85bc1f5b986bc65 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 5 May 2023 07:43:03 -0500 Subject: [PATCH 035/333] Merge changes from #5910 --- libs/full/parcelset/src/parcel.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/full/parcelset/src/parcel.cpp b/libs/full/parcelset/src/parcel.cpp index 8d385d23e0f5..5dff726b9cca 100644 --- a/libs/full/parcelset/src/parcel.cpp +++ b/libs/full/parcelset/src/parcel.cpp @@ -480,6 +480,7 @@ namespace hpx::parcelset::detail { reinterpret_cast( action_->get_parent_thread_id().get())); #endif + return false; } From 430d2f9aabd5db7ada58af3dccbe640edfbf30b4 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 11 Jul 2023 09:33:23 -0500 Subject: [PATCH 036/333] fixed header file issue --- libs/core/gasnet_base/src/gasnet_environment.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 32574862d90e..79de63e6ab99 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include @@ -193,7 +193,7 @@ void wait_done_obj(done_t* done, bool do_yield) while (!done->flag) { am_poll_try(); if (do_yield) - hpx::this_thread::yield(); + hpx::execution_base::this_thread::yield(); } } From eb2ded0cecde4cfea0f2ae7523e05ab9e6756923 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 11 Jul 2023 10:14:03 -0500 Subject: [PATCH 037/333] fixed missing header file issue --- libs/core/gasnet_base/src/gasnet_environment.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 79de63e6ab99..2b64060a5f75 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -193,7 +193,7 @@ void wait_done_obj(done_t* done, bool do_yield) while (!done->flag) { am_poll_try(); if (do_yield) - hpx::execution_base::this_thread::yield(); + hpx::this_thread::yield(); } } From e55a0efbcb370d2b7b83e59b0cb15b76db54a54a Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 11 Jul 2023 13:09:54 -0500 Subject: [PATCH 038/333] adding support for gasnet-ofi, gasnet-ucx, gasnet-ibv; fixed 1 incorrect compilation issue --- CMakeLists.txt | 24 ++++++++++ cmake/FindGasnet.cmake | 21 +++++++++ libs/core/gasnet_base/CMakeLists.txt | 2 +- .../gasnet_base/src/gasnet_environment.cpp | 2 +- .../full/command_line_handling/CMakeLists.txt | 6 +++ libs/full/parcelport_gasnet/CMakeLists.txt | 44 ++++++++++++++++++- 6 files changed, 96 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56c2540cedde..ed25ba277c2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1176,6 +1176,30 @@ if(HPX_WITH_NETWORKING) hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) endif() + hpx_option( + HPX_WITH_PARCELPORT_GASNET_OFI BOOL "Enable the GASNET-OFI based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET_OFI) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_OFI) + endif() + + hpx_option( + HPX_WITH_PARCELPORT_GASNET_UCX BOOL "Enable the GASNET-UCX based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET_UCX) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UCX) + endif() + + hpx_option( + HPX_WITH_PARCELPORT_GASNET_IBV BOOL "Enable the GASNET-IBV based parcelport." OFF + CATEGORY "Parcelport" + ) + if(HPX_WITH_PARCELPORT_GASNET_IBV) + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_IBV) + endif() + hpx_option( HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON CATEGORY "Parcelport" diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 1976599be29a..9da11fde43fa 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -10,6 +10,9 @@ macro(find_gasnet) set(GASNET_MPI_FOUND FALSE) set(GASNET_UDP_FOUND FALSE) set(GASNET_SMP_FOUND FALSE) +set(GASNET_OFI_FOUND FALSE) +set(GASNET_UCX_FOUND FALSE) +set(GASNET_IBV_FOUND FALSE) find_package(PkgConfig REQUIRED QUIET COMPONENTS) @@ -32,6 +35,24 @@ if(HPX_WITH_PARCELPORT_GASNET_SMP) set(GASNET_SMP_FOUND TRUE) endif() +if(HPX_WITH_PARCELPORT_GASNET_OFI) + message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-ofi-par) + set(GASNET_OFI_FOUND TRUE) +endif() + +if(HPX_WITH_PARCELPORT_GASNET_UCX) + message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-ucx-par) + set(GASNET_UCX_FOUND TRUE) +endif() + +if(HPX_WITH_PARCELPORT_GASNET_IBV) + message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-ibv-par) + set(GASNET_IBV_FOUND TRUE) +endif() + target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) if(GASNET_CFLAGS) diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 346ace51b80b..0280a579339b 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_GASNET_UDP))) +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_GASNET_UDP OR HPX_WITH_PARCELPORT_GASNET_OFI OR HPX_WITH_PARCELPORT_GASNET_UCX OR HPX_WITH_PARCELPORT_GASNET_IBV))) message(STATUS "*** PARCELPORT_GASNET not set") return() endif() diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 2b64060a5f75..e001b82f84dc 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -180,7 +180,7 @@ static inline void am_poll_try() { // AM-based (udp/amudp, mpi/ammpi) serializing can hurt performance. // #if defined(GASNET_CONDUIT_IBV) || defined(GASNET_CONDUIT_UCX) || defined(GASNET_CONDUIT_ARIES) || defined(GASNET_CONDUIT_OFI) - std::lock_guard lk(gasnet_environment::pollingLock); + std::lock_guard lk(hpx::util::gasnet_environment::pollingLock); (void) gasnet_AMPoll(); #else (void) gasnet_AMPoll(); diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index 9733990f4291..1247a71b61de 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -40,6 +40,12 @@ elseif(HPX_WITH_PARCELPORT_GASNET_UDP) set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) elseif(HPX_WITH_PARCELPORT_GASNET_SMP) set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +elseif(HPX_WITH_PARCELPORT_GASNET_OFI) + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +elseif(HPX_WITH_PARCELPORT_GASNET_UCX) + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +elseif(HPX_WITH_PARCELPORT_GASNET_IBV) + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) endif() include(HPX_AddModule) diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 04d17ba12609..d5e2ebf0e1cc 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_UDP))) +if(NOT (HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_UDP OR HPX_WITH_PARCELPORT_OFI OR HPX_WITH_PARCELPORT_UCX OR HPX_WITH_PARCELPORT_IBV))) return() endif() @@ -54,6 +54,48 @@ link_directories(${GASNET_LIBRARY_DIRS}) CMAKE_SUBDIRS examples tests ) +elseif(GASNET_OFI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OFI) +link_directories(${GASNET_LIBRARY_DIRS}) + include(HPX_AddModule) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests + ) + +elseif(GASNET_UCX_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UCX) +link_directories(${GASNET_LIBRARY_DIRS}) + include(HPX_AddModule) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests + ) + +elseif(GASNET_IBV_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_IBV) +link_directories(${GASNET_LIBRARY_DIRS}) + include(HPX_AddModule) + add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + COMPAT_HEADERS ${parcelport_gasnet_compat_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests + ) + elseif(GASNET_SMP_FOUND) link_directories(${GASNET_LIBRARY_DIRS}) include(HPX_AddModule) From 5067ef144d2c9cd30cd53b5aceb64cf90e858ab5 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 11 Jul 2023 20:11:03 -0500 Subject: [PATCH 039/333] added gasnet and the gasnet launcher (amudprun) to test file, hpxconfig, and hpxrun --- cmake/HPX_AddTest.cmake | 123 ++++++++++++++++++++++++++++- cmake/templates/HPXConfig.cmake.in | 9 ++- cmake/templates/hpxrun.py.in | 24 +++++- 3 files changed, 152 insertions(+), 4 deletions(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 738b529ccfe5..ce884f5cc0fe 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -6,7 +6,8 @@ function(add_hpx_test category name) set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI - NO_PARCELPORT_LCI + NO_PARCELPORT_LCI NO_PARCELPORT_GASNET_SMP NO_PARCELPORT_GASNET_UDP + NO_PARCELPORT_GASNET_UCX NO_PARCELPORT_GASNET_OFI NO_PARCELPORT_GASNET_IBV ) set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT RUNWRAPPER @@ -172,6 +173,126 @@ function(add_hpx_test category name) endif() endif() endif() + if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_SMP AND NOT ${${name}_NO_PARCELPORT_GASNET_SMP}) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.gasnet.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" + ${args} + ) + set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() + if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_UDP AND NOT ${${name}_NO_PARCELPORT_GASNET_UDP}) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.gasnet.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" + ${args} + ) + set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() + if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_OFI AND NOT ${${name}_NO_PARCELPORT_GASNET_OFI}) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.gasnet.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" + ${args} + ) + set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() + if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_UCX AND NOT ${${name}_NO_PARCELPORT_GASNET_UCX}) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.gasnet.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" + ${args} + ) + set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() + if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_IBV AND NOT ${${name}_NO_PARCELPORT_GASNET_IBV}) + set(_add_test FALSE) + if(DEFINED ${name}_PARCELPORTS) + set(PP_FOUND -1) + list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) + if(NOT PP_FOUND EQUAL -1) + set(_add_test TRUE) + endif() + else() + set(_add_test TRUE) + endif() + if(_add_test) + set(_full_name "${category}.distributed.gasnet.${name}") + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" + ${args} + ) + set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) + set_tests_properties( + "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} + ) + endif() + endif() + endif() if(HPX_WITH_PARCELPORT_TCP AND NOT ${${name}_NO_PARCELPORT_TCP}) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 7bb69ee4c570..7a68ee418a21 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,8 +52,15 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND + ( HPX_WITH_PARCELPORT_GASNET_SMP OR + HPX_WITH_PARCELPORT_GASNET_UDP OR + HPX_WITH_PARCELPORT_GASNET_OFI OR + HPX_WITH_PARCELPORT_GASNET_UCX OR + HPX_WITH_PARCELPORT_GASNET_IBV ) ) set(GASNET_ROOT "@GASNET_ROOT@") + include(FindGasnet) + find_gasnet() endif() # Eve can be installed by HPX or externally installed. In the first case we use diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 70a5991e22f0..4955cdf75537 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -195,6 +195,22 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) +# Run with amudprun +# This is executing mpiexec with the "-np" option set to the number of localities +def run_gasnet(cmd, localities, verbose): + mpiexec = 'amudprun' + if mpiexec == '': + mpiexec = 'amudprun' + if mpiexec == '': + msg = 'amudprun not available on this platform. ' + msg += 'Please rerun CMake with HPX_PARCELPORT_GASNET=True.' + print(msg, sys.stderr) + sys.exit(1) + exec_cmd = [mpiexec, '-np', str(localities)] + cmd + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) + # Run with srun # This is executing srun with the '-n' option set to the number of localities def run_srun(cmd, localities, verbose): @@ -218,6 +234,9 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'mpi': assert nodes is None, "nodes option only valid with tcp parcelport." run_mpi(cmd, localities, verbose) + if runwrapper == 'gasnet': + assert nodes is None, "nodes option only valid with tcp parcelport." + run_gasnet(cmd, localities, verbose) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -239,6 +258,7 @@ def build_cmd(options, args): select_parcelport = (lambda pp: ['--hpx:ini=hpx.parcel.mpi.priority=1000', '--hpx:ini=hpx.parcel.mpi.enable=1', '--hpx:ini=hpx.parcel.bootstrap=mpi'] if pp == 'mpi' else ['--hpx:ini=hpx.parcel.lci.priority=1000', '--hpx:ini=hpx.parcel.lci.enable=1', '--hpx:ini=hpx.parcel.bootstrap=lci'] if pp == 'lci' + else ['--hpx:ini=hpx.parcel.gasnet.priority=1000', '--hpx:ini=hpx.parcel.gasnet.enable=1', '--hpx:ini=hpx.parcel.bootstrap=gasnet'] if pp == 'gasnet' else ['--hpx:ini=hpx.parcel.tcp.priority=1000', '--hpx:ini=hpx.parcel.tcp.enable=1'] if pp == 'tcp' else []) cmd += select_parcelport(options.parcelport) @@ -275,7 +295,7 @@ def check_options(parser, options, args): print('Can not start less than one thread per locality', sys.stderr) sys.exit(1) - check_valid_parcelport = (lambda x: x == 'mpi' or x == 'lci' or x == 'tcp' or x == 'none'); + check_valid_parcelport = (lambda x: x == 'mpi' or x == 'lci' or x == 'gasnet' or x == 'tcp' or x == 'none'); if not check_valid_parcelport(options.parcelport): print('Error: Parcelport option not valid\n', sys.stderr) parser.print_help() @@ -361,7 +381,7 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, gaset, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' From fe03bf6dee674a1e17fbf7db7c2d71acf9b62f33 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 11 Jul 2023 20:39:01 -0500 Subject: [PATCH 040/333] fixed priority issue in gasnet_environment.cpp --- libs/core/gasnet_base/src/gasnet_environment.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index e001b82f84dc..44436a8934c6 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -238,13 +238,14 @@ namespace hpx { namespace util { // - The parcelport is explicitly disabled // - The application is not run in an GASNET environment // - The TCP parcelport is enabled and has higher priority + // if (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) == 0 || - (get_entry_as(cfg, "hpx.parcel.udp.enable", 1) && - (get_entry_as(cfg, "hpx.parcel.udp.priority", 1) > + (get_entry_as(cfg, "hpx.parcel.tcp.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > get_entry_as(cfg, "hpx.parcel.gasnet.priority", 0))) || (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) && (get_entry_as(cfg, "hpx.parcel.gasnet.priority", 1) > - get_entry_as(cfg, "hpx.parcel.gasnet.priority", 0)))) + get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) { LBT_(info) << "GASNET support disabled via configuration settings\n"; return false; From 4239bd343b9b3279b71665b5c86aa698090651dc Mon Sep 17 00:00:00 2001 From: ctaylor Date: Wed, 12 Jul 2023 20:00:12 -0500 Subject: [PATCH 041/333] improved gasnet-conduit selection support --- CMakeLists.txt | 56 +++++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed25ba277c2d..8c7718f1e981 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1153,51 +1153,27 @@ if(HPX_WITH_NETWORKING) endif() hpx_option( - HPX_WITH_PARCELPORT_GASNET_MPI BOOL "Enable the GASNET-MPI based parcelport." OFF - CATEGORY "Parcelport" + HPX_WITH_PARCELPORT_GASNET_CONDUIT + STRING + "Define which conduit to use for the gasnet parcelport" + "udp" + STRINGS "smp;udp;mpi;ofi;ucx;ibv" ) - if(HPX_WITH_PARCELPORT_GASNET_MPI) - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_MPI) - endif() - hpx_option( - HPX_WITH_PARCELPORT_GASNET_SMP BOOL "Enable the GASNET-SMP based parcelport." OFF - CATEGORY "Parcelport" - ) - if(HPX_WITH_PARCELPORT_GASNET_SMP) + if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_MPI) + elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "smp") hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_SMP) - endif() - - hpx_option( - HPX_WITH_PARCELPORT_GASNET_UDP BOOL "Enable the GASNET-UDP based parcelport." OFF - CATEGORY "Parcelport" - ) - if(HPX_WITH_PARCELPORT_GASNET_UDP) - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) - endif() - - hpx_option( - HPX_WITH_PARCELPORT_GASNET_OFI BOOL "Enable the GASNET-OFI based parcelport." OFF - CATEGORY "Parcelport" - ) - if(HPX_WITH_PARCELPORT_GASNET_OFI) + elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ibv") + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_IBV) + elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ofi") hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_OFI) - endif() - - hpx_option( - HPX_WITH_PARCELPORT_GASNET_UCX BOOL "Enable the GASNET-UCX based parcelport." OFF - CATEGORY "Parcelport" - ) - if(HPX_WITH_PARCELPORT_GASNET_UCX) + elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ucx") hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UCX) - endif() - - hpx_option( - HPX_WITH_PARCELPORT_GASNET_IBV BOOL "Enable the GASNET-IBV based parcelport." OFF - CATEGORY "Parcelport" - ) - if(HPX_WITH_PARCELPORT_GASNET_IBV) - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_IBV) + elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "udp") + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) + else() + hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) endif() hpx_option( From eecdfba7faff4d9f09b0a9d44b8c0f9220a55abe Mon Sep 17 00:00:00 2001 From: ctaylor Date: Wed, 12 Jul 2023 20:31:14 -0500 Subject: [PATCH 042/333] improved gasnet-conduit selection support --- CMakeLists.txt | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c7718f1e981..acd22b6c7c13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1161,19 +1161,26 @@ if(HPX_WITH_NETWORKING) ) if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_MPI) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_MPI) + set(HPX_WITH_PARCELPORT_GASNET_MPI ON) elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "smp") - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_SMP) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_SMP) + set(HPX_WITH_PARCELPORT_GASNET_SMP ON) elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ibv") - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_IBV) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_IBV) + set(HPX_WITH_PARCELPORT_GASNET_IBV ON) elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ofi") - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_OFI) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_OFI) + set(HPX_WITH_PARCELPORT_GASNET_OFI ON) elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ucx") - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UCX) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UCX) + set(HPX_WITH_PARCELPORT_GASNET_UCX ON) elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "udp") - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UDP) + set(HPX_WITH_PARCELPORT_GASNET_UDP ON) else() - hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET_UDP) + hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UDP) + set(HPX_WITH_PARCELPORT_GASNET_UDP ON) endif() hpx_option( From fb875fc2c84ac5809eb8502f08904d08a84dd3c3 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Wed, 12 Jul 2023 21:00:23 -0500 Subject: [PATCH 043/333] simplified cmake files --- cmake/FindGasnet.cmake | 40 ++------------ libs/core/gasnet_base/CMakeLists.txt | 4 +- libs/full/parcelport_gasnet/CMakeLists.txt | 63 ++-------------------- 3 files changed, 8 insertions(+), 99 deletions(-) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 9da11fde43fa..2dbcb190eb77 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -7,50 +7,16 @@ # macro(find_gasnet) -set(GASNET_MPI_FOUND FALSE) -set(GASNET_UDP_FOUND FALSE) -set(GASNET_SMP_FOUND FALSE) -set(GASNET_OFI_FOUND FALSE) -set(GASNET_UCX_FOUND FALSE) -set(GASNET_IBV_FOUND FALSE) - find_package(PkgConfig REQUIRED QUIET COMPONENTS) -if(HPX_WITH_PARCELPORT_GASNET_MPI) +if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-mpi-par) set(GASNET_MPI_FOUND TRUE) hpx_setup_mpi() -endif() - -if(HPX_WITH_PARCELPORT_GASNET_UDP) - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-udp-par) - set(GASNET_UDP_FOUND TRUE) -endif() - -if(HPX_WITH_PARCELPORT_GASNET_SMP) - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-smp-par) - set(GASNET_SMP_FOUND TRUE) -endif() - -if(HPX_WITH_PARCELPORT_GASNET_OFI) - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-ofi-par) - set(GASNET_OFI_FOUND TRUE) -endif() - -if(HPX_WITH_PARCELPORT_GASNET_UCX) - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-ucx-par) - set(GASNET_UCX_FOUND TRUE) -endif() - -if(HPX_WITH_PARCELPORT_GASNET_IBV) +else() message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-ibv-par) - set(GASNET_IBV_FOUND TRUE) + pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par) endif() target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 0280a579339b..2f4c4486f50a 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_GASNET_UDP OR HPX_WITH_PARCELPORT_GASNET_OFI OR HPX_WITH_PARCELPORT_GASNET_UCX OR HPX_WITH_PARCELPORT_GASNET_IBV))) +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT)) message(STATUS "*** PARCELPORT_GASNET not set") return() endif() @@ -30,7 +30,7 @@ set(gasnet_base_compat_headers set(gasnet_base_sources gasnet_environment.cpp) -if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) +if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") include(HPX_AddModule) add_hpx_module( core gasnet_base diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index d5e2ebf0e1cc..1cf8df58efab 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND (HPX_WITH_PARCELPORT_GASNET_MPI OR HPX_WITH_PARCELPORT_GASNET_SMP OR HPX_WITH_PARCELPORT_UDP OR HPX_WITH_PARCELPORT_OFI OR HPX_WITH_PARCELPORT_UCX OR HPX_WITH_PARCELPORT_IBV))) +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT)) return() endif() @@ -26,7 +26,7 @@ set(parcelport_gasnet_compat_headers) set(parcelport_gasnet_sources locality.cpp parcelport_gasnet.cpp) -if(GASNET_MPI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) +if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") link_directories(${GASNET_LIBRARY_DIRS}) include(HPX_AddModule) add_hpx_module( @@ -39,64 +39,7 @@ link_directories(${GASNET_LIBRARY_DIRS}) MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset CMAKE_SUBDIRS examples tests ) - -elseif(GASNET_UDP_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UDP) -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) - -elseif(GASNET_OFI_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OFI) -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) - -elseif(GASNET_UCX_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_UCX) -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) - -elseif(GASNET_IBV_FOUND AND HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_IBV) -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) - -elseif(GASNET_SMP_FOUND) +else() link_directories(${GASNET_LIBRARY_DIRS}) include(HPX_AddModule) add_hpx_module( From 3d36a26d737265984b1df73fff8113f73acb3158 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 13 Jul 2023 13:51:23 -0500 Subject: [PATCH 044/333] Cleaning up rebase problems --- cmake/HPX_AddModule.cmake | 1 - docs/sphinx/releases/whats_new_1_9_0.rst | 8 +-- libs/core/algorithms/CMakeLists.txt | 1 - .../hpx/algorithms/traits/is_relocatable.hpp | 22 ------ .../algorithms/traits/pointer_category.hpp | 3 - .../tests/unit/algorithms/util/CMakeLists.txt | 4 +- .../algorithms/util/test_is_relocatable.cpp | 70 ------------------- 7 files changed, 6 insertions(+), 103 deletions(-) delete mode 100644 libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp delete mode 100644 libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp diff --git a/cmake/HPX_AddModule.cmake b/cmake/HPX_AddModule.cmake index 52df66677975..3efec41ec1f1 100644 --- a/cmake/HPX_AddModule.cmake +++ b/cmake/HPX_AddModule.cmake @@ -23,7 +23,6 @@ function(add_hpx_module libname modulename) EXCLUDE_FROM_GLOBAL_HEADER ADD_TO_GLOBAL_HEADER ) - cmake_parse_arguments( ${modulename} "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN} diff --git a/docs/sphinx/releases/whats_new_1_9_0.rst b/docs/sphinx/releases/whats_new_1_9_0.rst index 805c3ec2a28f..d97d7b70ce33 100644 --- a/docs/sphinx/releases/whats_new_1_9_0.rst +++ b/docs/sphinx/releases/whats_new_1_9_0.rst @@ -169,7 +169,7 @@ Closed pull requests * :hpx-pr:`6117` - Initialize buffer_allocate_time_ to 0 * :hpx-pr:`6116` - Add new command line argument --hpx:loopback_network * :hpx-pr:`6115` - Modernize modules of levels 14, 15, and 16 -* :hpx-pr:`6114` - Enhance the formatting of the documentation +* :hpx-pr:`6114` - Enhance the formatting of the documentation * :hpx-pr:`6113` - Modernize modules in module level 11, 12, and 13 * :hpx-pr:`6112` - Modernize modules from levels 9 and 10 * :hpx-pr:`6111` - Modernize all modules from module level 8 @@ -247,7 +247,7 @@ Closed pull requests * :hpx-pr:`6030` - Adding parcelport initialization hook for resource partitioner operation * :hpx-pr:`6029` - Simplify startup code * :hpx-pr:`6027` - Add/Fix documentation in Public API page -* :hpx-pr:`6026` - add option hpx:force_ipv4 to force resolving hostnames to ipv4 addresses +* :hpx-pr:`6026` - add option hpx:force_ipv4 to force resolving hostnames to ipv4 adresses * :hpx-pr:`6025` - build(docs): remove leftover sections * :hpx-pr:`6023` - Minor fixes on "How to build on Windows" * :hpx-pr:`6022` - build(doxy): don't extract private members @@ -273,11 +273,11 @@ Closed pull requests * :hpx-pr:`5990` - P2300 enhancements * :hpx-pr:`5989` - Fix missing documentation in Public API page * :hpx-pr:`5987` - Attempting to fix timed executor API -* :hpx-pr:`5986` - Fix warnings when building docs +* :hpx-pr:`5986` - Fix warnings when building docs * :hpx-pr:`5985` - Re-add deprecated tag_policy_tag et.al. types that were removed in V1.8.1 * :hpx-pr:`5981` - docs: add docs for condition_variable.hpp * :hpx-pr:`5980` - More work on execution::read -* :hpx-pr:`5979` - Unsupported clang-v8 and clang-v9, switch LSU clang-v13 to C++17 +* :hpx-pr:`5979` - Unsupport clang-v8 and clang-v9, switch LSU clang-v13 to C++17 * :hpx-pr:`5977` - fix: Compilation errors for -std=c++17 builders * :hpx-pr:`5975` - docs: fix & improve parallel algorithms documentation 5 * :hpx-pr:`5974` - [P2300] Adapt get completion signatures for awaitable senders diff --git a/libs/core/algorithms/CMakeLists.txt b/libs/core/algorithms/CMakeLists.txt index 8f37b9277a31..c2fac77c66cf 100644 --- a/libs/core/algorithms/CMakeLists.txt +++ b/libs/core/algorithms/CMakeLists.txt @@ -6,7 +6,6 @@ set(algorithms_headers hpx/algorithms/traits/is_pair.hpp - hpx/algorithms/traits/is_relocatable.hpp hpx/algorithms/traits/is_value_proxy.hpp hpx/algorithms/traits/pointer_category.hpp hpx/algorithms/traits/projected.hpp diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp deleted file mode 100644 index 5d983c3ec168..000000000000 --- a/libs/core/algorithms/include/hpx/algorithms/traits/is_relocatable.hpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2023 Isidoros Tsaousis-Seiras -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#pragma once - -#include - -namespace hpx { - - template - struct is_relocatable - { - static constexpr bool value = - std::is_move_constructible_v && std::is_destructible_v; - }; - - template - inline constexpr bool is_relocatable_v = is_relocatable::value; -} // namespace hpx diff --git a/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp b/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp index fcb782fdb3fb..0cc691dad638 100644 --- a/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp +++ b/libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp @@ -105,9 +105,6 @@ namespace hpx::traits { template > - // Why do we need to check if the iterators are contiguous in the trivially - // copyable case? Is this category refering to buffers of - // the underlying types or the individual objects? struct pointer_move_category { using type = std::conditional_t< diff --git a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt index 25e81f7364a1..95c4813c0c3d 100644 --- a/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt +++ b/libs/core/algorithms/tests/unit/algorithms/util/CMakeLists.txt @@ -4,8 +4,8 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -set(tests test_is_relocatable test_low_level test_merge_four test_merge_vector - test_nbits test_range +set(tests test_low_level test_merge_four test_merge_vector test_nbits + test_range ) foreach(test ${tests}) diff --git a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp b/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp deleted file mode 100644 index af73f07695a5..000000000000 --- a/libs/core/algorithms/tests/unit/algorithms/util/test_is_relocatable.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2023 Isidoros Tsaousis-Seiras -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#include -#include - -#include - -#include - -// Integral types are relocatable -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); - -// Array types are not move-constructible and thus not relocatable -static_assert(!hpx::is_relocatable_v); -static_assert(!hpx::is_relocatable_v); -static_assert(!hpx::is_relocatable_v); -static_assert(!hpx::is_relocatable_v); - -// Function types are not move-constructible and thus not relocatable -static_assert(!hpx::is_relocatable_v); - -// Void types are not move-constructible and thus not relocatable -static_assert(!hpx::is_relocatable_v); -static_assert(!hpx::is_relocatable_v); - -// std::mutex is not relocatable -static_assert(!hpx::is_relocatable_v); - -struct NotDestructible -{ - NotDestructible(const NotDestructible&); - NotDestructible(NotDestructible&&); - ~NotDestructible() = delete; -}; - -static_assert(!hpx::is_relocatable_v); - -struct NotMoveConstructible -{ - NotMoveConstructible(const NotMoveConstructible&); - NotMoveConstructible(NotMoveConstructible&&) = delete; -}; - -static_assert(!hpx::is_relocatable_v); - -struct NotCopyConstructible -{ - NotCopyConstructible(const NotCopyConstructible&) = delete; - NotCopyConstructible(NotCopyConstructible&&); -}; - -static_assert(hpx::is_relocatable_v); - -// reference types are relocatable -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); - -int main(int, char*[]) {} From 32868695e93d47173e982752a8e738be31eb4ba3 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 13 Jul 2023 15:28:01 -0500 Subject: [PATCH 045/333] Attempt to simplify build system support for the gasnet pp --- .cmake-format.py | 3 +- CMakeLists.txt | 53 +- cmake/FindGasnet.cmake | 497 ---------------- cmake/HPX_AddTest.cmake | 105 +--- cmake/HPX_SetupGasnet.cmake | 550 ++++++++++++++++++ cmake/templates/HPXConfig.cmake.in | 12 +- cmake/templates/hpxrun.py.in | 16 +- libs/core/gasnet_base/CMakeLists.txt | 65 +-- .../include/hpx/gasnet_base/gasnet.hpp | 2 +- .../hpx/gasnet_base/gasnet_environment.hpp | 10 +- .../gasnet_base/src/gasnet_environment.cpp | 410 +++++++------ libs/core/gasnet_base/tests/CMakeLists.txt | 4 +- .../full/command_line_handling/CMakeLists.txt | 21 +- libs/full/parcelport_gasnet/CMakeLists.txt | 59 +- .../parcelport_gasnet/examples/CMakeLists.txt | 4 +- .../include/hpx/parcelport_gasnet/header.hpp | 2 +- .../hpx/parcelport_gasnet/locality.hpp | 4 +- .../hpx/parcelport_gasnet/receiver.hpp | 28 +- .../parcelport_gasnet/receiver_connection.hpp | 13 +- .../include/hpx/parcelport_gasnet/sender.hpp | 6 +- .../parcelport_gasnet/sender_connection.hpp | 88 +-- libs/full/parcelport_gasnet/src/locality.cpp | 2 +- .../src/parcelport_gasnet.cpp | 20 +- 23 files changed, 962 insertions(+), 1012 deletions(-) delete mode 100644 cmake/FindGasnet.cmake create mode 100644 cmake/HPX_SetupGasnet.cmake diff --git a/.cmake-format.py b/.cmake-format.py index a84ef9e1025c..c58cb99e0dd9 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -393,7 +393,8 @@ 'RUN_SERIAL', 'NO_PARCELPORT_TCP', 'NO_PARCELPORT_LCI', - 'NO_PARCELPORT_MPI'], + 'NO_PARCELPORT_MPI', + 'NO_PARCELPORT_GASNET'], 'nargs': '2+'}}, 'add_hpx_source_group': { 'kwargs': { 'CLASS': 1, 'NAME': 1, diff --git a/CMakeLists.txt b/CMakeLists.txt index acd22b6c7c13..1edb5aed9b77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,7 +588,6 @@ if(APPLE OR HPX_WITH_CUDA OR HPX_WITH_HIP OR HPX_WITH_PARCELPORT_LCI - OR HPX_WITH_PARCELPORT_GASNET ) set(HPX_WITH_PKGCONFIG_DEFAULT OFF) endif() @@ -1150,37 +1149,11 @@ if(HPX_WITH_NETWORKING) ) if(HPX_WITH_PARCELPORT_GASNET) hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) - endif() - - hpx_option( - HPX_WITH_PARCELPORT_GASNET_CONDUIT - STRING - "Define which conduit to use for the gasnet parcelport" - "udp" - STRINGS "smp;udp;mpi;ofi;ucx;ibv" - ) - - if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_MPI) - set(HPX_WITH_PARCELPORT_GASNET_MPI ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "smp") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_SMP) - set(HPX_WITH_PARCELPORT_GASNET_SMP ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ibv") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_IBV) - set(HPX_WITH_PARCELPORT_GASNET_IBV ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ofi") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_OFI) - set(HPX_WITH_PARCELPORT_GASNET_OFI ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "ucx") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UCX) - set(HPX_WITH_PARCELPORT_GASNET_UCX ON) - elseif(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "udp") - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UDP) - set(HPX_WITH_PARCELPORT_GASNET_UDP ON) - else() - hpx_add_config_define(HPX_WITH_PARCELPORT_GASNET_UDP) - set(HPX_WITH_PARCELPORT_GASNET_UDP ON) + hpx_option( + HPX_WITH_PARCELPORT_GASNET_CONDUIT STRING + "Define which conduit to use for the gasnet parcelport" "udp" + STRINGS "smp;udp;mpi;ofi;ucx;ibv" + ) endif() hpx_option( @@ -1220,7 +1193,10 @@ else(HPX_WITH_NETWORKING) ) endif(HPX_WITH_NETWORKING) -if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI OR HPX_WITH_PARCELPORT_GASNET_MPI) +if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) + OR HPX_WITH_ASYNC_MPI + OR ("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") +) # mpi parcelport settings hpx_option( HPX_WITH_PARCELPORT_MPI_ENV @@ -2165,13 +2141,20 @@ add_hpx_library_headers_noglob(hpx_external) # Setup plugins (set here cause if we include it inside plugins, it will not be # defined in src/CMakeLists.txt where we call add_static_parcelports) include(HPX_SetupMPI) # must come before APEX -if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) OR HPX_WITH_ASYNC_MPI OR HPX_WITH_PARCELPORT_GASNET_MPI) +if((HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_MPI) + OR HPX_WITH_ASYNC_MPI + OR ("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") +) hpx_setup_mpi() endif() -include(HPX_SetupLCI) if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) + include(HPX_SetupLCI) hpx_setup_lci() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGasnet) + hpx_setup_gasnet() +endif() # Setup packages and subprojects include(HPX_SetupSYCL) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake deleted file mode 100644 index 2dbcb190eb77..000000000000 --- a/cmake/FindGasnet.cmake +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright (c) 2017 Thomas Heller -# Copyright (c) 2023 Christopher Taylor -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# -macro(find_gasnet) - -find_package(PkgConfig REQUIRED QUIET COMPONENTS) - -if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-mpi-par) - set(GASNET_MPI_FOUND TRUE) - hpx_setup_mpi() -else() - message(STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`") - pkg_search_module(GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par) -endif() - -target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) - -if(GASNET_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS "${X}") - endforeach() -endif() - -if(GASNET_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS_OTHER "${X}") - endforeach() -endif() - -if(GASNET_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() -endif() - -if(GASNET_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") - endif() - endif() - -endif() - -if(GASNET_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS "${X}") - endforeach() -endif() - -if(GASNET_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") - endforeach() -endif() - -if(GASNET_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") - endif() - endif() -endif() - -if(GASNET_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" AND "${IDX}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "0" OR "${IS_PARAM}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library(FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} - NO_CACHE - NO_CMAKE_FIND_ROOT_PATH - NO_DEFAULT_PATH) - - list(LENGTH FOUND_LIB IDX) - - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" "${NEWLINK}" IDX) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - endif() -endif() - -set_target_properties(PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}") -set_target_properties(PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}") - -endmacro() diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index ce884f5cc0fe..6dfb45989c2b 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -6,8 +6,7 @@ function(add_hpx_test category name) set(options FAILURE_EXPECTED RUN_SERIAL NO_PARCELPORT_TCP NO_PARCELPORT_MPI - NO_PARCELPORT_LCI NO_PARCELPORT_GASNET_SMP NO_PARCELPORT_GASNET_UDP - NO_PARCELPORT_GASNET_UCX NO_PARCELPORT_GASNET_OFI NO_PARCELPORT_GASNET_IBV + NO_PARCELPORT_LCI NO_PARCELPORT_GASNET ) set(one_value_args EXECUTABLE LOCALITIES THREADS_PER_LOCALITY TIMEOUT RUNWRAPPER @@ -173,7 +172,7 @@ function(add_hpx_test category name) endif() endif() endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_SMP AND NOT ${${name}_NO_PARCELPORT_GASNET_SMP}) + if(HPX_WITH_PARCELPORT_GASNET AND NOT ${name}_NO_PARCELPORT_GASNET) set(_add_test FALSE) if(DEFINED ${name}_PARCELPORTS) set(PP_FOUND -1) @@ -186,104 +185,8 @@ function(add_hpx_test category name) endif() if(_add_test) set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_UDP AND NOT ${${name}_NO_PARCELPORT_GASNET_UDP}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_OFI AND NOT ${${name}_NO_PARCELPORT_GASNET_OFI}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_UCX AND NOT ${${name}_NO_PARCELPORT_GASNET_UCX}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} - ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) - if(${name}_TIMEOUT) - set_tests_properties( - "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} - ) - endif() - endif() - endif() - if(HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_IBV AND NOT ${${name}_NO_PARCELPORT_GASNET_IBV}) - set(_add_test FALSE) - if(DEFINED ${name}_PARCELPORTS) - set(PP_FOUND -1) - list(FIND ${name}_PARCELPORTS "gasnet" PP_FOUND) - if(NOT PP_FOUND EQUAL -1) - set(_add_test TRUE) - endif() - else() - set(_add_test TRUE) - endif() - if(_add_test) - set(_full_name "${category}.distributed.gasnet.${name}") - add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" "mpi" - ${args} + add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" + "gasnet" ${args} ) set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) if(${name}_TIMEOUT) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake new file mode 100644 index 000000000000..01bf57ad22d9 --- /dev/null +++ b/cmake/HPX_SetupGasnet.cmake @@ -0,0 +1,550 @@ +# Copyright (c) 2017 Thomas Heller +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# + +macro(hpx_setup_gasnet) + + if(NOT TARGET PkgConfig::GASNET) + + find_package(PkgConfig REQUIRED QUIET COMPONENTS) + + hpx_info( + STATUS + "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" + ) + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(GASNET_MPI_FOUND TRUE) + include(HPX_SetupMPI) + hpx_setup_mpi() + endif() + + target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) + + if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + + endif() + + if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + endif() + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES "${GASNET_LIBRARY_DIRS}" + ) + + endif() + +endmacro() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 7a68ee418a21..28cc6b872304 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,15 +52,9 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() -if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND - ( HPX_WITH_PARCELPORT_GASNET_SMP OR - HPX_WITH_PARCELPORT_GASNET_UDP OR - HPX_WITH_PARCELPORT_GASNET_OFI OR - HPX_WITH_PARCELPORT_GASNET_UCX OR - HPX_WITH_PARCELPORT_GASNET_IBV ) ) - set(GASNET_ROOT "@GASNET_ROOT@") - include(FindGasnet) - find_gasnet() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGasnet) + hpx_setup_gasnet() endif() # Eve can be installed by HPX or externally installed. In the first case we use diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 4955cdf75537..76eed0195555 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -196,17 +196,9 @@ def run_mpi(cmd, localities, verbose): subproc(exec_cmd) # Run with amudprun -# This is executing mpiexec with the "-np" option set to the number of localities +# This is executing amudprun with the "-np" option set to the number of localities def run_gasnet(cmd, localities, verbose): - mpiexec = 'amudprun' - if mpiexec == '': - mpiexec = 'amudprun' - if mpiexec == '': - msg = 'amudprun not available on this platform. ' - msg += 'Please rerun CMake with HPX_PARCELPORT_GASNET=True.' - print(msg, sys.stderr) - sys.exit(1) - exec_cmd = [mpiexec, '-np', str(localities)] + cmd + exec_cmd = ['amudprun', '-np', str(localities)] + cmd if verbose: print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) @@ -381,13 +373,13 @@ Used by the tcp parcelport only. parser.add_option('-p', '--parcelport' , action='store', type='string' , dest='parcelport', default=default_env('HPXRUN_PARCELPORT', 'tcp') - , help='Which parcelport to use (Options are: mpi, lci, gaset, tcp) ' + , help='Which parcelport to use (Options are: mpi, lci, gasnet, tcp) ' '(environment variable HPXRUN_PARCELPORT') parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') - , help='Which runwrapper to use (Options are: none, mpi, srun, jsrun) ' + , help='Which runwrapper to use (Options are: none, mpi, gasnet, srun, jsrun) ' '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 2f4c4486f50a..0a5736df497b 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -1,57 +1,38 @@ -# Copyright (c) 2019-2023 The STE||AR-Group +# Copyright (c) 2023 Chistopher Taylor # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT)) - message(STATUS "*** PARCELPORT_GASNET not set") +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_GASNET + AND HPX_WITH_PARCELPORT_GASNET_CONDUIT) +) return() endif() -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") - -include(FindGasnet) -find_gasnet() +include(HPX_SetupGasnet) +hpx_setup_gasnet() # Default location is $HPX_ROOT/libs/gasnet_base/include -set(gasnet_base_headers - hpx/gasnet_base/gasnet.hpp - hpx/gasnet_base/gasnet_environment.hpp +set(gasnet_base_headers hpx/gasnet_base/gasnet.hpp + hpx/gasnet_base/gasnet_environment.hpp ) -# Default location is $HPX_ROOT/libs/gasnet_base/include -# cmake-format: off -set(gasnet_base_compat_headers - hpx/plugins/parcelport/gasnet/gasnet.hpp => hpx/modules/gasnet_base.hpp - hpx/plugins/parcelport/gasnet/gasnet_environment.hpp => hpx/modules/gasnet_base.hpp -) -# cmake-format: on - set(gasnet_base_sources gasnet_environment.cpp) -if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") - include(HPX_AddModule) - add_hpx_module( - core gasnet_base - GLOBAL_HEADER_GEN ON - SOURCES ${gasnet_base_sources} - HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} - DEPENDENCIES PkgConfig::GASNET Mpi::mpi - MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - CMAKE_SUBDIRS examples tests - ) -else() - include(HPX_AddModule) - add_hpx_module( - core gasnet_base - GLOBAL_HEADER_GEN ON - SOURCES ${gasnet_base_sources} - HEADERS ${gasnet_base_headers} - COMPAT_HEADERS ${gasnet_base_compat_headers} - DEPENDENCIES PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util - CMAKE_SUBDIRS examples tests - ) +if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(gasnet_additional_dependencies Mpi::mpi) endif() + +include(HPX_AddModule) +add_hpx_module( + core gasnet_base + GLOBAL_HEADER_GEN ON + SOURCES ${gasnet_base_sources} + HEADERS ${gasnet_base_headers} + DEPENDENCIES PkgConfig::GASNET ${gasnet_additional_dependencies} + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util + CMAKE_SUBDIRS examples tests +) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp index e76245428664..b13044c7ad59 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -6,7 +6,7 @@ #pragma once -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #if defined(__clang__) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp index 98c7310ab124..fe66baaeca11 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -9,13 +9,13 @@ #include -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #include #include -#include #include +#include #include #include @@ -44,9 +44,11 @@ namespace hpx { namespace util { static bool gettable(const int node, void* start, const size_t len); - static void put(std::uint8_t * addr, const int rank, std::uint8_t * raddr, const std::size_t size); + static void put(std::uint8_t* addr, const int rank, std::uint8_t* raddr, + const std::size_t size); - static void get(std::uint8_t * addr, const int rank, std::uint8_t * raddr, const std::size_t size); + static void get(std::uint8_t* addr, const int rank, std::uint8_t* raddr, + const std::size_t size); struct HPX_CORE_EXPORT scoped_lock { diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 44436a8934c6..1f8accd5837a 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -9,8 +9,8 @@ #include -#include #include +#include #include #include #include @@ -19,76 +19,77 @@ #include #include -#include #include +#include // // AM functions // -typedef enum { - SIGNAL = 136, // ack to a done_t via gasnet_AMReplyShortM() - SIGNAL_LONG, // ack to a done_t via gasnet_AMReplyLongM() - DO_REPLY_PUT = 143, // do a PUT here from another locale - DO_COPY_PAYLOAD // copy AM payload to another address +typedef enum +{ + SIGNAL = 136, // ack to a done_t via gasnet_AMReplyShortM() + SIGNAL_LONG, // ack to a done_t via gasnet_AMReplyLongM() + DO_REPLY_PUT = 143, // do a PUT here from another locale + DO_COPY_PAYLOAD // copy AM payload to another address } AM_handler_function_idx_t; -typedef struct { - void* ack; // acknowledgement object - void* tgt; // target memory address - void* src; // source memory address - size_t size; // number of bytes. +typedef struct +{ + void* ack; // acknowledgement object + void* tgt; // target memory address + void* src; // source memory address + size_t size; // number of bytes. } xfer_info_t; // Gasnet AM handler arguments are only 32 bits, so here we have // functions to get the 2 arguments for a 64-bit pointer, // and a function to reconstitute the pointer from the 2 arguments. // -static inline -gasnet_handlerarg_t get_arg_from_ptr0(uintptr_t addr) +static inline gasnet_handlerarg_t get_arg_from_ptr0(uintptr_t addr) { - // This one returns the bottom 32 bits. - return ((gasnet_handlerarg_t) - ((((uint64_t) (addr)) << 32UL) >> 32UL)); + // This one returns the bottom 32 bits. + return ((gasnet_handlerarg_t)((((uint64_t)(addr)) << 32UL) >> 32UL)); } -static inline -gasnet_handlerarg_t get_arg_from_ptr1(uintptr_t addr) +static inline gasnet_handlerarg_t get_arg_from_ptr1(uintptr_t addr) { - // this one returns the top 32 bits. - return ((gasnet_handlerarg_t) - (((uint64_t) (addr)) >> 32UL)); + // this one returns the top 32 bits. + return ((gasnet_handlerarg_t)(((uint64_t)(addr)) >> 32UL)); } -static inline -uintptr_t get_uintptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) +static inline uintptr_t get_uintptr_from_args( + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - return (uintptr_t) - (((uint64_t) (uint32_t) a0) - | (((uint64_t) (uint32_t) a1) << 32UL)); + return (uintptr_t)( + ((uint64_t)(uint32_t) a0) | (((uint64_t)(uint32_t) a1) << 32UL)); } -static inline -void* get_ptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) +static inline void* get_ptr_from_args( + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - return (void*) get_uintptr_from_args(a0, a1); + return (void*) get_uintptr_from_args(a0, a1); } // Build acknowledgement address arguments for gasnetAMRequest*() calls. // -#define Arg0(addr) get_arg_from_ptr0((uintptr_t)addr) -#define Arg1(addr) get_arg_from_ptr1((uintptr_t)addr) +#define Arg0(addr) get_arg_from_ptr0((uintptr_t) addr) +#define Arg1(addr) get_arg_from_ptr1((uintptr_t) addr) // The following macro is from the GASNet test.h distribution // -#define GASNET_Safe(fncall) do { \ - int _retval; \ - if ((_retval = fncall) != GASNET_OK) { \ - fprintf(stderr, "ERROR calling: %s\n" \ - " at: %s:%i\n" \ - " error: %s (%s)\n", \ - #fncall, __FILE__, __LINE__, \ - gasnet_ErrorName(_retval), gasnet_ErrorDesc(_retval)); \ - fflush(stderr); \ - gasnet_exit(_retval); \ - } \ - } while(0) +#define GASNET_Safe(fncall) \ + do \ + { \ + int _retval; \ + if ((_retval = fncall) != GASNET_OK) \ + { \ + fprintf(stderr, \ + "ERROR calling: %s\n" \ + " at: %s:%i\n" \ + " error: %s (%s)\n", \ + #fncall, __FILE__, __LINE__, gasnet_ErrorName(_retval), \ + gasnet_ErrorDesc(_retval)); \ + fflush(stderr); \ + gasnet_exit(_retval); \ + } \ + } while (0) // This is the type of object we use to manage GASNet acknowledgements. // @@ -100,101 +101,105 @@ void* get_ptr_from_args(gasnet_handlerarg_t a0, gasnet_handlerarg_t a1 ) // the condition for a BLOCKUNTIL can only be changed by the execution // of an AM handler.) // -typedef struct { - std::atomic count; - std::uint32_t target; - volatile int flag; +typedef struct +{ + std::atomic count; + std::uint32_t target; + volatile int flag; } done_t; -static void AM_signal(gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - done_t* done = (done_t*) get_ptr_from_args(a0, a1); - uint_least32_t prev; - prev = done->count.fetch_add(1, std::memory_order_seq_cst); - if (prev + 1 == done->target) - done->flag = 1; +static void AM_signal( + gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) +{ + done_t* done = (done_t*) get_ptr_from_args(a0, a1); + uint_least32_t prev; + prev = done->count.fetch_add(1, std::memory_order_seq_cst); + if (prev + 1 == done->target) + done->flag = 1; } -static void AM_signal_long(gasnet_token_t token, void *buf, size_t nbytes, - gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - done_t* done = (done_t*) get_ptr_from_args(a0, a1); - uint_least32_t prev; - prev = done->count.fetch_add(1, std::memory_order_seq_cst); - if (prev + 1 == done->target) - done->flag = 1; +static void AM_signal_long(gasnet_token_t token, void* buf, size_t nbytes, + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) +{ + done_t* done = (done_t*) get_ptr_from_args(a0, a1); + uint_least32_t prev; + prev = done->count.fetch_add(1, std::memory_order_seq_cst); + if (prev + 1 == done->target) + done->flag = 1; } // Put from arg->src (which is local to the AM handler) back to // arg->dst (which is local to the caller of this AM). // nbytes is < gasnet_AMMaxLongReply here (see chpl_comm_get). // -static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) { - xfer_info_t* x = static_cast(buf); +static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) +{ + xfer_info_t* x = static_cast(buf); - assert(nbytes == sizeof(xfer_info_t)); + assert(nbytes == sizeof(xfer_info_t)); - GASNET_Safe(gasnet_AMReplyLong2(token, SIGNAL_LONG, - x->src, x->size, x->tgt, - Arg0(x->ack), Arg1(x->ack))); + GASNET_Safe(gasnet_AMReplyLong2(token, SIGNAL_LONG, x->src, x->size, x->tgt, + Arg0(x->ack), Arg1(x->ack))); } // Copy from the payload in this active message to dst. // -static -void AM_copy_payload(gasnet_token_t token, void* buf, size_t nbytes, - gasnet_handlerarg_t ack0, gasnet_handlerarg_t ack1, - gasnet_handlerarg_t dst0, gasnet_handlerarg_t dst1) +static void AM_copy_payload(gasnet_token_t token, void* buf, size_t nbytes, + gasnet_handlerarg_t ack0, gasnet_handlerarg_t ack1, + gasnet_handlerarg_t dst0, gasnet_handlerarg_t dst1) { - void* dst = get_ptr_from_args(dst0, dst1); - { - // would prefer to protect the memory segments - // associated with each node (n-node mutex) - // will require future work - // - std::lock_guard lk(hpx::util::gasnet_environment::dshm_mut); - std::memcpy(dst, buf, nbytes); - } - - GASNET_Safe(gasnet_AMReplyShort2(token, SIGNAL, ack0, ack1)); + void* dst = get_ptr_from_args(dst0, dst1); + { + // would prefer to protect the memory segments + // associated with each node (n-node mutex) + // will require future work + // + std::lock_guard lk(hpx::util::gasnet_environment::dshm_mut); + std::memcpy(dst, buf, nbytes); + } + + GASNET_Safe(gasnet_AMReplyShort2(token, SIGNAL, ack0, ack1)); } -static gasnet_handlerentry_t ftable[] = { - {SIGNAL, (void (*)())&AM_signal}, - {SIGNAL_LONG, (void (*)())&AM_signal_long}, - {DO_REPLY_PUT, (void (*)())&AM_reply_put}, - {DO_COPY_PAYLOAD, (void (*)())&AM_copy_payload} -}; +static gasnet_handlerentry_t ftable[] = {{SIGNAL, (void (*)()) & AM_signal}, + {SIGNAL_LONG, (void (*)()) & AM_signal_long}, + {DO_REPLY_PUT, (void (*)()) & AM_reply_put}, + {DO_COPY_PAYLOAD, (void (*)()) & AM_copy_payload}}; // // Initialize one of the above. // -static inline -void init_done_obj(done_t* done, int target) { - done->count.store(0, std::memory_order_seq_cst); - done->target = target; - done->flag = 0; +static inline void init_done_obj(done_t* done, int target) +{ + done->count.store(0, std::memory_order_seq_cst); + done->target = target; + done->flag = 0; } -static inline void am_poll_try() { - // Serialize polling for IBV, UCX, Aries, and OFI. Concurrent polling causes - // contention in these configurations. For other configurations that are - // AM-based (udp/amudp, mpi/ammpi) serializing can hurt performance. - // -#if defined(GASNET_CONDUIT_IBV) || defined(GASNET_CONDUIT_UCX) || defined(GASNET_CONDUIT_ARIES) || defined(GASNET_CONDUIT_OFI) - std::lock_guard lk(hpx::util::gasnet_environment::pollingLock); +static inline void am_poll_try() +{ + // Serialize polling for IBV, UCX, Aries, and OFI. Concurrent polling causes + // contention in these configurations. For other configurations that are + // AM-based (udp/amudp, mpi/ammpi) serializing can hurt performance. + // +#if defined(GASNET_CONDUIT_IBV) || defined(GASNET_CONDUIT_UCX) || \ + defined(GASNET_CONDUIT_ARIES) || defined(GASNET_CONDUIT_OFI) + std::lock_guard lk( + hpx::util::gasnet_environment::pollingLock); (void) gasnet_AMPoll(); #else (void) gasnet_AMPoll(); #endif } -static inline -void wait_done_obj(done_t* done, bool do_yield) +static inline void wait_done_obj(done_t* done, bool do_yield) { - while (!done->flag) { - am_poll_try(); - if (do_yield) - hpx::this_thread::yield(); - } + while (!done->flag) + { + am_poll_try(); + if (do_yield) + hpx::this_thread::yield(); + } } /////////////////////////////////////////////////////////////////////////////// @@ -223,8 +228,9 @@ namespace hpx { namespace util { } } - LBT_(info) << "No known GASNET environment variable found, disabling " - "GASNET support\n"; + LBT_(info) + << "No known GASNET environment variable found, disabling " + "GASNET support\n"; return false; } } // namespace detail @@ -238,7 +244,7 @@ namespace hpx { namespace util { // - The parcelport is explicitly disabled // - The application is not run in an GASNET environment // - The TCP parcelport is enabled and has higher priority - // + // if (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) == 0 || (get_entry_as(cfg, "hpx.parcel.tcp.enable", 1) && (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > @@ -247,7 +253,8 @@ namespace hpx { namespace util { (get_entry_as(cfg, "hpx.parcel.gasnet.priority", 1) > get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) { - LBT_(info) << "GASNET support disabled via configuration settings\n"; + LBT_(info) + << "GASNET support disabled via configuration settings\n"; return false; } @@ -272,59 +279,69 @@ namespace hpx { namespace util { gasnet_seginfo_t* gasnet_environment::segments = nullptr; /////////////////////////////////////////////////////////////////////////// - int gasnet_environment::init( - int* argc, char*** argv, const int minimal, const int required, int& provided) + int gasnet_environment::init(int* argc, char*** argv, const int minimal, + const int required, int& provided) { - if(!has_called_init_) { - gasnet_environment::init_val_ = gasnet_init(argc, argv); - has_called_init_ = true; + if (!has_called_init_) + { + gasnet_environment::init_val_ = gasnet_init(argc, argv); + has_called_init_ = true; } - if(gasnet_environment::init_val_ == GASNET_ERR_NOT_INIT) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET initialization error"); + if (gasnet_environment::init_val_ == GASNET_ERR_NOT_INIT) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET initialization error"); } - else if(gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET resource error"); + else if (gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", "GASNET resource error"); } - else if(gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET bad argument error"); + else if (gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET bad argument error"); } - else if(gasnet_environment::init_val_ == GASNET_ERR_NOT_READY) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET not ready error"); + else if (gasnet_environment::init_val_ == GASNET_ERR_NOT_READY) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET not ready error"); } - if (provided < minimal) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET doesn't provide minimal requested thread level"); + if (provided < minimal) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET doesn't provide minimal requested thread level"); } - if(gasnet_attach(nullptr, 0, gasnet_getMaxLocalSegmentSize(), 0) != GASNET_OK) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET failed to attach to memory"); + if (gasnet_attach(nullptr, 0, gasnet_getMaxLocalSegmentSize(), 0) != + GASNET_OK) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::init", + "GASNET failed to attach to memory"); } // create a number of segments equal to the number of hardware // threads per machine (locality) // gasnet_environment::segments = new gasnet_seginfo_t[size()]; - gasnet_environment::segment_mutex = new hpx::mutex[size()]; + gasnet_environment::segment_mutex = new hpx::mutex[size()]; GASNET_Safe(gasnet_getSegmentInfo(segments, size())); int retval; gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); - while ((retval = gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS)) == GASNET_ERR_NOT_READY) {} + while ((retval = gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS)) == + GASNET_ERR_NOT_READY) + { + } gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS); gasnet_set_waitmode(GASNET_WAIT_BLOCK); @@ -364,8 +381,8 @@ namespace hpx { namespace util { int msglen = 0; char message[1024 + 1]; - std::snprintf(message, 1024+1, "%s\n", gasnet_ErrorDesc(retval)); - msglen = strnlen(message, 1025+1); + std::snprintf(message, 1024 + 1, "%s\n", gasnet_ErrorDesc(retval)); + msglen = strnlen(message, 1025 + 1); std::string msg("gasnet_environment::init: gasnet_init failed: "); msg = msg + message + "."; @@ -389,10 +406,12 @@ namespace hpx { namespace util { gasnet_environment::segment_mutex = new hpx::mutex[size()]; #if defined(HPX_HAVE_NETWORKING) - if (this_rank == 0) { + if (this_rank == 0) + { rtcfg.mode_ = hpx::runtime_mode::console; } - else { + else + { rtcfg.mode_ = hpx::runtime_mode::worker; } #elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) @@ -402,7 +421,8 @@ namespace hpx { namespace util { #endif rtcfg.add_entry("hpx.parcel.gasnet.rank", std::to_string(this_rank)); - rtcfg.add_entry("hpx.parcel.gasnet.processorname", get_processor_name()); + rtcfg.add_entry( + "hpx.parcel.gasnet.processorname", get_processor_name()); } std::string gasnet_environment::get_processor_name() @@ -410,51 +430,60 @@ namespace hpx { namespace util { char name[1024 + 1] = {'\0'}; const std::string rnkstr = std::to_string(rank()); const int len = rnkstr.size(); - if(1025 < len) { - HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::get_processor_name", - "GASNET processor name is larger than 1025"); + if (1025 < len) + { + HPX_THROW_EXCEPTION(invalid_status, + "hpx::util::gasnet_environment::get_processor_name", + "GASNET processor name is larger than 1025"); } std::copy(std::begin(rnkstr), std::end(rnkstr), name); return name; } - bool gasnet_environment::gettable(const int node, void* start, const size_t len) + bool gasnet_environment::gettable( + const int node, void* start, const size_t len) { - const uintptr_t segstart = (uintptr_t) gasnet_environment::segments[node].addr; - const uintptr_t segend = segstart + gasnet_environment::segments[node].size; + const uintptr_t segstart = + (uintptr_t) gasnet_environment::segments[node].addr; + const uintptr_t segend = + segstart + gasnet_environment::segments[node].size; const uintptr_t reqstart = (uintptr_t) start; const uintptr_t reqend = reqstart + len; - return ( segstart <= reqstart && - reqstart <= segend && - segstart <= reqend && - reqend <= segend); + return (segstart <= reqstart && reqstart <= segend && + segstart <= reqend && reqend <= segend); } - void gasnet_environment::put(std::uint8_t * addr, const int node, std::uint8_t * raddr, const std::size_t size) { + void gasnet_environment::put(std::uint8_t* addr, const int node, + std::uint8_t* raddr, const std::size_t size) + { const bool in_remote_seg = gettable(node, raddr, size); - if(in_remote_seg) { + if (in_remote_seg) + { const std::lock_guard lk(segment_mutex[node]); - gasnet_put(node, static_cast(raddr), static_cast(addr), size); + gasnet_put(node, static_cast(raddr), + static_cast(addr), size); } - else { + else + { // tell the remote node to copy the data being sent // size_t max_chunk = gasnet_AMMaxMedium(); size_t start = 0; // AMRequestMedium will send put; the active message handler - // will memcpy on the remote host + // will memcpy on the remote host // - for(start = 0; start < size; start += max_chunk) { + for (start = 0; start < size; start += max_chunk) + { size_t this_size; void* addr_chunk; void* raddr_chunk; done_t done; this_size = size - start; - if( this_size > max_chunk ) { + if (this_size > max_chunk) + { this_size = max_chunk; } @@ -466,10 +495,8 @@ namespace hpx { namespace util { // Send an AM over to ask for a them to copy the data // passed in the active message (addr_chunk) to raddr_chunk. GASNET_Safe(gasnet_AMRequestMedium4(node, DO_COPY_PAYLOAD, - addr_chunk, this_size, - Arg0(&done), Arg1(&done), - Arg0(raddr_chunk), - Arg1(raddr_chunk))); + addr_chunk, this_size, Arg0(&done), Arg1(&done), + Arg0(raddr_chunk), Arg1(raddr_chunk))); // Wait for the PUT to complete. wait_done_obj(&done, false); @@ -477,13 +504,17 @@ namespace hpx { namespace util { } } - void gasnet_environment::get(std::uint8_t * addr, const int node, std::uint8_t * raddr, const std::size_t size) { + void gasnet_environment::get(std::uint8_t* addr, const int node, + std::uint8_t* raddr, const std::size_t size) + { bool remote_in_segment = false; - if (rank() == node) { + if (rank() == node) + { std::memmove(addr, raddr, size); } - else { + else + { // Handle remote address not in remote segment. // The GASNet Spec says: // The source memory address for all gets and the target memory address @@ -496,13 +527,15 @@ namespace hpx { namespace util { // remote_in_segment = gettable(node, raddr, size); - if( remote_in_segment ) { + if (remote_in_segment) + { // If raddr is in the remote segment, do a normal gasnet_get. // GASNet will handle the local portion not being in the segment. // - gasnet_get(addr, node, raddr, size); // dest, node, src, size + gasnet_get(addr, node, raddr, size); // dest, node, src, size } - else { + else + { // If raddr is not in the remote segment, we need to send an // active message; the other node will PUT back to us. // The local side has to be in the registered memory segment. @@ -518,9 +551,11 @@ namespace hpx { namespace util { // do the GET into a temporary buffer instead, and then // copy the result back. // - if( ! local_in_segment ) { + if (!local_in_segment) + { size_t buf_sz = size; - if( buf_sz > max_chunk ) { + if (buf_sz > max_chunk) + { buf_sz = max_chunk; } @@ -531,14 +566,16 @@ namespace hpx { namespace util { // do a PUT on the remote locale back to here. // But do it in chunks of size gasnet_AMMaxLongReply() // since we use gasnet_AMReplyLong to do the PUT. - for(start = 0; start < size; start += max_chunk) { + for (start = 0; start < size; start += max_chunk) + { size_t this_size; void* addr_chunk; xfer_info_t info; done_t done; this_size = size - start; - if( this_size > max_chunk ) { + if (this_size > max_chunk) + { this_size = max_chunk; } @@ -547,25 +584,27 @@ namespace hpx { namespace util { init_done_obj(&done, 1); info.ack = &done; - info.tgt = local_buf?local_buf:addr_chunk; + info.tgt = local_buf ? local_buf : addr_chunk; info.src = ((char*) raddr) + start; info.size = this_size; // Send an AM over to ask for a PUT back to us - GASNET_Safe(gasnet_AMRequestMedium0(node, DO_REPLY_PUT, - &info, sizeof(info))); + GASNET_Safe(gasnet_AMRequestMedium0( + node, DO_REPLY_PUT, &info, sizeof(info))); // Wait for the PUT to complete. wait_done_obj(&done, false); // Now copy from local_buf back to addr if necessary. - if( local_buf ) { + if (local_buf) + { std::memcpy(addr_chunk, local_buf, this_size); } } // If we were using a temporary local buffer free it - if( local_buf ) { + if (local_buf) + { free(local_buf); } } @@ -574,7 +613,8 @@ namespace hpx { namespace util { void gasnet_environment::finalize() { - if (enabled() && has_called_init()) { + if (enabled() && has_called_init()) + { gasnet_exit(1); } } diff --git a/libs/core/gasnet_base/tests/CMakeLists.txt b/libs/core/gasnet_base/tests/CMakeLists.txt index e6593cda9ac9..765589ee57bd 100644 --- a/libs/core/gasnet_base/tests/CMakeLists.txt +++ b/libs/core/gasnet_base/tests/CMakeLists.txt @@ -10,7 +10,9 @@ include(HPX_Option) if(HPX_WITH_TESTS) if(HPX_WITH_TESTS_UNIT) add_hpx_pseudo_target(tests.unit.modules.gasnet_base) - add_hpx_pseudo_dependencies(tests.unit.modules tests.unit.modules.gasnet_base) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.gasnet_base + ) add_subdirectory(unit) endif() diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index 1247a71b61de..966ba221406e 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -34,18 +34,13 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() -if(HPX_WITH_PARCELPORT_GASNET_MPI) - set(additional_dependencies ${additional_dependencies} Mpi::mpi PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_UDP) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_SMP) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_OFI) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_UCX) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) -elseif(HPX_WITH_PARCELPORT_GASNET_IBV) - set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +if(HPX_WITH_PARCELPORT_GASNET) + include(HPX_SetupGasnet) + hpx_setup_gasnet() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(additional_dependencies ${additional_dependencies} Mpi::mpi) + endif() + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) endif() include(HPX_AddModule) diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 1cf8df58efab..50c4380e81f2 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -1,59 +1,48 @@ -# Copyright (c) 2019-2021 The STE||AR-Group +# Copyright (c) 2023 Chistopher Taylor # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT)) +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_GASNET + AND HPX_WITH_PARCELPORT_GASNET_CONDUIT) +) return() endif() -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") -include(FindGasnet) -find_gasnet() +include(HPX_SetupGasnet) +hpx_setup_gasnet() set(parcelport_gasnet_headers hpx/parcelport_gasnet/header.hpp hpx/parcelport_gasnet/locality.hpp hpx/parcelport_gasnet/receiver.hpp + hpx/parcelport_gasnet/receiver_connection.hpp hpx/parcelport_gasnet/sender.hpp hpx/parcelport_gasnet/sender_connection.hpp + hpx/parcelport_gasnet/tag_provider.hpp ) -# cmake-format: off -set(parcelport_gasnet_compat_headers) -# cmake-format: on - set(parcelport_gasnet_sources locality.cpp parcelport_gasnet.cpp) -if(HPX_WITH_PARCELPORT_GASNET_CONDUIT AND ${HPX_WITH_PARCELPORT_GASNET_CONDUIT} STREQUAL "mpi") -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET Mpi::mpi - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) -else() -link_directories(${GASNET_LIBRARY_DIRS}) - include(HPX_AddModule) - add_hpx_module( - full parcelport_gasnet - GLOBAL_HEADER_GEN ON - SOURCES ${parcelport_gasnet_sources} - HEADERS ${parcelport_gasnet_headers} - COMPAT_HEADERS ${parcelport_gasnet_compat_headers} - DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET - MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset - CMAKE_SUBDIRS examples tests - ) +if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(gasnet_additional_dependencies Mpi::mpi) endif() +include(HPX_AddModule) +add_hpx_module( + full parcelport_gasnet + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_gasnet_sources} + HEADERS ${parcelport_gasnet_headers} + DEPENDENCIES hpx_core hpx_gasnet_base PkgConfig::GASNET + ${gasnet_additional_dependencies} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests +) + set(HPX_STATIC_PARCELPORT_PLUGINS ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_gasnet CACHE INTERNAL "" FORCE diff --git a/libs/full/parcelport_gasnet/examples/CMakeLists.txt b/libs/full/parcelport_gasnet/examples/CMakeLists.txt index 57dd90f4214e..24c8de88643c 100644 --- a/libs/full/parcelport_gasnet/examples/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/examples/CMakeLists.txt @@ -6,7 +6,9 @@ if(HPX_WITH_EXAMPLES) add_hpx_pseudo_target(examples.modules.parcelport_gasnet) - add_hpx_pseudo_dependencies(examples.modules examples.modules.parcelport_gasnet) + add_hpx_pseudo_dependencies( + examples.modules examples.modules.parcelport_gasnet + ) if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) add_hpx_pseudo_target(tests.examples.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp index d0282a2b0ed7..886118b244cf 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2023 Christopher Taylor // Copyright (c) 2013-2021 Hartmut Kaiser // Copyright (c) 2013-2015 Thomas Heller // diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp index 82c9b6c97892..98d315b97c5b 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/locality.hpp @@ -11,8 +11,8 @@ #include #if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET) -#include #include +#include #include @@ -43,7 +43,7 @@ namespace hpx::parcelset::policies::gasnet { explicit constexpr operator bool() const noexcept { - return rank_ != ((unsigned int)-1); + return rank_ != ((unsigned int) -1); } HPX_EXPORT void save(serialization::output_archive& ar) const; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index 6797de62e7bb..5d81749e0b7f 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -24,7 +25,6 @@ #include #include #include -#include namespace hpx::parcelset::policies::gasnet { @@ -37,23 +37,29 @@ namespace hpx::parcelset::policies::gasnet { using connection_ptr = std::shared_ptr; using connection_list = std::deque; - struct exp_backoff { + struct exp_backoff + { int numTries; const static int maxRetries = 10; - void operator()() { - if(numTries <= maxRetries) { + void operator()() + { + if (numTries <= maxRetries) + { gasnet_AMPoll(); - hpx::this_thread::suspend(std::chrono::microseconds(1 << numTries)); + hpx::this_thread::suspend( + std::chrono::microseconds(1 << numTries)); } - else { + else + { numTries = 0; } } }; explicit constexpr receiver(Parcelport& pp) noexcept - : pp_(pp), bo() + : pp_(pp) + , bo() { } @@ -122,7 +128,8 @@ namespace hpx::parcelset::policies::gasnet { // remote localities 'put' into the gasnet shared memory segment on this machine // - res.reset(new connection_type(hpx::util::gasnet_environment::rank(), h, pp_)); + res.reset(new connection_type( + hpx::util::gasnet_environment::rank(), h, pp_)); return res; } return res; @@ -133,8 +140,9 @@ namespace hpx::parcelset::policies::gasnet { header h = rcv_header_; rcv_header_.reset(); - while(rcv_header_.data() == 0) { - bo(); + while (rcv_header_.data() == 0) + { + bo(); } return h; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index edc74460fb17..512b18339c24 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -138,8 +138,8 @@ namespace hpx::parcelset::policies::gasnet { auto self_ = hpx::util::gasnet_environment::rank(); hpx::util::gasnet_environment::scoped_lock l; std::memcpy(buffer_.data_.data(), - hpx::util::gasnet_environment::segments[self_].addr, - buffer_.data_.size()); + hpx::util::gasnet_environment::segments[self_].addr, + buffer_.data_.size()); request_ptr_ = true; } @@ -150,7 +150,6 @@ namespace hpx::parcelset::policies::gasnet { bool receive_chunks(std::size_t num_thread = -1) { - while (chunks_idx_ < buffer_.chunks_.size()) { if (!request_done()) @@ -168,8 +167,8 @@ namespace hpx::parcelset::policies::gasnet { auto self_ = hpx::util::gasnet_environment::rank(); hpx::util::gasnet_environment::scoped_lock l; std::memcpy(c.data(), - hpx::util::gasnet_environment::segments[self_].addr, - c.size()); + hpx::util::gasnet_environment::segments[self_].addr, + c.size()); request_ptr_ = true; } } @@ -193,8 +192,8 @@ namespace hpx::parcelset::policies::gasnet { auto self_ = hpx::util::gasnet_environment::rank(); hpx::util::gasnet_environment::scoped_lock l; std::memcpy(&tag_, - hpx::util::gasnet_environment::segments[self_].addr, - sizeof(int)); + hpx::util::gasnet_environment::segments[self_].addr, + sizeof(int)); request_ptr_ = true; } diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index 744ac3a6ab95..faee1482d804 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -140,10 +140,12 @@ namespace hpx::parcelset::policies::gasnet { int get_next_free_tag() noexcept { int next_free = next_free_tag_; - + hpx::util::gasnet_environment::scoped_lock l; std::memcpy(&next_free, - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, sizeof(int)); return next_free; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index c7b7b3d84b80..1520400b41f9 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -16,13 +16,13 @@ #include #include +#include #include #include #include #include #include #include -#include #include #include @@ -152,25 +152,27 @@ namespace hpx::parcelset::policies::gasnet { // compute + send the number of GASNET_PAGEs to send and the remainder number of bytes to a GASNET_PAGE // - const std::size_t chunks[] = { static_cast(header_.data_size_ / GASNET_PAGESIZE), static_cast(header_.data_size_ % GASNET_PAGESIZE) }; + const std::size_t chunks[] = { + static_cast(header_.data_size_ / GASNET_PAGESIZE), + static_cast(header_.data_size_ % GASNET_PAGESIZE)}; const std::size_t sizeof_chunks = sizeof(chunks); - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, - chunks, sizeof_chunks); + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, + chunks, sizeof_chunks); // put from this localities gasnet shared memory segment // into the remote locality (dst_)'s shared memory segment // hpx::util::gasnet_environment::put( static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), - sizeof_chunks - ); + hpx::util::gasnet_environment::segments[dst_].addr), + sizeof_chunks); } state_ = sent_header; @@ -190,22 +192,22 @@ namespace hpx::parcelset::policies::gasnet { if (!chunks.empty()) { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, - chunks.data(), - static_cast(chunks.size() * - sizeof(parcel_buffer_type::transmission_chunk_type)) - ); - - gasnet_put_bulk(dst_, + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, + chunks.data(), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type))); + + gasnet_put_bulk(dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), + hpx::util::gasnet_environment::segments[dst_].addr), static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), - static_cast(chunks.size() * sizeof(parcel_buffer_type::transmission_chunk_type)) - ); + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type))); } state_ = sent_transmission_chunks; @@ -223,20 +225,20 @@ namespace hpx::parcelset::policies::gasnet { if (!header_.piggy_back()) { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, buffer_.data_.data(), buffer_.data_.size()); hpx::util::gasnet_environment::put( static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), - buffer_.data_.size() - ); + hpx::util::gasnet_environment::segments[dst_].addr), + buffer_.data_.size()); } state_ = sent_data; @@ -260,22 +262,20 @@ namespace hpx::parcelset::policies::gasnet { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr, - c.data_.cpos_, - static_cast(c.size_) - ); + std::memcpy(hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr, + c.data_.cpos_, static_cast(c.size_)); hpx::util::gasnet_environment::put( static_cast( - hpx::util::gasnet_environment::segments[hpx::util::gasnet_environment::rank()].addr - ), + hpx::util::gasnet_environment::segments + [hpx::util::gasnet_environment::rank()] + .addr), dst_, static_cast( - hpx::util::gasnet_environment::segments[dst_].addr - ), - static_cast(c.size_) - ); + hpx::util::gasnet_environment::segments[dst_].addr), + static_cast(c.size_)); } ++chunks_idx_; diff --git a/libs/full/parcelport_gasnet/src/locality.cpp b/libs/full/parcelport_gasnet/src/locality.cpp index 150ce6e82834..5e9e309c1375 100644 --- a/libs/full/parcelport_gasnet/src/locality.cpp +++ b/libs/full/parcelport_gasnet/src/locality.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2023 Christopher Taylor // Copyright (c) 2007-2021 Hartmut Kaiser // Copyright (c) 2013-2014 Thomas Heller // diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 728d107f6258..b29d21006ac6 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -106,7 +106,7 @@ namespace hpx::parcelset { static std::size_t background_threads( util::runtime_configuration const& ini) { -/* + /* return hpx::util::get_entry_as(ini, "hpx.parcel.gasnet.background_threads", HPX_HAVE_PARCELPORT_GASNET_BACKGROUND_THREADS); @@ -160,18 +160,20 @@ namespace hpx::parcelset { "gasnet::parcelport::do_stop"); } - bool expected = false; if (stopped_.compare_exchange_strong(expected, true)) { stopped_ = true; gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); - if((retval = gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) { + if ((retval = gasnet_barrier_wait( + 0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) + { // throw exception HPX_THROW_EXCEPTION(invalid_status, - "hpx::util::gasnet_environment::init", - "GASNET failed ", std::string{gasnet_ErrorName(retval)}, " ", - std::string{gasnet_ErrorDesc(retval)}); + "hpx::util::gasnet_environment::init", + "GASNET failed ", + std::string{gasnet_ErrorName(retval)}, " ", + std::string{gasnet_ErrorDesc(retval)}); gasnet_exit(retval); } } @@ -291,12 +293,14 @@ namespace hpx::traits { return "1"; } - static void init(int* argc, char*** argv, util::command_line_handling& cfg) + static void init( + int* argc, char*** argv, util::command_line_handling& cfg) { util::gasnet_environment::init(argc, argv, cfg.rtcfg_); cfg.num_localities_ = static_cast(util::gasnet_environment::size()); - cfg.node_ = static_cast(util::gasnet_environment::rank()); + cfg.node_ = + static_cast(util::gasnet_environment::rank()); } // by default no additional initialization using the resource From 88b431b92c32e2808a54b3e75a6ab3a1f2e9beae Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 13 Jul 2023 16:29:51 -0500 Subject: [PATCH 046/333] Adding gasnet parcelport to CI builders - flyby: fixing various inspect, clang-format, etc. errors --- .jenkins/lsu/env-clang-11.sh | 2 + .jenkins/lsu/env-clang-12.sh | 2 + .jenkins/lsu/env-clang-13.sh | 2 + .jenkins/lsu/env-clang-14.sh | 2 + .jenkins/lsu/env-clang-15.sh | 2 + .jenkins/lsu/env-gcc-10.sh | 2 + .jenkins/lsu/env-gcc-12.sh | 2 + .jenkins/lsu/env-gcc-13.sh | 2 + .jenkins/lsu/env-gcc-9.sh | 2 + cmake/HPX_SetupGasnet.cmake | 6 +- libs/CMakeLists.txt | 4 ++ libs/core/gasnet_base/CMakeLists.txt | 3 +- libs/core/gasnet_base/README.rst | 16 ----- libs/core/gasnet_base/docs/index.rst | 12 ++-- .../hpx/gasnet_base/gasnet_environment.hpp | 2 + .../gasnet_base/src/gasnet_environment.cpp | 69 ++++++++++--------- libs/core/version/src/version.cpp | 6 +- .../src/command_line_handling.cpp | 2 +- libs/full/parcelport_gasnet/README.rst | 16 ----- libs/full/parcelport_gasnet/docs/index.rst | 4 +- .../hpx/parcelport_gasnet/receiver.hpp | 3 +- .../include/hpx/parcelport_gasnet/sender.hpp | 1 + .../parcelport_gasnet/sender_connection.hpp | 5 +- .../src/parcelport_gasnet.cpp | 4 +- 24 files changed, 86 insertions(+), 85 deletions(-) delete mode 100644 libs/core/gasnet_base/README.rst delete mode 100644 libs/full/parcelport_gasnet/README.rst diff --git a/.jenkins/lsu/env-clang-11.sh b/.jenkins/lsu/env-clang-11.sh index afb97de69955..77b0f84614c5 100644 --- a/.jenkins/lsu/env-clang-11.sh +++ b/.jenkins/lsu/env-clang-11.sh @@ -11,6 +11,7 @@ module load boost/1.73.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" @@ -23,6 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index e7d9d51020c5..712e11d6077a 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -11,6 +11,7 @@ module load boost/1.75.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -24,6 +25,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index d37c78c912a3..e23c2ccd28bb 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -11,6 +11,7 @@ module load boost/1.78.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" @@ -24,6 +25,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index a0662d0d94c3..09b9022267f4 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -11,6 +11,7 @@ module load boost/1.79.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -24,6 +25,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index a18f45524e26..27129b473f0e 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -11,6 +11,7 @@ module load boost/1.82.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -24,6 +25,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index ba838f6a050e..10eda35ad81f 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -11,6 +11,7 @@ module load boost/1.75.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" @@ -24,6 +25,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index f01d1804cf87..10d48af9bd60 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -11,6 +11,7 @@ module load boost/1.80.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -23,6 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index e49b301c5cc0..c4b4326b57da 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -11,6 +11,7 @@ module load boost/1.82.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -23,6 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-9.sh b/.jenkins/lsu/env-gcc-9.sh index 4ec9d2ff5b3d..f78a0175e0c9 100644 --- a/.jenkins/lsu/env-gcc-9.sh +++ b/.jenkins/lsu/env-gcc-9.sh @@ -11,6 +11,7 @@ module load boost/1.73.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 +module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" @@ -23,6 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 01bf57ad22d9..97e4aec4bcab 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -13,7 +13,6 @@ macro(hpx_setup_gasnet) find_package(PkgConfig REQUIRED QUIET COMPONENTS) hpx_info( - STATUS "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" ) pkg_search_module( @@ -27,8 +26,6 @@ macro(hpx_setup_gasnet) hpx_setup_mpi() endif() - target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) - if(GASNET_CFLAGS) set(IS_PARAM "0") set(PARAM_FOUND "0") @@ -542,7 +539,8 @@ macro(hpx_setup_gasnet) PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" ) set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES "${GASNET_LIBRARY_DIRS}" + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" ) endif() diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 1791882be2e0..0f37a7ed1d71 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -377,6 +377,10 @@ if(HPX_WITH_ITTNOTIFY) target_link_libraries(hpx_core PUBLIC Amplifier::amplifier) endif() +if(HPX_WITH_PARCELPORT_GASNET AND GASNET_LIBRARY_DIRS) + target_link_directories(hpx_core PUBLIC ${GASNET_LIBRARY_DIRS}) +endif() + if(HPX_WITH_MODULES_AS_STATIC_LIBRARIES OR HPX_WITH_STATIC_LINKING) target_link_libraries(hpx_core PUBLIC hpx_public_flags) target_link_libraries(hpx_core PUBLIC hpx_base_libraries) diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 0a5736df497b..6a6401e03b0e 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -33,6 +33,7 @@ add_hpx_module( SOURCES ${gasnet_base_sources} HEADERS ${gasnet_base_headers} DEPENDENCIES PkgConfig::GASNET ${gasnet_additional_dependencies} - MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_util + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_string_util + hpx_threading_base hpx_util CMAKE_SUBDIRS examples tests ) diff --git a/libs/core/gasnet_base/README.rst b/libs/core/gasnet_base/README.rst deleted file mode 100644 index 1ae4ac39ce3b..000000000000 --- a/libs/core/gasnet_base/README.rst +++ /dev/null @@ -1,16 +0,0 @@ - -.. - Copyright (c) 2020 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -======== -mpi_base -======== - -This library is part of HPX. - -Documentation can be found `here -`__. diff --git a/libs/core/gasnet_base/docs/index.rst b/libs/core/gasnet_base/docs/index.rst index 43dbdb7c2a03..cab19901c949 100644 --- a/libs/core/gasnet_base/docs/index.rst +++ b/libs/core/gasnet_base/docs/index.rst @@ -5,14 +5,14 @@ Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -.. _modules_mpi_base: +.. _modules_gasnet_base: -======== -mpi_base -======== +=========== +gasnet_base +=========== -This module provides helper functionality for detecting MPI environments. +This module provides helper functionality for detecting GASNET environments. -See the :ref:`API reference ` of this module for more +See the :ref:`API reference ` of this module for more details. diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp index fe66baaeca11..617e7477e9ee 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet_environment.hpp @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 1f8accd5837a..f99d0fe6507b 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -8,16 +8,17 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include - +#include #include #include #include +#include +#include #include -#include - -#include +#include #include +#include #include #include #include @@ -48,18 +49,24 @@ typedef struct static inline gasnet_handlerarg_t get_arg_from_ptr0(uintptr_t addr) { // This one returns the bottom 32 bits. - return ((gasnet_handlerarg_t)((((uint64_t)(addr)) << 32UL) >> 32UL)); + // clang-format off + return ((gasnet_handlerarg_t) ((((uint64_t) (addr)) << 32UL) >> 32UL)); + // clang-format on } static inline gasnet_handlerarg_t get_arg_from_ptr1(uintptr_t addr) { // this one returns the top 32 bits. - return ((gasnet_handlerarg_t)(((uint64_t)(addr)) >> 32UL)); + // clang-format off + return ((gasnet_handlerarg_t) (((uint64_t) (addr)) >> 32UL)); + // clang-format on } static inline uintptr_t get_uintptr_from_args( gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - return (uintptr_t)( - ((uint64_t)(uint32_t) a0) | (((uint64_t)(uint32_t) a1) << 32UL)); + // clang-format off + return (uintptr_t) (((uint64_t) (uint32_t) a0) | + (((uint64_t) (uint32_t) a1) << 32UL)); + // clang-format on } static inline void* get_ptr_from_args( gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) @@ -111,7 +118,7 @@ typedef struct static void AM_signal( gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - done_t* done = (done_t*) get_ptr_from_args(a0, a1); + done_t* done = reinterpret_cast(get_ptr_from_args(a0, a1)); uint_least32_t prev; prev = done->count.fetch_add(1, std::memory_order_seq_cst); if (prev + 1 == done->target) @@ -121,7 +128,7 @@ static void AM_signal( static void AM_signal_long(gasnet_token_t token, void* buf, size_t nbytes, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { - done_t* done = (done_t*) get_ptr_from_args(a0, a1); + done_t* done = reinterpret_cast(get_ptr_from_args(a0, a1)); uint_least32_t prev; prev = done->count.fetch_add(1, std::memory_order_seq_cst); if (prev + 1 == done->target) @@ -136,7 +143,7 @@ static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) { xfer_info_t* x = static_cast(buf); - assert(nbytes == sizeof(xfer_info_t)); + HPX_ASSERT(nbytes == sizeof(xfer_info_t)); GASNET_Safe(gasnet_AMReplyLong2(token, SIGNAL_LONG, x->src, x->size, x->tgt, Arg0(x->ack), Arg1(x->ack))); @@ -198,12 +205,15 @@ static inline void wait_done_obj(done_t* done, bool do_yield) { am_poll_try(); if (do_yield) - hpx::this_thread::yield(); + { + this_thread::suspend(threads::thread_schedule_state::pending, + "gasnet::wait_done_obj"); + } } } /////////////////////////////////////////////////////////////////////////////// -namespace hpx { namespace util { +namespace hpx::util { namespace detail { @@ -213,9 +223,8 @@ namespace hpx { namespace util { std::string gasnet_environment_strings = cfg.get_entry("hpx.parcel.gasnet.env", default_env); - boost::char_separator sep(";,: "); - boost::tokenizer> tokens( - gasnet_environment_strings, sep); + hpx::string_util::char_separator sep(";,: "); + hpx::string_util::tokenizer tokens(gasnet_environment_strings, sep); for (auto const& tok : tokens) { char* env = std::getenv(tok.c_str()); @@ -238,7 +247,7 @@ namespace hpx { namespace util { bool gasnet_environment::check_gasnet_environment( util::runtime_configuration const& cfg) { -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE)) +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE) // We disable the GASNET parcelport if any of these hold: // // - The parcelport is explicitly disabled @@ -259,13 +268,15 @@ namespace hpx { namespace util { } return true; +#else + return false; #endif } -}} // namespace hpx::util +} // namespace hpx::util #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_GASNET_BASE)) -namespace hpx { namespace util { +namespace hpx::util { hpx::spinlock gasnet_environment::pollingLock{}; hpx::mutex gasnet_environment::dshm_mut{}; @@ -379,10 +390,8 @@ namespace hpx { namespace util { enabled_ = false; - int msglen = 0; char message[1024 + 1]; std::snprintf(message, 1024 + 1, "%s\n", gasnet_ErrorDesc(retval)); - msglen = strnlen(message, 1025 + 1); std::string msg("gasnet_environment::init: gasnet_init failed: "); msg = msg + message + "."; @@ -477,8 +486,6 @@ namespace hpx { namespace util { for (start = 0; start < size; start += max_chunk) { size_t this_size; - void* addr_chunk; - void* raddr_chunk; done_t done; this_size = size - start; @@ -487,8 +494,8 @@ namespace hpx { namespace util { this_size = max_chunk; } - addr_chunk = ((char*) addr) + start; - raddr_chunk = ((char*) raddr) + start; + void* addr_chunk = addr + start; + void* raddr_chunk = raddr + start; init_done_obj(&done, 1); @@ -507,8 +514,6 @@ namespace hpx { namespace util { void gasnet_environment::get(std::uint8_t* addr, const int node, std::uint8_t* raddr, const std::size_t size) { - bool remote_in_segment = false; - if (rank() == node) { std::memmove(addr, raddr, size); @@ -525,7 +530,7 @@ namespace hpx { namespace util { // In other words, it is OK if the local side of a GET or PUT // is not in the registered memory region. // - remote_in_segment = gettable(node, raddr, size); + bool remote_in_segment = gettable(node, raddr, size); if (remote_in_segment) { @@ -560,7 +565,7 @@ namespace hpx { namespace util { } local_buf = calloc(1, buf_sz); - assert(gettable(node, local_buf, buf_sz)); + HPX_ASSERT(gettable(node, local_buf, buf_sz)); } // do a PUT on the remote locale back to here. @@ -579,13 +584,13 @@ namespace hpx { namespace util { this_size = max_chunk; } - addr_chunk = ((char*) addr) + start; + addr_chunk = addr + start; init_done_obj(&done, 1); info.ack = &done; info.tgt = local_buf ? local_buf : addr_chunk; - info.src = ((char*) raddr) + start; + info.src = raddr + start; info.size = this_size; // Send an AM over to ask for a PUT back to us @@ -691,6 +696,6 @@ namespace hpx { namespace util { mtx_.unlock(); } } -}} // namespace hpx::util +} // namespace hpx::util #endif diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index 445af6cb6f5c..368be5087f32 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -131,7 +131,7 @@ namespace hpx { } #endif -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) std::string gasnet_version() { @@ -304,7 +304,7 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) " LCI: {}\n" #endif -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) " GASNET: {}\n" #endif @@ -324,7 +324,7 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) lci_version(), #endif -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) gasnet_version(), #endif diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index a2946563f961..b50ed8b8a14e 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -990,7 +990,7 @@ namespace hpx::util { node_ = static_cast(util::lci_environment::rank()); } #endif -#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) // better to put GASNET init after MPI init, since GASNET will also // initialize MPI if MPI is not already initialized. diff --git a/libs/full/parcelport_gasnet/README.rst b/libs/full/parcelport_gasnet/README.rst deleted file mode 100644 index c0729a670a32..000000000000 --- a/libs/full/parcelport_gasnet/README.rst +++ /dev/null @@ -1,16 +0,0 @@ - -.. - Copyright (c) 2020-2021 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -================= -parcelport_gasnet -================= - -This module is part of HPX. - -Documentation can be found `here -`__. diff --git a/libs/full/parcelport_gasnet/docs/index.rst b/libs/full/parcelport_gasnet/docs/index.rst index d61191bdd7d7..e26471c5e5e6 100644 --- a/libs/full/parcelport_gasnet/docs/index.rst +++ b/libs/full/parcelport_gasnet/docs/index.rst @@ -7,9 +7,9 @@ .. _modules_parcelport_gasnet: -============== +================= parcelport_gasnet -============== +================= TODO: High-level description of the module. diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index 5d81749e0b7f..99911618c058 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -126,7 +126,8 @@ namespace hpx::parcelset::policies::gasnet { l.unlock(); header_lock.unlock(); - // remote localities 'put' into the gasnet shared memory segment on this machine + // remote localities 'put' into the gasnet shared + // memory segment on this machine // res.reset(new connection_type( hpx::util::gasnet_environment::rank(), h, pp_)); diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index faee1482d804..e56d4b6a7aa7 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp index 1520400b41f9..1ca36622e663 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender_connection.hpp @@ -25,6 +25,8 @@ #include #include +#include +#include #include #include #include @@ -150,7 +152,8 @@ namespace hpx::parcelset::policies::gasnet { hpx::util::gasnet_environment::scoped_lock l; HPX_ASSERT(state_ == initialized); - // compute + send the number of GASNET_PAGEs to send and the remainder number of bytes to a GASNET_PAGE + // compute + send the number of GASNET_PAGEs to send and the + // remainder number of bytes to a GASNET_PAGE // const std::size_t chunks[] = { static_cast(header_.data_size_ / GASNET_PAGESIZE), diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index b29d21006ac6..159f9b0f84bc 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -151,7 +151,6 @@ namespace hpx::parcelset { // Stop the handling of connections. void do_stop() { - int retval; while (do_background_work(0, parcelport_background_mode_all)) { if (threads::get_self_ptr()) @@ -164,6 +163,8 @@ namespace hpx::parcelset { if (stopped_.compare_exchange_strong(expected, true)) { stopped_ = true; + + int retval; gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); if ((retval = gasnet_barrier_wait( 0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) @@ -174,7 +175,6 @@ namespace hpx::parcelset { "GASNET failed ", std::string{gasnet_ErrorName(retval)}, " ", std::string{gasnet_ErrorDesc(retval)}); - gasnet_exit(retval); } } } From b0f1f2546e19dce77057eb3f145eb55f3b7f781b Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 14 Jul 2023 06:46:48 -0500 Subject: [PATCH 047/333] Fixing HPX compilation errors --- .../core/gasnet_base/src/gasnet_environment.cpp | 17 +++++++++-------- .../parcelport_gasnet/src/parcelport_gasnet.cpp | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index f99d0fe6507b..3673b2f85a16 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -206,7 +206,8 @@ static inline void wait_done_obj(done_t* done, bool do_yield) am_poll_try(); if (do_yield) { - this_thread::suspend(threads::thread_schedule_state::pending, + hpx::this_thread::suspend( + hpx::threads::thread_schedule_state::pending, "gasnet::wait_done_obj"); } } @@ -301,31 +302,31 @@ namespace hpx::util { if (gasnet_environment::init_val_ == GASNET_ERR_NOT_INIT) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET initialization error"); } else if (gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::vinvalid_status, "hpx::util::gasnet_environment::init", "GASNET resource error"); } else if (gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET bad argument error"); } else if (gasnet_environment::init_val_ == GASNET_ERR_NOT_READY) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET not ready error"); } if (provided < minimal) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET doesn't provide minimal requested thread level"); } @@ -333,7 +334,7 @@ namespace hpx::util { if (gasnet_attach(nullptr, 0, gasnet_getMaxLocalSegmentSize(), 0) != GASNET_OK) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET failed to attach to memory"); } @@ -441,7 +442,7 @@ namespace hpx::util { const int len = rnkstr.size(); if (1025 < len) { - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::get_processor_name", "GASNET processor name is larger than 1025"); } diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 159f9b0f84bc..2dd46216b8ae 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -170,7 +170,7 @@ namespace hpx::parcelset { 0, GASNET_BARRIERFLAG_ANONYMOUS)) != GASNET_OK) { // throw exception - HPX_THROW_EXCEPTION(invalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET failed ", std::string{gasnet_ErrorName(retval)}, " ", From 40146e553ea7ec2a3afbe87becefc8097a2f4764 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 14 Jul 2023 08:40:49 -0400 Subject: [PATCH 048/333] Update libs/core/gasnet_base/src/gasnet_environment.cpp corrections to use of gasnet polling Co-authored-by: Dan Bonachea --- libs/core/gasnet_base/src/gasnet_environment.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 3673b2f85a16..ede35114f684 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -350,11 +350,7 @@ namespace hpx::util { int retval; gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); - while ((retval = gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS)) == - GASNET_ERR_NOT_READY) - { - } - gasnet_barrier_try(0, GASNET_BARRIERFLAG_ANONYMOUS); + gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS); gasnet_set_waitmode(GASNET_WAIT_BLOCK); From 3efb6c465ae84b3b71e3c3c57e88fbd86b53f992 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Fri, 14 Jul 2023 07:41:49 -0500 Subject: [PATCH 049/333] rm'd define and corrected envronment/versioning informaiton --- libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp | 4 ---- libs/core/gasnet_base/src/gasnet_environment.cpp | 2 +- libs/core/version/src/version.cpp | 5 ++++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp index b13044c7ad59..d8c4ce1a5672 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -17,10 +17,6 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif -#ifndef GASNET_PAR -#define GASNET_PAR 1 -#endif - #include #if defined(__clang__) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index f99d0fe6507b..4c7cc0e5d6f0 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -206,7 +206,7 @@ static inline void wait_done_obj(done_t* done, bool do_yield) am_poll_try(); if (do_yield) { - this_thread::suspend(threads::thread_schedule_state::pending, + hpx::this_thread::suspend(hpx::threads::thread_schedule_state::pending, "gasnet::wait_done_obj"); } } diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index 368be5087f32..f856cbc324a5 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -136,7 +136,10 @@ namespace hpx { std::string gasnet_version() { std::ostringstream strm; - strm << "GASNET"; + strm << "GASNET_VERSION:" << GASNET_RELEASE_VERSION_MAJOR << ':' + << GASNET_RELEASE_VERSION_MINOR << ':' + << GASNET_RELEASE_VERSION_PATCH << '-' + << "GASNET_CONDUIT:" << GASNET_CONDUIT_NAME_STR; return strm.str(); } #endif From aea15b1cae75e51c8ea51d4907ee5e6790acc777 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 14 Jul 2023 08:20:47 -0500 Subject: [PATCH 050/333] Fixing clang-format and inspect errors --- libs/core/version/src/version.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index f856cbc324a5..f965786e8eeb 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -137,9 +137,9 @@ namespace hpx { { std::ostringstream strm; strm << "GASNET_VERSION:" << GASNET_RELEASE_VERSION_MAJOR << ':' - << GASNET_RELEASE_VERSION_MINOR << ':' - << GASNET_RELEASE_VERSION_PATCH << '-' - << "GASNET_CONDUIT:" << GASNET_CONDUIT_NAME_STR; + << GASNET_RELEASE_VERSION_MINOR << ':' + << GASNET_RELEASE_VERSION_PATCH << '-' + << "GASNET_CONDUIT:" << GASNET_CONDUIT_NAME_STR; return strm.str(); } #endif From 227485818506145b72cf559dcecd50c6cac71247 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Fri, 14 Jul 2023 09:07:25 -0500 Subject: [PATCH 051/333] fixed compilation errors and cmake config --- libs/core/gasnet_base/src/gasnet_environment.cpp | 2 +- libs/core/version/CMakeLists.txt | 7 +++++++ libs/core/version/src/version.cpp | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index ede35114f684..2a9fef43e0a4 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -308,7 +308,7 @@ namespace hpx::util { } else if (gasnet_environment::init_val_ == GASNET_ERR_RESOURCE) { - HPX_THROW_EXCEPTION(error::vinvalid_status, + HPX_THROW_EXCEPTION(error::invalid_status, "hpx::util::gasnet_environment::init", "GASNET resource error"); } else if (gasnet_environment::init_val_ == GASNET_ERR_BAD_ARG) diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index a6013607bcbd..e4dd1faeec16 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -20,6 +20,13 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_GASNET + AND HPX_WITH_PARCELPORT_GASNET_CONDUIT) + include(HPX_SetupGasnet) + hpx_setup_gasnet() + set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) +endif() include(HPX_AddModule) add_hpx_module( diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index f856cbc324a5..791418bbc332 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -46,6 +46,11 @@ #include #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ + defined(HPX_HAVE_MODULE_GASNET_BASE) +#include +#endif + #include #include From 88b15c5d178b42d4533b17dbb57403439668f362 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Fri, 14 Jul 2023 11:37:57 -0500 Subject: [PATCH 052/333] fixed formatting issues; found an issue in a gasnet-init method --- libs/core/gasnet_base/src/gasnet_environment.cpp | 10 ++-------- libs/core/version/src/version.cpp | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 2a9fef43e0a4..3fb9e919a706 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -342,6 +342,8 @@ namespace hpx::util { // create a number of segments equal to the number of hardware // threads per machine (locality) // + //segments.resize(hpx::threads::hardware_concurrency() * size()); + // gasnet_environment::segments = new gasnet_seginfo_t[size()]; gasnet_environment::segment_mutex = new hpx::mutex[size()]; @@ -402,14 +404,6 @@ namespace hpx::util { } this_rank = rank(); - // need to figure this out to improve efficiency; create - // a number of segments equal to the number of hardware - // threads per machine (locality) - // - //segments.resize(hpx::threads::hardware_concurrency() * size()); - // - gasnet_environment::segments = new gasnet_seginfo_t[size()]; - gasnet_environment::segment_mutex = new hpx::mutex[size()]; #if defined(HPX_HAVE_NETWORKING) if (this_rank == 0) diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index bd7404b213a5..96d1662efdf2 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -48,7 +48,7 @@ #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) -#include +#include #endif #include From 8a8c884707e76bb6349268182c0e64c3807a8a4d Mon Sep 17 00:00:00 2001 From: ctaylor Date: Mon, 17 Jul 2023 09:16:05 -0500 Subject: [PATCH 053/333] added Cray ARIES conduit support --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d32ef46c522..3208b4d7b884 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1152,7 +1152,7 @@ if(HPX_WITH_NETWORKING) hpx_option( HPX_WITH_PARCELPORT_GASNET_CONDUIT STRING "Define which conduit to use for the gasnet parcelport" "udp" - STRINGS "smp;udp;mpi;ofi;ucx;ibv" + STRINGS "smp;udp;mpi;ofi;ucx;ibv;aries" ) endif() From 99f44a3f73cbff8f32f14d0f65a8d0d8427c04fe Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 17 Jul 2023 12:27:20 -0400 Subject: [PATCH 054/333] Update CMakeLists.txt Co-authored-by: Hartmut Kaiser --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3208b4d7b884..e178d15eeee0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1148,11 +1148,14 @@ if(HPX_WITH_NETWORKING) CATEGORY "Parcelport" ) if(HPX_WITH_PARCELPORT_GASNET) + if(...is cray aries available) + set(additional_conduits ";aries" + endif() hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) hpx_option( HPX_WITH_PARCELPORT_GASNET_CONDUIT STRING "Define which conduit to use for the gasnet parcelport" "udp" - STRINGS "smp;udp;mpi;ofi;ucx;ibv;aries" + STRINGS "smp;udp;mpi;ofi;ucx;ibv" ${additional_conduits} ) endif() From da9cc163191324e83d5f558e9ab2adbf6c5f2f84 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 18 Jul 2023 14:13:33 -0500 Subject: [PATCH 055/333] fixed aries support; turned off --- CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e178d15eeee0..9d32ef46c522 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1148,14 +1148,11 @@ if(HPX_WITH_NETWORKING) CATEGORY "Parcelport" ) if(HPX_WITH_PARCELPORT_GASNET) - if(...is cray aries available) - set(additional_conduits ";aries" - endif() hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) hpx_option( HPX_WITH_PARCELPORT_GASNET_CONDUIT STRING "Define which conduit to use for the gasnet parcelport" "udp" - STRINGS "smp;udp;mpi;ofi;ucx;ibv" ${additional_conduits} + STRINGS "smp;udp;mpi;ofi;ucx;ibv" ) endif() From 8cb32ba57a4664b1141a2ec277811731b021d530 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Thu, 20 Jul 2023 13:27:50 -0400 Subject: [PATCH 056/333] hpx/cmake : added proper linking support for clang --- cmake/HPX_SetupGasnet.cmake | 327 ++++++++++++++++++++++++------------ 1 file changed, 222 insertions(+), 105 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 97e4aec4bcab..f91eba83be68 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -157,32 +157,62 @@ macro(hpx_setup_gasnet) list(LENGTH LIB_LIST IDX) if(NOT "${IDX}" EQUAL "0") set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(NEWLINK "SHELL:-Wl,-force_load,") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:-Wl,-force_load" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() endif() endif() endif() @@ -246,33 +276,62 @@ macro(hpx_setup_gasnet) list(LENGTH LIB_LIST IDX) if(NOT "${IDX}" EQUAL "0") set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(NEWLINK "SHELL:-Wl,-force_load,") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:-Wl,-force_load" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() endif() endif() @@ -409,34 +468,63 @@ macro(hpx_setup_gasnet) list(LENGTH LIB_LIST IDX) if(NOT "${IDX}" EQUAL "0") set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(NEWLINK "SHELL:-Wl,-force_load,") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:-Wl,-force_load" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() endif() endif() endif() @@ -500,34 +588,63 @@ macro(hpx_setup_gasnet) list(LENGTH LIB_LIST IDX) if(NOT "${IDX}" EQUAL "0") set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(NEWLINK "SHELL:-Wl,-force_load,") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:-Wl,-force_load" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() endif() endif() endif() From c9734ea7cb737a65afa3ac3d1f5b5a06760736c5 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Thu, 20 Jul 2023 19:03:21 -0500 Subject: [PATCH 057/333] added support for x86 clang --- cmake/HPX_SetupGasnet.cmake | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index f91eba83be68..5b4e49d6a886 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -187,7 +187,11 @@ macro(hpx_setup_gasnet) list(APPEND GASNET_LDFLAGS "${NEWLINK}") endif() elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(NEWLINK "SHELL:-Wl,-force_load,") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() foreach(X IN ITEMS ${LIB_LIST}) set(DIRSTR "") string(REPLACE ";" " " DIRSTR "${DIR_LIST}") @@ -207,7 +211,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:-Wl,-force_load" + string(FIND "SHELL:" "${NEWLINK}" IDX ) if("${IDX}" EQUAL "-1") @@ -306,7 +310,11 @@ macro(hpx_setup_gasnet) list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") endif() elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(NEWLINK "SHELL:-Wl,-force_load,") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() foreach(X IN ITEMS ${LIB_LIST}) set(DIRSTR "") string(REPLACE ";" " " DIRSTR "${DIR_LIST}") @@ -326,7 +334,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:-Wl,-force_load" + string(FIND "SHELL:" "${NEWLINK}" IDX ) if("${IDX}" EQUAL "-1") @@ -499,7 +507,11 @@ macro(hpx_setup_gasnet) list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") endif() elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(NEWLINK "SHELL:-Wl,-force_load,") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() foreach(X IN ITEMS ${LIB_LIST}) set(DIRSTR "") string(REPLACE ";" " " DIRSTR "${DIR_LIST}") @@ -519,7 +531,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:-Wl,-force_load" + string(FIND "SHELL:" "${NEWLINK}" IDX ) if("${IDX}" EQUAL "-1") @@ -619,7 +631,11 @@ macro(hpx_setup_gasnet) list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") endif() elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(NEWLINK "SHELL:-Wl,-force_load,") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() foreach(X IN ITEMS ${LIB_LIST}) set(DIRSTR "") string(REPLACE ";" " " DIRSTR "${DIR_LIST}") @@ -639,7 +655,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:-Wl,-force_load" + string(FIND "SHELL:" "${NEWLINK}" IDX ) if("${IDX}" EQUAL "-1") From 246fbfa1be5bc0731f373c3a0417789ed07cb85e Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 22 Jul 2023 21:11:27 -0400 Subject: [PATCH 058/333] missing extern 'C' wrapper around gasnet.h include --- libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp index d8c4ce1a5672..2cec92aea106 100644 --- a/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp +++ b/libs/core/gasnet_base/include/hpx/gasnet_base/gasnet.hpp @@ -17,8 +17,16 @@ #pragma GCC diagnostic ignored "-Wcast-qual" #endif +#ifdef __cplusplus +extern "C" { +#endif + #include +#ifdef __cplusplus +} +#endif + #if defined(__clang__) #pragma clang diagnostic pop #elif defined(__GNUC__) From 833955ea0012d13e9a5b7d27e85fe67c17f32ca0 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 27 Jul 2023 10:38:20 -0500 Subject: [PATCH 059/333] Working around false positive diagnostics generated by gcc --- .../src/command_line_handling_local.cpp | 7 +++++ .../src/late_command_line_handling_local.cpp | 7 +++++ .../parcelport_gasnet/receiver_connection.hpp | 3 +- .../action_invocation_counter_discoverer.cpp | 7 +++++ .../performance_counters/src/counters.cpp | 28 +++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/libs/core/command_line_handling_local/src/command_line_handling_local.cpp b/libs/core/command_line_handling_local/src/command_line_handling_local.cpp index 71f286d596d5..fbffe1e7a3d6 100644 --- a/libs/core/command_line_handling_local/src/command_line_handling_local.cpp +++ b/libs/core/command_line_handling_local/src/command_line_handling_local.cpp @@ -739,8 +739,15 @@ namespace hpx::local::detail { auto const end = unregistered_options.end(); for (auto it = unregistered_options.begin(); it != end; ++it) { +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif unregistered_options_cmd_line += " " + detail::encode_and_enquote(*it); +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif } ini_config_.emplace_back("hpx.unknown_cmd_line!=" + diff --git a/libs/core/command_line_handling_local/src/late_command_line_handling_local.cpp b/libs/core/command_line_handling_local/src/late_command_line_handling_local.cpp index 2ce4f4eda96e..4e8e6401b0c8 100644 --- a/libs/core/command_line_handling_local/src/late_command_line_handling_local.cpp +++ b/libs/core/command_line_handling_local/src/late_command_line_handling_local.cpp @@ -22,7 +22,14 @@ namespace hpx::local::detail { { if (arg.find_first_of(" \t\"") != std::string::npos) { +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif return std::string("\"") + HPX_MOVE(arg) + "\""; +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif } return arg; } diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index 512b18339c24..16b61b4a79dc 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -106,7 +106,8 @@ namespace hpx::parcelset::policies::gasnet { buffer_.chunks_.resize(num_zero_copy_chunks); { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy(buffer_.transmission_chunks_.data(), + std::memcpy(static_cast( + buffer_.transmission_chunks_.data()), hpx::util::gasnet_environment::segments[self_].addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); diff --git a/libs/full/performance_counters/src/action_invocation_counter_discoverer.cpp b/libs/full/performance_counters/src/action_invocation_counter_discoverer.cpp index 83dbb364927b..19a84330dc73 100644 --- a/libs/full/performance_counters/src/action_invocation_counter_discoverer.cpp +++ b/libs/full/performance_counters/src/action_invocation_counter_discoverer.cpp @@ -62,7 +62,14 @@ namespace hpx { namespace performance_counters { return f(cinfo, ec) && !ec; } +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif p.parameters_ = "*"; +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif } if (p.parameters_.find_first_of("*?[]") != std::string::npos) diff --git a/libs/full/performance_counters/src/counters.cpp b/libs/full/performance_counters/src/counters.cpp index 3fff99df0552..2fefc351b9ab 100644 --- a/libs/full/performance_counters/src/counters.cpp +++ b/libs/full/performance_counters/src/counters.cpp @@ -108,6 +108,10 @@ namespace hpx::performance_counters { return counter_status::invalid_data; } +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif result = "/"; result += path.objectname_; @@ -164,6 +168,9 @@ namespace hpx::performance_counters { result += "@"; result += path.parameters_; } +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif if (&ec != &throws) ec = make_success_code(); @@ -182,7 +189,14 @@ namespace hpx::performance_counters { return counter_status::invalid_data; } +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif result = "/"; +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif result += path.objectname_; if (!path.countername_.empty()) @@ -209,7 +223,14 @@ namespace hpx::performance_counters { return counter_status::invalid_data; } +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif result = "/"; +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif result += path.objectname_; if (!path.countername_.empty()) @@ -247,6 +268,10 @@ namespace hpx::performance_counters { } else { +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wrestrict" +#endif result = "/"; result += path.parentinstancename_; if (path.parentinstanceindex_ == -1) @@ -290,6 +315,9 @@ namespace hpx::performance_counters { result += "#" + std::to_string(path.subinstanceindex_); } } +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 +#pragma GCC diagnostic pop +#endif } if (&ec != &throws) From e39ce07bf75e2c0d05f72cb10e459bcbe1f463d2 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Tue, 19 Sep 2023 20:34:53 +0000 Subject: [PATCH 060/333] fixes an outstanding implementation issue in receiver_connection and implements GASNet Fetch support in Cmake --- cmake/HPX_SetupGasnet.cmake | 72 ++++++++++++++++++- .../parcelport_gasnet/receiver_connection.hpp | 6 +- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 5b4e49d6a886..a648bd4a5f71 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -16,10 +16,80 @@ macro(hpx_setup_gasnet) "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" ) pkg_search_module( - GASNET REQUIRED IMPORTED_TARGET GLOBAL + GASNET IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par ) + if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) + + if(NOT CMAKE_C_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") + endif() + if(NOT CMAKE_CXX_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + + include(FetchContent) + FetchContent_Declare(gasnet + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz + ) + + fetchcontent_getproperties(gasnet) + if(NOT gasnet) + FetchContent_Populate(gasnet) + endif() + + message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") + + set(GASNET_DIR "${gasnet_SOURCE_DIR}") + set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") + set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + endif() + + if(GASNET_BUILD_STATUS) + message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") + endif() + + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) + message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") set(GASNET_MPI_FOUND TRUE) include(HPX_SetupMPI) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index 16b61b4a79dc..43935f378c84 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -106,8 +106,10 @@ namespace hpx::parcelset::policies::gasnet { buffer_.chunks_.resize(num_zero_copy_chunks); { hpx::util::gasnet_environment::scoped_lock l; - std::memcpy(static_cast( - buffer_.transmission_chunks_.data()), + std::pair * dataptr = + buffer_.transmission_chunks_.data(); + unsigned long elem[2] = { dataptr->first, dataptr->second }; + std::memcpy(elem, hpx::util::gasnet_environment::segments[self_].addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); From 2bdea4e3a994343d217399a784a863efae703344 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Wed, 20 Sep 2023 00:30:49 +0000 Subject: [PATCH 061/333] misread on previous receiver_connection modification - fixed potential bug --- .../hpx/parcelport_gasnet/receiver_connection.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index 43935f378c84..bdfad9f04e11 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -106,14 +106,15 @@ namespace hpx::parcelset::policies::gasnet { buffer_.chunks_.resize(num_zero_copy_chunks); { hpx::util::gasnet_environment::scoped_lock l; - std::pair * dataptr = - buffer_.transmission_chunks_.data(); - unsigned long elem[2] = { dataptr->first, dataptr->second }; + //std::pair * dataptr = + // buffer_.transmission_chunks_.data(); + unsigned long elem[2] = { 0, 0 }; std::memcpy(elem, hpx::util::gasnet_environment::segments[self_].addr, static_cast(buffer_.transmission_chunks_.size() * sizeof(buffer_type::transmission_chunk_type))); - + buffer_.transmission_chunks_.data()->first = elem[0]; + buffer_.transmission_chunks_.data()->second = elem[1]; request_ptr_ = true; } } From d41c539905d8bf9477004705d420d171cc382d24 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Wed, 20 Sep 2023 00:33:13 +0000 Subject: [PATCH 062/333] misread on previous receiver_connection modification - fixed potential bug --- .../include/hpx/parcelport_gasnet/receiver_connection.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index bdfad9f04e11..bc63472cad43 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -106,8 +106,6 @@ namespace hpx::parcelset::policies::gasnet { buffer_.chunks_.resize(num_zero_copy_chunks); { hpx::util::gasnet_environment::scoped_lock l; - //std::pair * dataptr = - // buffer_.transmission_chunks_.data(); unsigned long elem[2] = { 0, 0 }; std::memcpy(elem, hpx::util::gasnet_environment::segments[self_].addr, From 7010cc2a60abcbdd089cc207c0a81d03137ee88d Mon Sep 17 00:00:00 2001 From: ctaylor Date: Wed, 20 Sep 2023 01:52:47 +0000 Subject: [PATCH 063/333] added deletion of dynamic memory allocations during parcelport finalization --- libs/core/gasnet_base/src/gasnet_environment.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 3fb9e919a706..3717970a99dc 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -612,6 +612,8 @@ namespace hpx::util { if (enabled() && has_called_init()) { gasnet_exit(1); + delete gasnet_environment::segments; + delete gasnet_environment::segment_mutex; } } From 607f8b37e82fe94824a99d1d4bdd5108d4de29fb Mon Sep 17 00:00:00 2001 From: "Steven R. Brandt" Date: Thu, 20 Jul 2023 13:56:55 +0000 Subject: [PATCH 064/333] The hpxcxx script was broken such that it could only compile for _release. This modification makes it more robust in finding a compiled hpx installation and changes the 2 character flags to use -- instead of -. --- cmake/templates/hpxcxx.in | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/cmake/templates/hpxcxx.in b/cmake/templates/hpxcxx.in index ec1d9da2908a..100432454034 100755 --- a/cmake/templates/hpxcxx.in +++ b/cmake/templates/hpxcxx.in @@ -39,8 +39,8 @@ pkgconfpath += [ os.path.join("@HPX_CONF_PREFIX@","lib64","pkgconfig"), # install directory os.path.join(os.path.dirname(sys.argv[0]),"..","lib","pkgconfig"), os.path.join("opt","hpx","lib","pkgconfig"), - os.path.join("usr","bin","hpx","lib","pkgconfig"), - os.path.join("usr","local","bin","hpx","lib","pkgconfig"), + os.path.join("/usr","bin","hpx","lib","pkgconfig"), + os.path.join("/usr","local","bin","hpx","lib","pkgconfig"), os.path.join(os.environ["HOME"],"install","hpx","lib","pkgconfig"), os.path.join(os.environ["HOME"],"hpx","lib","pkgconfig") ] @@ -54,14 +54,14 @@ Usage: hpxcxx -c flags files The hpxcxx command requires that you build either a component, an application, or that you specify the -c flag. If you are building against a debug -build, you need to specify -g. -If release-with-debug-info, specify -rd -If minsize-release specify -mr. All other flags +build, you need to specify --db. +If release-with-debug-info, specify --rd +If minsize-release specify --mr. All other flags are passed through to the underlying C++ compiler. """) sys.exit(2) -pkgconf_suffix = ['_release', '_relwithdebuginfo', '_debug', '_minsizerel'] +pkgconf_suffix = '_release' i = 1 while i < len(sys.argv): @@ -76,6 +76,7 @@ while i < len(sys.argv): elif sys.argv[i].startswith('--exe='): output=sys.argv[i][6:] app = output + #args += ['-o',app+'.exe'] application = True elif sys.argv[i].startswith('--comp='): app_name = sys.argv[i][7:] @@ -93,15 +94,12 @@ while i < len(sys.argv): elif sys.argv[i] == '-c': minusc = True pass - elif sys.argv[i].startswith('-g'): - pkgconf_suffix = ['_debug'] - args += [sys.argv[i]] - elif sys.argv[i] == '--rd': - pkgconf_suffix = ['_relwithdebuginfo'] - elif sys.argv[i] == '--mr': - pkgconf_suffix = ['_minsizerel'] - elif sys.argv[i] == '-r': - pkgconf_suffix = ['_release'] + elif sys.argv[i].startswith('-db'): + pkgconf_suffix = '_debug' + elif sys.argv[i].startswith('--rd'): + pkgconf_suffix = '_relwithdebuginfo' + elif sys.argv[i].startswith('--mr'): + pkgconf_suffix = '_minsizerel' else: args += [sys.argv[i]] @@ -109,12 +107,11 @@ while i < len(sys.argv): pkgconf = None for path in pkgconfpath: - if pkgconf is not None: - break - for suffix in pkgconf_suffix: + for suffix in [pkgconf_suffix, "_release", "_relwithdebuginfo","_debug"]: hpath = os.path.join(path,"hpx_application")+suffix+".pc" if os.path.exists(hpath): pkgconf = path + pkgconf_suffix = suffix break if pkgconf == None: @@ -128,11 +125,11 @@ else: os.environ[pkg] = pkgconf if application: - args += ["`pkg-config --cflags --libs hpx_application" + suffix + "`"] + args += ["`pkg-config --cflags --libs hpx_application" + pkgconf_suffix + "`"] elif component: - args += ["`pkg-config --cflags --libs hpx_component" + suffix + "`"] + args += ["`pkg-config --cflags --libs hpx_component" + pkgconf_suffix + "`"] else: - args += ["`pkg-config --cflags hpx_application" + suffix + "`"] + args += ["`pkg-config --cflags hpx_application" + pkgconf_suffix + "`"] if not component and not application and not minusc: usage() From db92288cbb3f139405af3b18c78b1032129d29f4 Mon Sep 17 00:00:00 2001 From: "Steven R. Brandt" Date: Tue, 25 Jul 2023 13:30:42 +0000 Subject: [PATCH 065/333] Add minsizerel --- cmake/templates/hpxcxx.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/templates/hpxcxx.in b/cmake/templates/hpxcxx.in index 100432454034..8e2385265784 100755 --- a/cmake/templates/hpxcxx.in +++ b/cmake/templates/hpxcxx.in @@ -107,7 +107,7 @@ while i < len(sys.argv): pkgconf = None for path in pkgconfpath: - for suffix in [pkgconf_suffix, "_release", "_relwithdebuginfo","_debug"]: + for suffix in [pkgconf_suffix, "_release", "_relwithdebuginfo","_debug","_minsizerel"]: hpath = os.path.join(path,"hpx_application")+suffix+".pc" if os.path.exists(hpath): pkgconf = path From f641ede756f9b9ed43c7e20c792b97e9c374a1e4 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Wed, 26 Jul 2023 11:16:59 -0500 Subject: [PATCH 066/333] Fixing warning/error --- libs/core/async_cuda/tests/unit/cuda_multi_device_polling.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/async_cuda/tests/unit/cuda_multi_device_polling.cpp b/libs/core/async_cuda/tests/unit/cuda_multi_device_polling.cpp index ca1ee5579c7f..c5d6a4caa1b4 100644 --- a/libs/core/async_cuda/tests/unit/cuda_multi_device_polling.cpp +++ b/libs/core/async_cuda/tests/unit/cuda_multi_device_polling.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Gregor Daiß +// Copyright (c) 2023 Gregor Daiß // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -39,7 +39,7 @@ extern void launch_saxpy_kernel( unsigned int& threads, void** args); // ------------------------------------------------------------------------- -int hpx_main(hpx::program_options::variables_map& vm) +int hpx_main(hpx::program_options::variables_map&) { // install cuda future polling handler hpx::cuda::experimental::enable_user_polling poll("default"); From f13d2fdb20894e60526ceb0d58129405135582eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gregor=20Dai=C3=9F?= Date: Tue, 25 Jul 2023 18:31:27 -0500 Subject: [PATCH 067/333] Define HPX_COMPUTE_CODE in builds with SYCL This resolves a compilation issue where timestamp_cuda is required for a certain build configuration (using ONEAPI/SYCL/dpcpp with an enabled CUDA backend and specific GPU architecture) but not found. The underlying issue is that the include for the timestamp_cuda header is protected with a ifdef HPX_COMPUTE_CODE and the header was thus not included - hence the timestamp method was not found and the code did not compile in this configuration! This commit resolves the issue! Not by changing or removing the ifdef timestamp guard, but instead by defining HPX_COMPUTE_CODE for SYCL builds in the compiler_specific header. This ensures that the behaviour for non-SYCL builds stays the same as before! --- libs/core/config/include/hpx/config/compiler_specific.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/core/config/include/hpx/config/compiler_specific.hpp b/libs/core/config/include/hpx/config/compiler_specific.hpp index 49be3a20407d..d50189cf24bb 100644 --- a/libs/core/config/include/hpx/config/compiler_specific.hpp +++ b/libs/core/config/include/hpx/config/compiler_specific.hpp @@ -160,6 +160,11 @@ #endif #define HPX_HOST_DEVICE HPX_HOST HPX_DEVICE +// Define this AFTER HPX_[DEVICE|HOST] for SYCL +// as we do not want the __device__ modifiers +#if defined(HPX_HAVE_SYCL) +# define HPX_COMPUTE_CODE +#endif #if !defined(HPX_CDECL) #define HPX_CDECL From a9aae8378a0f7889a55a8c7b0a8cbf801b0798f4 Mon Sep 17 00:00:00 2001 From: Panos Date: Tue, 11 Jul 2023 18:38:53 +0300 Subject: [PATCH 068/333] Add internal reference counting to semaphores. --- .../synchronization/counting_semaphore.hpp | 51 ++++++++++++------- .../detail/counting_semaphore.hpp | 31 +++++++++++ .../detail/sliding_semaphore.hpp | 32 ++++++++++++ .../hpx/synchronization/sliding_semaphore.hpp | 38 +++++++++----- 4 files changed, 120 insertions(+), 32 deletions(-) diff --git a/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp index fbfa9defb66b..e957cf663283 100644 --- a/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp @@ -10,6 +10,7 @@ #pragma once #include +#include #include #include #include @@ -350,6 +351,8 @@ namespace hpx { protected: using mutex_type = Mutex; + using data_type = + lcos::local::detail::counting_semaphore_data; public: static constexpr std::ptrdiff_t(max)() noexcept @@ -358,7 +361,7 @@ namespace hpx { } explicit counting_semaphore(std::ptrdiff_t value) noexcept - : sem_(value) + : data_(new data_type(value), false) { } @@ -366,27 +369,31 @@ namespace hpx { void release(std::ptrdiff_t update = 1) { - std::unique_lock l(mtx_); - sem_.signal(HPX_MOVE(l), update); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.signal(HPX_MOVE(l), update); } bool try_acquire() noexcept { - std::unique_lock l(mtx_); - return sem_.try_acquire(l); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + return data->sem_.try_acquire(l); } void acquire() { - std::unique_lock l(mtx_); - sem_.wait(l, 1); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.wait(l, 1); } bool try_acquire_until( hpx::chrono::steady_time_point const& abs_time) { - std::unique_lock l(mtx_); - return sem_.wait_until(l, abs_time, 1); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + return data->sem_.wait_until(l, abs_time, 1); } bool try_acquire_for(hpx::chrono::steady_duration const& rel_time) @@ -395,8 +402,7 @@ namespace hpx { } protected: - mutable mutex_type mtx_; - hpx::lcos::local::detail::counting_semaphore sem_; + hpx::intrusive_ptr data_; }; } // namespace detail @@ -410,6 +416,7 @@ namespace hpx { { private: using mutex_type = Mutex; + using detail::counting_semaphore::data_; public: explicit counting_semaphore_var(std::ptrdiff_t value = N) noexcept @@ -420,29 +427,35 @@ namespace hpx { counting_semaphore_var(counting_semaphore_var const&) = delete; counting_semaphore_var& operator=( counting_semaphore_var const&) = delete; + counting_semaphore_var(counting_semaphore_var&&) = delete; + counting_semaphore_var& operator=(counting_semaphore_var&&) = delete; void wait(std::ptrdiff_t count = 1) { - std::unique_lock l(this->mtx_); - this->sem_.wait(l, count); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.wait(l, count); } bool try_wait(std::ptrdiff_t count = 1) { - std::unique_lock l(this->mtx_); - return this->sem_.try_wait(l, count); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + return data->sem_.try_wait(l, count); } void signal(std::ptrdiff_t count = 1) { - std::unique_lock l(this->mtx_); - this->sem_.signal(HPX_MOVE(l), count); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.signal(HPX_MOVE(l), count); } std::ptrdiff_t signal_all() { - std::unique_lock l(this->mtx_); - return this->sem_.signal_all(HPX_MOVE(l)); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + return data->sem_.signal_all(HPX_MOVE(l)); } }; } // namespace hpx diff --git a/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp index a70a8d00c22c..2c7857ec6c94 100644 --- a/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp @@ -55,6 +55,37 @@ namespace hpx::lcos::local::detail { std::ptrdiff_t value_; local::detail::condition_variable cond_; }; + + template + struct counting_semaphore_data + { + counting_semaphore_data(std::ptrdiff_t value) noexcept + : sem_(value) + , count_(1) + { + } + + mutable Mutex mtx_; + detail::counting_semaphore sem_; + + private: + friend void intrusive_ptr_add_ref( + counting_semaphore_data* p) noexcept + { + ++p->count_; + } + + friend void intrusive_ptr_release( + counting_semaphore_data* p) noexcept + { + if (0 == --p->count_) + { + delete p; + } + } + + hpx::util::atomic_count count_; + }; } // namespace hpx::lcos::local::detail #if defined(HPX_MSVC_WARNING_PRAGMA) diff --git a/libs/core/synchronization/include/hpx/synchronization/detail/sliding_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/detail/sliding_semaphore.hpp index 6560f62cee5e..e0d9ad708aa3 100644 --- a/libs/core/synchronization/include/hpx/synchronization/detail/sliding_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/detail/sliding_semaphore.hpp @@ -51,6 +51,38 @@ namespace hpx::lcos::local::detail { std::int64_t lower_limit_; local::detail::condition_variable cond_; }; + + template + struct sliding_semaphore_data + { + sliding_semaphore_data( + std::int64_t max_difference, std::int64_t lower_limit) noexcept + : sem_(max_difference, lower_limit) + , count_(1) + { + } + + mutable Mutex mtx_; + detail::sliding_semaphore sem_; + + private: + friend void intrusive_ptr_add_ref( + sliding_semaphore_data* p) noexcept + { + ++p->count_; + } + + friend void intrusive_ptr_release( + sliding_semaphore_data* p) noexcept + { + if (0 == --p->count_) + { + delete p; + } + } + + hpx::util::atomic_count count_; + }; } // namespace hpx::lcos::local::detail #if defined(HPX_MSVC_WARNING_PRAGMA) diff --git a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp index 7f05e90b88d5..55a5c17e3a37 100644 --- a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include @@ -43,8 +44,15 @@ namespace hpx { { private: using mutex_type = Mutex; + using data_type = + lcos::local::detail::sliding_semaphore_data; public: + sliding_semaphore_var(sliding_semaphore_var const&) = delete; + sliding_semaphore_var& operator=(sliding_semaphore_var const&) = delete; + sliding_semaphore_var(sliding_semaphore_var&&) = delete; + sliding_semaphore_var& operator=(sliding_semaphore_var&&) = delete; + /// \brief Construct a new sliding semaphore /// /// \param max_difference @@ -55,7 +63,7 @@ namespace hpx { /// \param lower_limit [in] The initial lower limit. explicit sliding_semaphore_var( std::int64_t max_difference, std::int64_t lower_limit = 0) noexcept - : sem_(max_difference, lower_limit) + : data_(new data_type(max_difference, lower_limit), false) { } @@ -70,8 +78,9 @@ namespace hpx { void set_max_difference( std::int64_t max_difference, std::int64_t lower_limit = 0) noexcept { - std::unique_lock l(mtx_); - sem_.set_max_difference(l, max_difference, lower_limit); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.set_max_difference(l, max_difference, lower_limit); } /// \brief Wait for the semaphore to be signaled @@ -82,8 +91,9 @@ namespace hpx { /// set by signal() is larger than the max_difference. void wait(std::int64_t upper_limit) { - std::unique_lock l(mtx_); - sem_.wait(l, upper_limit); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.wait(l, upper_limit); } /// \brief Try to wait for the semaphore to be signaled @@ -97,8 +107,9 @@ namespace hpx { /// would not block if it was calling wait(). bool try_wait(std::int64_t upper_limit = 1) { - std::unique_lock l(mtx_); - return sem_.try_wait(l, upper_limit); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + return data->sem_.try_wait(l, upper_limit); } /// \brief Signal the semaphore @@ -110,19 +121,20 @@ namespace hpx { /// limit plus the max_difference. void signal(std::int64_t lower_limit) { - std::unique_lock l(mtx_); - sem_.signal(HPX_MOVE(l), lower_limit); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + data->sem_.signal(HPX_MOVE(l), lower_limit); } std::int64_t signal_all() { - std::unique_lock l(mtx_); - return sem_.signal_all(HPX_MOVE(l)); + auto data = data_; //keep alive + std::unique_lock l(data->mtx_); + return data->sem_.signal_all(HPX_MOVE(l)); } private: - mutable mutex_type mtx_; - lcos::local::detail::sliding_semaphore sem_; + hpx::intrusive_ptr data_; }; using sliding_semaphore = sliding_semaphore_var<>; From 0a699f6dba6b2b32b27fa5f274b25d3ff72027e8 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Tue, 25 Jul 2023 16:05:04 -0500 Subject: [PATCH 069/333] Fix clang format & inspect issues --- .../include/hpx/synchronization/detail/counting_semaphore.hpp | 2 +- .../include/hpx/synchronization/sliding_semaphore.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp index 2c7857ec6c94..852b45f4abdb 100644 --- a/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/detail/counting_semaphore.hpp @@ -59,7 +59,7 @@ namespace hpx::lcos::local::detail { template struct counting_semaphore_data { - counting_semaphore_data(std::ptrdiff_t value) noexcept + explicit counting_semaphore_data(std::ptrdiff_t value) noexcept : sem_(value) , count_(1) { diff --git a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp index 55a5c17e3a37..f82c2b2f6292 100644 --- a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp @@ -78,7 +78,7 @@ namespace hpx { void set_max_difference( std::int64_t max_difference, std::int64_t lower_limit = 0) noexcept { - auto data = data_; //keep alive + auto data = data_; //keep alive std::unique_lock l(data->mtx_); data->sem_.set_max_difference(l, max_difference, lower_limit); } From 46d039d3f6789b2fd52566ca460a820296af4829 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 27 Jul 2023 18:37:05 -0500 Subject: [PATCH 070/333] Fixing inspect errors --- .../include/hpx/synchronization/counting_semaphore.hpp | 2 +- .../include/hpx/synchronization/sliding_semaphore.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp index e957cf663283..8a09113a611f 100644 --- a/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp @@ -10,7 +10,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp index f82c2b2f6292..6bcfb3e24b44 100644 --- a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp @@ -7,7 +7,7 @@ #pragma once #include -#include +#include #include #include From 6f1837b4255928fffe94ddf0caa44691d6a3bcb1 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 25 Jul 2023 20:14:54 -0500 Subject: [PATCH 071/333] Adding executor parallel_invoke CPO --- .../include/hpx/parallel/util/partitioner.hpp | 4 +- .../host/block_fork_join_executor.hpp | 124 ++++++-- .../tests/unit/block_fork_join_executor.cpp | 136 +++++++++ .../hpx/execution/executors/execution.hpp | 73 ++++- .../include/hpx/execution_base/execution.hpp | 89 +++++- .../hpx/executors/fork_join_executor.hpp | 266 +++++++++++++++--- .../tests/unit/fork_join_executor.cpp | 162 ++++++++++- libs/core/functional/CMakeLists.txt | 1 + .../hpx/functional/detail/runtime_get.hpp | 98 +++++++ 9 files changed, 879 insertions(+), 74 deletions(-) create mode 100644 libs/core/functional/include/hpx/functional/detail/runtime_get.hpp diff --git a/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp b/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp index 19b6d0bb549d..2decc47e4f19 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp @@ -300,7 +300,7 @@ namespace hpx::parallel::util::detail { // wait for all tasks to finish if (hpx::wait_all_nothrow(items)) { - // always rethrow workitems has at least one exceptional + // always rethrow if items has at least one exceptional // future handle_local_exceptions::call(items); } @@ -324,7 +324,7 @@ namespace hpx::parallel::util::detail { // wait for all tasks to finish if (hpx::wait_all_nothrow(items)) { - // always rethrow workitems has at least one exceptional + // always rethrow if items has at least one exceptional // future handle_local_exceptions::call(items); } diff --git a/libs/core/compute_local/include/hpx/compute_local/host/block_fork_join_executor.hpp b/libs/core/compute_local/include/hpx/compute_local/host/block_fork_join_executor.hpp index b82305772143..1c08ebe61ed0 100644 --- a/libs/core/compute_local/include/hpx/compute_local/host/block_fork_join_executor.hpp +++ b/libs/core/compute_local/include/hpx/compute_local/host/block_fork_join_executor.hpp @@ -59,8 +59,9 @@ namespace hpx::execution::experimental { static hpx::threads::mask_type cores_for_targets( std::vector const& targets) { - auto& rp = hpx::resource::get_partitioner(); - std::size_t this_pu = rp.get_pu_num(hpx::get_worker_thread_num()); + auto const& rp = hpx::resource::get_partitioner(); + std::size_t const this_pu = + rp.get_pu_num(hpx::get_worker_thread_num()); if (targets.size() == 1) { // don't build a hierarchy of executors if there is only one @@ -141,9 +142,8 @@ namespace hpx::execution::experimental { /// \param stacksize The stacksize of the worker threads. Must not be /// nostack. /// \param schedule The loop schedule of the parallel regions. - /// \param yield_delay The time after which the executor yields to - /// other work if it hasn't received any new work for bulk - /// execution. + /// \param yield_delay The time after which the executor yields to other + /// work if it has not received any new work for execution. /// /// \note This constructor will create one fork_join_executor for /// each numa domain @@ -151,7 +151,7 @@ namespace hpx::execution::experimental { threads::thread_priority priority = threads::thread_priority::bound, threads::thread_stacksize stacksize = threads::thread_stacksize::small_, - fork_join_executor::loop_schedule schedule = + fork_join_executor::loop_schedule const schedule = fork_join_executor::loop_schedule::static_, std::chrono::nanoseconds yield_delay = std::chrono::milliseconds(1)) : block_fork_join_executor(compute::host::numa_domains(), priority, @@ -166,9 +166,8 @@ namespace hpx::execution::experimental { /// \param stacksize The stacksize of the worker threads. Must not be /// nostack. /// \param schedule The loop schedule of the parallel regions. - /// \param yield_delay The time after which the executor yields to - /// other work if it hasn't received any new work for bulk - /// execution. + /// \param yield_delay The time after which the executor yields to other + /// work if it has not received any new work for execution. /// /// \note This constructor will create one fork_join_executor for /// each given target @@ -177,7 +176,7 @@ namespace hpx::execution::experimental { threads::thread_priority priority = threads::thread_priority::bound, threads::thread_stacksize stacksize = threads::thread_stacksize::small_, - fork_join_executor::loop_schedule schedule = + fork_join_executor::loop_schedule const schedule = fork_join_executor::loop_schedule::static_, std::chrono::nanoseconds yield_delay = std::chrono::milliseconds(1)) : exec_(cores_for_targets(targets), priority, stacksize, @@ -212,7 +211,7 @@ namespace hpx::execution::experimental { template void bulk_sync_execute_helper(F&& f, S const& shape, Ts&&... ts) { - std::size_t num_targets = block_execs_.size(); + std::size_t const num_targets = block_execs_.size(); if (num_targets == 0) { // simply forward call if there is no executor hierarchy @@ -221,21 +220,21 @@ namespace hpx::execution::experimental { return; } - std::size_t size = std::size(shape); - auto outer_func = [&](std::size_t index, auto&& f, - auto const& shape, auto&&... ts) { - // calculate the inner shape dimensions + std::size_t const size = std::size(shape); + auto outer_func = [&](std::size_t index, auto&& func, + auto const& full_shape, auto&&... args) { + // calculate the inner inner_shape dimensions auto const part_begin = (index * size) / num_targets; auto const part_end = ((index + 1) * size) / num_targets; - auto begin = std::next(std::begin(shape), part_begin); + auto begin = std::next(std::begin(full_shape), part_begin); auto inner_shape = hpx::util::iterator_range( begin, std::next(begin, part_end - part_begin)); // invoke bulk_sync_execute on one of the inner executors hpx::parallel::execution::bulk_sync_execute(block_execs_[index], - HPX_FORWARD(decltype(f), f), inner_shape, - HPX_FORWARD(decltype(ts), ts)...); + HPX_FORWARD(decltype(func), func), inner_shape, + HPX_FORWARD(decltype(args), args)...); }; auto outer_shape = hpx::util::counting_shape(num_targets); @@ -244,7 +243,12 @@ namespace hpx::execution::experimental { outer_shape, HPX_FORWARD(F, f), shape, HPX_FORWARD(Ts, ts)...); } - template + // clang-format off + template + )> + // clang-format on friend void tag_invoke(hpx::parallel::execution::bulk_sync_execute_t, block_fork_join_executor& exec, F&& f, S const& shape, Ts&&... ts) { @@ -252,18 +256,94 @@ namespace hpx::execution::experimental { HPX_FORWARD(F, f), shape, HPX_FORWARD(Ts, ts)...); } - template + // clang-format off + template + )> + // clang-format on friend decltype(auto) tag_invoke( hpx::parallel::execution::bulk_async_execute_t, block_fork_join_executor& exec, F&& f, S const& shape, Ts&&... ts) + { + // Forward to the synchronous version as we can't create futures to + // the completion of the parallel region (this HPX thread + // participates in computation). + return hpx::detail::try_catch_exception_ptr( + [&]() { + exec.bulk_sync_execute_helper( + HPX_FORWARD(F, f), shape, HPX_FORWARD(Ts, ts)...); + return hpx::make_ready_future(); + }, + [&](std::exception_ptr&& ep) { + return hpx::make_exceptional_future(HPX_MOVE(ep)); + }); + } + + template + void sync_invoke_helper(Fs&&... fs) const + { + std::size_t const num_targets = block_execs_.size(); + if (num_targets == 0 || sizeof...(Fs) == 1) + { + // simply forward call if there is no executor hierarchy + hpx::parallel::execution::sync_invoke( + exec_, HPX_FORWARD(Fs, fs)...); + return; + } + + if constexpr (sizeof...(Fs) > 1) + { + auto function_pack = + hpx::forward_as_tuple(HPX_FORWARD(Fs, fs)...); + + constexpr std::size_t Size = sizeof...(Fs); + auto outer_func = [&](std::size_t index) { + auto const part_begin = (index * Size) / num_targets; + auto const part_end = ((index + 1) * Size) / num_targets; + + // invoke sync_invoke on one of the inner executors + block_execs_[index].sync_invoke_helper( + function_pack, part_begin, part_end - part_begin); + }; + + auto outer_shape = hpx::util::counting_shape(num_targets); + + hpx::parallel::execution::bulk_sync_execute( + exec_, outer_func, outer_shape); + } + } + + // clang-format off + template && (std::is_invocable_v && ...) + )> + // clang-format on + friend decltype(auto) tag_invoke( + hpx::parallel::execution::sync_invoke_t, + block_fork_join_executor const& exec, F&& f, Fs&&... fs) + { + exec.sync_invoke_helper(HPX_FORWARD(F, f), HPX_FORWARD(Fs, fs)...); + } + + // clang-format off + template && (std::is_invocable_v && ...) + )> + // clang-format on + friend decltype(auto) tag_invoke( + hpx::parallel::execution::async_invoke_t, + block_fork_join_executor const& exec, F&& f, Fs&&... fs) { // Forward to the synchronous version as we can't create // futures to the completion of the parallel region (this HPX // thread participates in computation). return hpx::detail::try_catch_exception_ptr( [&]() { - exec.bulk_sync_execute_helper( - HPX_FORWARD(F, f), shape, HPX_FORWARD(Ts, ts)...); + exec.sync_invoke_helper( + HPX_FORWARD(F, f), HPX_FORWARD(Fs, fs)...); return hpx::make_ready_future(); }, [&](std::exception_ptr&& ep) { diff --git a/libs/core/compute_local/tests/unit/block_fork_join_executor.cpp b/libs/core/compute_local/tests/unit/block_fork_join_executor.cpp index 3d97c68369dc..a6bce02e3511 100644 --- a/libs/core/compute_local/tests/unit/block_fork_join_executor.cpp +++ b/libs/core/compute_local/tests/unit/block_fork_join_executor.cpp @@ -153,6 +153,137 @@ void test_bulk_async_exception(ExecutorArgs&&... args) HPX_TEST(caught_exception); } +static std::atomic count1{0}; +static std::atomic count2{0}; +static std::atomic count3{0}; +static std::atomic count4{0}; + +template +void test_invoke_sync_homogeneous(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync_homogeneous\n"; + + auto f1 = [] { ++count1; }; + + block_fork_join_executor exec{std::forward(args)...}; + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1); + HPX_TEST_EQ(count1.load(), static_cast(1)); + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f1); + HPX_TEST_EQ(count1.load(), static_cast(2)); + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f1, f1, f1, f1); + HPX_TEST_EQ(count1.load(), static_cast(5)); + + count1 = 0; + hpx::parallel::execution::sync_invoke( + exec, f1, f1, f1, f1, f1, f1, f1, f1, f1, f1, f1); + HPX_TEST_EQ(count1.load(), static_cast(11)); +} + +template +void test_invoke_sync(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync\n"; + + auto f1 = [] { ++count1; }; + auto f2 = [] { ++count2; }; + auto f3 = [] { ++count3; }; + auto f4 = [] { ++count4; }; + + block_fork_join_executor exec{std::forward(args)...}; + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1); + HPX_TEST_EQ(count1.load(), static_cast(1)); + + count1 = 0; + count2 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f2); + HPX_TEST_EQ(count1.load(), static_cast(1)); + HPX_TEST_EQ(count2.load(), static_cast(1)); + + count1 = 0; + count2 = 0; + count3 = 0; + count4 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f2, f3, f4, f1); + HPX_TEST_EQ(count1.load(), static_cast(2)); + HPX_TEST_EQ(count2.load(), static_cast(1)); + HPX_TEST_EQ(count3.load(), static_cast(1)); + HPX_TEST_EQ(count4.load(), static_cast(1)); + + count1 = 0; + count2 = 0; + count3 = 0; + count4 = 0; + hpx::parallel::execution::sync_invoke( + exec, f1, f2, f3, f4, f1, f4, f1, f2, f3, f4, f1); + HPX_TEST_EQ(count1.load(), static_cast(4)); + HPX_TEST_EQ(count2.load(), static_cast(2)); + HPX_TEST_EQ(count3.load(), static_cast(2)); + HPX_TEST_EQ(count4.load(), static_cast(3)); +} + +template +void test_invoke_sync_homogeneous_exception(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync_homogeneous_exception\n"; + + auto f1 = [] { throw std::runtime_error("test"); }; + + block_fork_join_executor exec{std::forward(args)...}; + + bool caught_exception = false; + try + { + hpx::parallel::execution::sync_invoke(exec, f1, f1, f1); + HPX_TEST(false); + } + catch (std::runtime_error const& /*e*/) + { + caught_exception = true; + } + catch (...) + { + HPX_TEST(false); + } + + HPX_TEST(caught_exception); +} + +template +void test_invoke_sync_exception(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync_exception\n"; + + auto f1 = [] {}; + auto f2 = [] { throw std::runtime_error("test"); }; + + block_fork_join_executor exec{std::forward(args)...}; + + bool caught_exception = false; + try + { + hpx::parallel::execution::sync_invoke(exec, f1, f2); + HPX_TEST(false); + } + catch (std::runtime_error const& /*e*/) + { + caught_exception = true; + } + catch (...) + { + HPX_TEST(false); + } + + HPX_TEST(caught_exception); +} + template void test_executor(hpx::threads::thread_priority priority, hpx::threads::thread_stacksize stacksize, @@ -165,6 +296,11 @@ void test_executor(hpx::threads::thread_priority priority, test_bulk_async(priority, stacksize, schedule); test_bulk_sync_exception(priority, stacksize, schedule); test_bulk_async_exception(priority, stacksize, schedule); + + test_invoke_sync_homogeneous(priority, stacksize, schedule); + test_invoke_sync(priority, stacksize, schedule); + test_invoke_sync_homogeneous_exception(priority, stacksize, schedule); + test_invoke_sync_exception(priority, stacksize, schedule); } /////////////////////////////////////////////////////////////////////////////// diff --git a/libs/core/execution/include/hpx/execution/executors/execution.hpp b/libs/core/execution/include/hpx/execution/executors/execution.hpp index a6d008dfb525..d5637e4ef656 100644 --- a/libs/core/execution/include/hpx/execution/executors/execution.hpp +++ b/libs/core/execution/include/hpx/execution/executors/execution.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Hartmut Kaiser +// Copyright (c) 2017-2023 Hartmut Kaiser // Copyright (c) 2017 Google // // SPDX-License-Identifier: BSL-1.0 @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -1423,4 +1424,74 @@ namespace hpx::parallel::execution { }; } // namespace detail /// \endcond + + /// \cond NOINTERNAL + namespace detail { + /////////////////////////////////////////////////////////////////////// + // async_invoke() + + template + struct async_invoke_fn_helper + { + template + HPX_FORCEINLINE static hpx::future call( + Executor_&& exec, Fs&&... fs) + { + std::vector> + results; + results.reserve(sizeof...(Fs)); + + (results.push_back(hpx::parallel::execution::async_execute( + exec, HPX_FORWARD(Fs, fs))), + ...); + + return hpx::when_all(HPX_MOVE(results)); + } + + template + struct result + { + // clang-format off + using type = decltype(call( + std::declval(), std::declval()...)); + // clang-format on + }; + }; + } // namespace detail + /// \endcond + + /// \cond NOINTERNAL + namespace detail { + /////////////////////////////////////////////////////////////////////// + // sync_invoke() + + template + struct sync_invoke_fn_helper + { + template + HPX_FORCEINLINE static auto call(Executor_&& exec, Fs&&... fs) + { + std::vector> + results; + results.reserve(sizeof...(Fs)); + + (results.push_back(hpx::parallel::execution::async_execute( + exec, HPX_FORWARD(Fs, fs))), + ...); + + hpx::wait_all(HPX_MOVE(results)); + } + + template + struct result + { + // clang-format off + using type = decltype(call( + std::declval(), std::declval()...)); + // clang-format on + }; + }; + } // namespace detail + /// \endcond + } // namespace hpx::parallel::execution diff --git a/libs/core/execution_base/include/hpx/execution_base/execution.hpp b/libs/core/execution_base/include/hpx/execution_base/execution.hpp index 33e5ef3d79d8..943331c0ccd3 100644 --- a/libs/core/execution_base/include/hpx/execution_base/execution.hpp +++ b/libs/core/execution_base/include/hpx/execution_base/execution.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Hartmut Kaiser +// Copyright (c) 2017-2023 Hartmut Kaiser // Copyright (c) 2017 Google // // SPDX-License-Identifier: BSL-1.0 @@ -74,6 +74,12 @@ namespace hpx::parallel::execution { template struct bulk_then_execute_fn_helper; + + template + struct async_invoke_fn_helper; + + template + struct sync_invoke_fn_helper; /// \endcond } // namespace detail @@ -466,4 +472,85 @@ namespace hpx::parallel::execution { HPX_FORWARD(Future, predecessor), HPX_FORWARD(Ts, ts)...); } } bulk_then_execute{}; + + /// Asynchronously invoke the given set of nullary functions, each on its + /// own execution agent + /// + /// This creates a group of function invocations whose ordering is given by + /// the execution_category associated with the executor. + /// + /// All exceptions thrown by invocations of the functions are reported in a + /// manner consistent with parallel algorithm execution through the returned + /// future. + /// + /// \param exec [in] The executor object to use for scheduling of the + /// functions \a fs. + /// \param fs [in] The functions which will be scheduled using the + /// given executor. + /// + /// \returns The return type of \a executor_type::async_invoke if defined by + /// \a executor_type. Otherwise a future + /// representing finishing the execution of all functions \a fs. + /// + /// \note This calls exec.async_invoke(fs...) if it exists; otherwise it + /// executes async_execute(fs) for each fs. + /// + inline constexpr struct async_invoke_t final + : hpx::functional::detail::tag_fallback + { + private: + // clang-format off + template && (std::is_invocable_v && ...) + )> + // clang-format on + friend HPX_FORCEINLINE decltype(auto) tag_fallback_invoke( + async_invoke_t, Executor&& exec, F&& f, Fs&&... fs) + { + return detail::async_invoke_fn_helper>::call( + HPX_FORWARD(Executor, exec), HPX_FORWARD(F, f), + HPX_FORWARD(Fs, fs)...); + } + } async_invoke{}; + + /// Synchronously invoke the given set of nullary functions, each on its own + /// execution agent + /// + /// This creates a group of function invocations whose ordering is given by + /// the execution_category associated with the executor. + /// + /// All exceptions thrown by invocations of the functions are reported in a + /// manner consistent with parallel algorithm execution through the returned + /// future. + /// + /// \param exec [in] The executor object to use for scheduling of the + /// functions \a fs. + /// \param fs [in] The functions which will be scheduled using the + /// given executor. + /// + /// \returns The return type of \a executor_type::async_invoke if defined by + /// \a executor_type. + /// + /// \note This calls exec.sync_invoke(fs...) if it exists; otherwise it + /// executes sync_execute(fs) for each fs. + /// + inline constexpr struct sync_invoke_t final + : hpx::functional::detail::tag_fallback + { + private: + // clang-format off + template && (std::is_invocable_v && ...) + )> + // clang-format on + friend HPX_FORCEINLINE decltype(auto) tag_fallback_invoke( + sync_invoke_t, Executor&& exec, F&& f, Fs&&... fs) + { + return detail::sync_invoke_fn_helper>::call( + HPX_FORWARD(Executor, exec), HPX_FORWARD(F, f), + HPX_FORWARD(Fs, fs)...); + } + } sync_invoke{}; } // namespace hpx::parallel::execution diff --git a/libs/core/executors/include/hpx/executors/fork_join_executor.hpp b/libs/core/executors/include/hpx/executors/fork_join_executor.hpp index 238988ba9d90..381fa2b76084 100644 --- a/libs/core/executors/include/hpx/executors/fork_join_executor.hpp +++ b/libs/core/executors/include/hpx/executors/fork_join_executor.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -169,7 +170,7 @@ namespace hpx::execution::experimental { { HPX_SMT_PAUSE; - std::uint64_t base_time = util::hardware::timestamp(); + std::uint64_t const base_time = util::hardware::timestamp(); current = tstate.load(std::memory_order_acquire); while (HPX_LIKELY(op(current, state))) { @@ -219,7 +220,7 @@ namespace hpx::execution::experimental { region_data_type& region_data_; queues_type& queues_; - void set_state_this_thread(thread_state state) noexcept + void set_state_this_thread(thread_state state) const noexcept { region_data_[thread_index_].data_.state_.store( state, std::memory_order_release); @@ -231,13 +232,13 @@ namespace hpx::execution::experimental { std::memory_order_relaxed); } - void operator()() noexcept + void operator()() const noexcept { HPX_ASSERT( get_state_this_thread() == thread_state::starting); set_state_this_thread(thread_state::idle); - region_data& data = region_data_[thread_index_].data_; + region_data const& data = region_data_[thread_index_].data_; // wait as long the state is 'idle' auto state = shared_data::wait_state_this_thread_while( @@ -320,7 +321,7 @@ namespace hpx::execution::experimental { std::size_t t = 0; bool main_thread_ok = false; - auto& rp = hpx::resource::get_partitioner(); + auto const& rp = hpx::resource::get_partitioner(); std::size_t main_pu_num = rp.get_pu_num(main_thread_); if (!hpx::threads::test(pu_mask_, main_pu_num) || num_threads_ == 1) @@ -333,12 +334,12 @@ namespace hpx::execution::experimental { if (num_threads_ > 1) { - std::size_t num_pus = pool_->get_os_thread_count(); + std::size_t const num_pus = pool_->get_os_thread_count(); for (std::size_t pu = 0; t != num_threads_ && pu != num_pus; ++pu) { - std::size_t pu_num = rp.get_pu_num(pu); + std::size_t const pu_num = rp.get_pu_num(pu); if (!main_thread_ok && pu == main_thread_) { // the initializing thread is expected to @@ -408,7 +409,7 @@ namespace hpx::execution::experimental { static hpx::threads::mask_type full_mask(std::size_t num_threads) { - auto& rp = hpx::resource::get_partitioner(); + auto const& rp = hpx::resource::get_partitioner(); hpx::threads::mask_type mask( hpx::threads::hardware_concurrency()); @@ -428,12 +429,10 @@ namespace hpx::execution::experimental { , priority_(priority) , stacksize_(stacksize) , schedule_(schedule) - , yield_delay_(std::uint64_t( + , yield_delay_(static_cast( yield_delay.count() / pool_->timestamp_scale())) , num_threads_(pool_->get_os_thread_count()) , pu_mask_(full_mask(num_threads_)) - , exception_mutex_() - , exception_() , region_data_(num_threads_) { HPX_ASSERT(pool_); @@ -449,12 +448,10 @@ namespace hpx::execution::experimental { , priority_(priority) , stacksize_(stacksize) , schedule_(schedule) - , yield_delay_(std::uint64_t( + , yield_delay_(static_cast( yield_delay.count() / pool_->timestamp_scale())) , num_threads_(hpx::threads::count(pu_mask)) , pu_mask_(pu_mask) - , exception_mutex_() - , exception_() , region_data_(num_threads_) { HPX_ASSERT(pool_); @@ -463,15 +460,19 @@ namespace hpx::execution::experimental { { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "for_join_executor::shared_data::shared_data", - hpx::util::format("unexpected number of PUs in given " - "mask: {}, available threads: {}", - pu_mask, - pool_ ? pool_->get_os_thread_count() : -1)); + "unexpected number of PUs in given mask: {}, available " + "threads: {}", + pu_mask, pool_ ? pool_->get_os_thread_count() : -1); } init_threads(); } + shared_data(shared_data const&) = delete; + shared_data(shared_data&&) = delete; + shared_data& operator=(shared_data const&) = delete; + shared_data& operator=(shared_data&&) = delete; + ~shared_data() { set_state_all(thread_state::stopping); @@ -530,7 +531,7 @@ namespace hpx::execution::experimental { { region_data& data = rdata[thread_index].data_; hpx::detail::try_catch_exception_ptr( - [&]() { + [&] { // Cast void pointers back to the actual types given // to bulk_sync_execute. auto& element_function = @@ -540,7 +541,7 @@ namespace hpx::execution::experimental { *static_cast(data.argument_pack_); // Set up the local queues and state. - std::size_t size = hpx::util::size(shape); + std::size_t const size = hpx::util::size(shape); auto part_begin = static_cast( (thread_index * size) / num_threads); @@ -579,7 +580,7 @@ namespace hpx::execution::experimental { { region_data& data = rdata[thread_index].data_; hpx::detail::try_catch_exception_ptr( - [&]() { + [&] { // Cast void pointers back to the actual types given // to bulk_sync_execute. auto& element_function = @@ -591,7 +592,7 @@ namespace hpx::execution::experimental { // Set up the local queues and state. queue_type& local_queue = queues[thread_index].data_; - std::size_t size = hpx::util::size(shape); + std::size_t const size = hpx::util::size(shape); init_local_work_queue( local_queue, thread_index, num_threads, size); @@ -612,7 +613,7 @@ namespace hpx::execution::experimental { for (std::size_t offset = 1; offset < num_threads; ++offset) { - std::size_t neighbor_index = + std::size_t const neighbor_index = (thread_index + offset) % num_threads; if (rdata[neighbor_index].data_.state_.load( @@ -647,12 +648,73 @@ namespace hpx::execution::experimental { } }; + template + struct thread_function_helper_invoke + { + using function_pack_type = std::decay_t; + using index_pack_type = hpx::detail::fused_index_pack_t; + + static constexpr std::uint32_t Size = hpx::tuple_size_v; + + static void set_state(std::atomic& tstate, + thread_state state) noexcept + { + tstate.store(state, std::memory_order_release); + } + + // Main entry point for a single parallel invoke region + static void call(region_data_type& rdata, + std::size_t thread_index, std::size_t num_threads, + queues_type&, hpx::spinlock& exception_mutex, + std::exception_ptr& exception) noexcept + { + region_data& data = rdata[thread_index].data_; + hpx::detail::try_catch_exception_ptr( + [&] { + // Cast void pointers back to the actual types given + // to sync_invoke. + auto& fs = + *static_cast(data.element_function_); + auto& args = + *static_cast(data.argument_pack_); + + auto part_begin = + static_cast(hpx::get<0>(args) + + (thread_index * hpx::get<1>(args)) / + num_threads); + auto const part_end = + static_cast(hpx::get<0>(args) + + ((thread_index + 1) * hpx::get<1>(args)) / + num_threads); + + set_state(data.state_, thread_state::active); + + // Process local items. + for (/**/; part_begin != part_end; ++part_begin) + { + hpx::visit([](auto&& f) { f(); }, + hpx::detail::runtime_get(fs, part_begin)); + } + }, + [&](std::exception_ptr&& ep) { + std::lock_guard l( + exception_mutex); + if (!exception) + { + exception = HPX_MOVE(ep); + } + }); + + set_state(data.state_, thread_state::idle); + } + }; + template thread_function_helper_type* set_all_states_and_region_data( thread_state state, F& f, S const& shape, Args& argument_pack) noexcept { - thread_function_helper_type* func = nullptr; + thread_function_helper_type* func; if (schedule_ == loop_schedule::static_ || num_threads_ == 1) { func = &thread_function_helper::call_static; @@ -676,14 +738,47 @@ namespace hpx::execution::experimental { return func; } + template + thread_function_helper_type* set_all_states_and_region_data_invoke( + thread_state state, Fs& function_pack, Args& args) noexcept + { + constexpr thread_function_helper_type* func = + &thread_function_helper_invoke::call; + + for (std::size_t t = 0; t != num_threads_; ++t) + { + region_data& data = region_data_[t].data_; + + data.element_function_ = &function_pack; + data.shape_ = nullptr; + data.argument_pack_ = &args; + data.thread_function_helper_ = func; + + data.state_.store(state, std::memory_order_release); + } + + return func; + } + public: template void bulk_sync_execute(F&& f, S const& shape, Ts&&... ts) { + // protect against nested use of this executor instance + if (region_data_[main_thread_].data_.state_.load( + std::memory_order_relaxed) != thread_state::idle) + { + HPX_THROW_EXCEPTION(error::bad_request, "bulk_sync_execute", + "unexpected state, is this instance of " + "fork_join_executor being used in nested ways?"); + } + hpx::scoped_annotation annotate( generate_annotation(hpx::get_worker_thread_num(), "fork_join_executor::bulk_sync_execute")); + exception_ = std::exception_ptr(); + // Set the data for this parallel region auto argument_pack = hpx::forward_as_tuple(HPX_FORWARD(Ts, ts)...); @@ -703,7 +798,7 @@ namespace hpx::execution::experimental { // them in this parallel region. wait_state_all(thread_state::idle); - std::lock_guard l(exception_mutex_); + // rethrow exception, if any if (exception_) { std::rethrow_exception(HPX_MOVE(exception_)); @@ -727,12 +822,88 @@ namespace hpx::execution::experimental { return hpx::make_exceptional_future(HPX_MOVE(ep)); }); } + + template + void sync_invoke_helper(FunctionPack& function_pack, + std::size_t first, std::size_t size) + { + // protect against nested use of this executor instance + if (region_data_[main_thread_].data_.state_.load( + std::memory_order_relaxed) != thread_state::idle) + { + HPX_THROW_EXCEPTION(error::bad_request, + "sync_invoke_helper", + "unexpected state, is this instance of " + "fork_join_executor being used in nested ways?"); + } + + // Set the data for this parallel region + hpx::scoped_annotation annotate( + generate_annotation(hpx::get_worker_thread_num(), + "fork_join_executor::sync_invoke")); + + exception_ = std::exception_ptr(); + + auto args = hpx::make_tuple(first, size); + + // Signal all worker threads to start partitioning work for + // themselves, and then starting the actual work. + thread_function_helper_type* func = + set_all_states_and_region_data_invoke( + thread_state::partitioning_work, function_pack, args); + + // Start work on the main thread. + func(region_data_, main_thread_, num_threads_, queues_, + exception_mutex_, exception_); + + // Wait for all threads to finish their work assigned to + // them in this parallel region. + wait_state_all(thread_state::idle); + + // rethrow exception, if any + if (exception_) + { + std::rethrow_exception(HPX_MOVE(exception_)); + } + } + + template + void sync_invoke(Fs&&... fs) + { + auto function_pack = + hpx::forward_as_tuple(HPX_FORWARD(Fs, fs)...); + + sync_invoke_helper(function_pack, 0, sizeof...(Fs)); + } + + template + hpx::future async_invoke(Fs&&... fs) + { + // Forward to the synchronous version as we can't create + // futures to the completion of the parallel region (this HPX + // thread participates in computation). + return hpx::detail::try_catch_exception_ptr( + [&]() { + sync_invoke(HPX_FORWARD(Fs, fs)...); + return hpx::make_ready_future(); + }, + [&](std::exception_ptr&& ep) { + return hpx::make_exceptional_future(HPX_MOVE(ep)); + }); + } }; + public: + template + void sync_invoke_helper(FunctionPack& function_pack, std::size_t first, + std::size_t size) const + { + shared_data_->sync_invoke_helper(function_pack, first, size); + } + private: std::shared_ptr shared_data_ = nullptr; - private: // clang-format off template && (std::is_invocable_v && ...) + )> + // clang-format on + friend decltype(auto) tag_invoke( + hpx::parallel::execution::async_invoke_t, + fork_join_executor const& exec, F&& f, Fs&&... fs) + { + return exec.shared_data_->async_invoke( + HPX_FORWARD(F, f), HPX_FORWARD(Fs, fs)...); + } + + // clang-format off + template && (std::is_invocable_v && ...) + )> + // clang-format on + friend decltype(auto) tag_invoke( + hpx::parallel::execution::sync_invoke_t, + fork_join_executor const& exec, F&& f, Fs&&... fs) + { + exec.shared_data_->sync_invoke( + HPX_FORWARD(F, f), HPX_FORWARD(Fs, fs)...); + } + public: bool operator==(fork_join_executor const& rhs) const noexcept { @@ -784,9 +983,8 @@ namespace hpx::execution::experimental { /// \param stacksize The stacksize of the worker threads. Must not be /// nostack. /// \param schedule The loop schedule of the parallel regions. - /// \param yield_delay The time after which the executor yields to - /// other work if it hasn't received any new work for bulk - /// execution. + /// \param yield_delay The time after which the executor yields to other + /// work if it has not received any new work for execution. explicit fork_join_executor( threads::thread_priority priority = threads::thread_priority::bound, threads::thread_stacksize stacksize = @@ -814,9 +1012,8 @@ namespace hpx::execution::experimental { /// \param stacksize The stacksize of the worker threads. Must not be /// nostack. /// \param schedule The loop schedule of the parallel regions. - /// \param yield_delay The time after which the executor yields to - /// other work if it hasn't received any new work for bulk - /// execution. + /// \param yield_delay The time after which the executor yields to other + /// work if it has not received any new work for execution. explicit fork_join_executor(hpx::threads::mask_cref_type pu_mask, threads::thread_priority priority = threads::thread_priority::bound, threads::thread_stacksize stacksize = @@ -890,7 +1087,8 @@ namespace hpx::execution::experimental { std::ostream& os, fork_join_executor::loop_schedule schedule); } // namespace hpx::execution::experimental -namespace hpx { namespace parallel { namespace execution { +namespace hpx::parallel::execution { + /// \cond NOINTERNAL template <> struct is_bulk_one_way_executor< @@ -904,4 +1102,4 @@ namespace hpx { namespace parallel { namespace execution { { }; /// \endcond -}}} // namespace hpx::parallel::execution +} // namespace hpx::parallel::execution diff --git a/libs/core/executors/tests/unit/fork_join_executor.cpp b/libs/core/executors/tests/unit/fork_join_executor.cpp index 4d1d9baa80e6..c1d9314e415a 100644 --- a/libs/core/executors/tests/unit/fork_join_executor.cpp +++ b/libs/core/executors/tests/unit/fork_join_executor.cpp @@ -25,7 +25,10 @@ using hpx::execution::experimental::fork_join_executor; -static std::atomic count{0}; +static std::atomic count1{0}; +static std::atomic count2{0}; +static std::atomic count3{0}; +static std::atomic count4{0}; /////////////////////////////////////////////////////////////////////////////// template @@ -33,23 +36,23 @@ void test_processing_mask(ExecutorArgs&&... args) { std::cerr << "test_processing_mask\n"; - auto& rp = hpx::resource::get_partitioner(); + auto const& rp = hpx::resource::get_partitioner(); auto const& expected_mask = rp.get_used_pus_mask(hpx::get_worker_thread_num()); fork_join_executor exec{expected_mask, std::forward(args)...}; - auto pus_mask = + auto const pus_mask = hpx::execution::experimental::get_processing_units_mask(exec); HPX_TEST(pus_mask == expected_mask); - auto cores_mask = hpx::execution::experimental::get_cores_mask(exec); + auto const cores_mask = hpx::execution::experimental::get_cores_mask(exec); HPX_TEST(cores_mask == expected_mask); } /////////////////////////////////////////////////////////////////////////////// void bulk_test(int, int passed_through) //-V813 { - ++count; + ++count1; HPX_TEST_EQ(passed_through, 42); } @@ -58,8 +61,8 @@ void test_bulk_sync(ExecutorArgs&&... args) { std::cerr << "test_bulk_sync\n"; - count = 0; - std::size_t const n = 107; + count1 = 0; + constexpr std::size_t n = 107; std::vector v(n); std::iota(std::begin(v), std::end(v), std::rand()); @@ -69,10 +72,10 @@ void test_bulk_sync(ExecutorArgs&&... args) fork_join_executor exec{std::forward(args)...}; hpx::parallel::execution::bulk_sync_execute( exec, hpx::bind(&bulk_test, _1, _2), v, 42); - HPX_TEST_EQ(count.load(), n); + HPX_TEST_EQ(count1.load(), n); hpx::parallel::execution::bulk_sync_execute(exec, &bulk_test, v, 42); - HPX_TEST_EQ(count.load(), 2 * n); + HPX_TEST_EQ(count1.load(), 2 * n); } template @@ -80,7 +83,7 @@ void test_bulk_async(ExecutorArgs&&... args) { std::cerr << "test_bulk_async\n"; - count = 0; + count1 = 0; std::size_t const n = 107; std::vector v(n); std::iota(std::begin(v), std::end(v), std::rand()); @@ -92,12 +95,12 @@ void test_bulk_async(ExecutorArgs&&... args) hpx::when_all(hpx::parallel::execution::bulk_async_execute( exec, hpx::bind(&bulk_test, _1, _2), v, 42)) .get(); - HPX_TEST_EQ(count.load(), n); + HPX_TEST_EQ(count1.load(), n); hpx::when_all( hpx::parallel::execution::bulk_async_execute(exec, &bulk_test, v, 42)) .get(); - HPX_TEST_EQ(count.load(), 2 * n); + HPX_TEST_EQ(count1.load(), 2 * n); } /////////////////////////////////////////////////////////////////////////////// @@ -112,7 +115,7 @@ void test_bulk_sync_exception(ExecutorArgs&&... args) { std::cerr << "test_bulk_sync_exception\n"; - count = 0; + count1 = 0; std::size_t const n = 107; std::vector v(n); std::iota(std::begin(v), std::end(v), std::rand()); @@ -143,7 +146,7 @@ void test_bulk_async_exception(ExecutorArgs&&... args) { std::cerr << "test_bulk_async_exception\n"; - count = 0; + count1 = 0; std::size_t const n = 107; std::vector v(n); std::iota(std::begin(v), std::end(v), std::rand()); @@ -170,6 +173,132 @@ void test_bulk_async_exception(ExecutorArgs&&... args) HPX_TEST(caught_exception); } +template +void test_invoke_sync_homogeneous(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync_homogeneous\n"; + + auto f1 = [] { ++count1; }; + + fork_join_executor exec{std::forward(args)...}; + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1); + HPX_TEST_EQ(count1.load(), static_cast(1)); + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f1); + HPX_TEST_EQ(count1.load(), static_cast(2)); + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f1, f1, f1, f1); + HPX_TEST_EQ(count1.load(), static_cast(5)); + + count1 = 0; + hpx::parallel::execution::sync_invoke( + exec, f1, f1, f1, f1, f1, f1, f1, f1, f1, f1, f1); + HPX_TEST_EQ(count1.load(), static_cast(11)); +} + +template +void test_invoke_sync(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync\n"; + + auto f1 = [] { ++count1; }; + auto f2 = [] { ++count2; }; + auto f3 = [] { ++count3; }; + auto f4 = [] { ++count4; }; + + fork_join_executor exec{std::forward(args)...}; + + count1 = 0; + hpx::parallel::execution::sync_invoke(exec, f1); + HPX_TEST_EQ(count1.load(), static_cast(1)); + + count1 = 0; + count2 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f2); + HPX_TEST_EQ(count1.load(), static_cast(1)); + HPX_TEST_EQ(count2.load(), static_cast(1)); + + count1 = 0; + count2 = 0; + count3 = 0; + count4 = 0; + hpx::parallel::execution::sync_invoke(exec, f1, f2, f3, f4, f1); + HPX_TEST_EQ(count1.load(), static_cast(2)); + HPX_TEST_EQ(count2.load(), static_cast(1)); + HPX_TEST_EQ(count3.load(), static_cast(1)); + HPX_TEST_EQ(count4.load(), static_cast(1)); + + count1 = 0; + count2 = 0; + count3 = 0; + count4 = 0; + hpx::parallel::execution::sync_invoke( + exec, f1, f2, f3, f4, f1, f4, f1, f2, f3, f4, f1); + HPX_TEST_EQ(count1.load(), static_cast(4)); + HPX_TEST_EQ(count2.load(), static_cast(2)); + HPX_TEST_EQ(count3.load(), static_cast(2)); + HPX_TEST_EQ(count4.load(), static_cast(3)); +} + +template +void test_invoke_sync_homogeneous_exception(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync_homogeneous_exception\n"; + + auto f1 = [] { throw std::runtime_error("test"); }; + + fork_join_executor exec{std::forward(args)...}; + + bool caught_exception = false; + try + { + hpx::parallel::execution::sync_invoke(exec, f1, f1, f1); + HPX_TEST(false); + } + catch (std::runtime_error const& /*e*/) + { + caught_exception = true; + } + catch (...) + { + HPX_TEST(false); + } + + HPX_TEST(caught_exception); +} + +template +void test_invoke_sync_exception(ExecutorArgs&&... args) +{ + std::cerr << "test_invoke_sync_exception\n"; + + auto f1 = [] {}; + auto f2 = [] { throw std::runtime_error("test"); }; + + fork_join_executor exec{std::forward(args)...}; + + bool caught_exception = false; + try + { + hpx::parallel::execution::sync_invoke(exec, f1, f2); + HPX_TEST(false); + } + catch (std::runtime_error const& /*e*/) + { + caught_exception = true; + } + catch (...) + { + HPX_TEST(false); + } + + HPX_TEST(caught_exception); +} + void static_check_executor() { using namespace hpx::traits; @@ -193,6 +322,11 @@ void test_executor(hpx::threads::thread_priority priority, test_bulk_sync_exception(priority, stacksize, schedule); test_bulk_async_exception(priority, stacksize, schedule); + test_invoke_sync_homogeneous(priority, stacksize, schedule); + test_invoke_sync(priority, stacksize, schedule); + test_invoke_sync_homogeneous_exception(priority, stacksize, schedule); + test_invoke_sync_exception(priority, stacksize, schedule); + test_processing_mask(priority, stacksize, schedule); } diff --git a/libs/core/functional/CMakeLists.txt b/libs/core/functional/CMakeLists.txt index b237980e355a..1b78978ffb3d 100644 --- a/libs/core/functional/CMakeLists.txt +++ b/libs/core/functional/CMakeLists.txt @@ -13,6 +13,7 @@ set(functional_headers hpx/functional/detail/empty_function.hpp hpx/functional/detail/function_registration.hpp hpx/functional/detail/reset_function.hpp + hpx/functional/detail/runtime_get.hpp hpx/functional/detail/vtable/callable_vtable.hpp hpx/functional/detail/vtable/copyable_vtable.hpp hpx/functional/detail/vtable/function_vtable.hpp diff --git a/libs/core/functional/include/hpx/functional/detail/runtime_get.hpp b/libs/core/functional/include/hpx/functional/detail/runtime_get.hpp new file mode 100644 index 000000000000..39d7b125c02e --- /dev/null +++ b/libs/core/functional/include/hpx/functional/detail/runtime_get.hpp @@ -0,0 +1,98 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace hpx::detail { + + /////////////////////////////////////////////////////////////////////////// + template + struct access_table + { + using tuple_type = Tuple; + using return_type = R; + + template + [[nodiscard]] static constexpr return_type access_tuple( + tuple_type& t, F& f) noexcept + { + return HPX_INVOKE(f, hpx::get(t)); + } + + using accessor_fun_ptr = return_type (*)(tuple_type&, F&) noexcept; + static constexpr std::size_t table_size = sizeof...(Is); + + static constexpr std::array lookup_table = + {{&access_tuple...}}; + }; + + /////////////////////////////////////////////////////////////////////////// + template + [[nodiscard]] constexpr decltype(auto) call_access_function( + Tuple& t, std::size_t i, F&& f, hpx::util::index_pack) noexcept + { + HPX_ASSERT_MSG( + i < sizeof...(Is), "index must be smaller than tuple size"); + + constexpr auto& table = access_table::lookup_table; + return table[i](t, f); + } + + /////////////////////////////////////////////////////////////////////////// + template + using first_tuple_element_t = + hpx::tuple_element_t<0, std::remove_reference_t>; + + template + [[nodiscard]] constexpr decltype(auto) homogenous_runtime_get( + Tuple& t, std::size_t i) noexcept + { + return call_access_function< + first_tuple_element_t&>>(t, i, hpx::identity_v, + hpx::util::make_index_pack< + hpx::tuple_size_v>>{}); + } + + /////////////////////////////////////////////////////////////////////////// + // Generate variant that uniquely holds all of the tuple types + template + struct variant_from_tuple; + + template + struct variant_from_tuple> + { + using type = + hpx::meta::invoke>, + std::reference_wrapper>...>; + }; + + template + using variant_from_tuple_t = typename variant_from_tuple::type; + + /////////////////////////////////////////////////////////////////////////// + template + [[nodiscard]] constexpr decltype(auto) runtime_get( + Tuple& t, std::size_t i) noexcept + { + return call_access_function>>( + t, i, [](auto& element) { return std::ref(element); }, + hpx::util::make_index_pack< + hpx::tuple_size_v>>{}); + } +} // namespace hpx::detail From e2484eb6e67e8858cf3cb3286b0620715c78ebec Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 25 Jul 2021 13:10:47 -0500 Subject: [PATCH 072/333] Futures attempt to execute threads directly if those have not started executing - adding new API function hpx::threads::get_outer_self_id - flyby: fixing initialization of resource partitioner --- .jenkins/cscs/env-clang-13.sh | 1 + .jenkins/lsu/batch.sh | 1 + .jenkins/lsu/env-gcc-9.sh | 2 + .../iostreams/src/server/output_stream.cpp | 23 +- .../manual/hpx_runtime_and_resources.rst | 15 +- .../hpx/async_base/scheduling_properties.hpp | 20 ++ libs/core/coroutines/CMakeLists.txt | 20 +- .../include/hpx/coroutines/coroutine.hpp | 8 +- .../detail/context_generic_context.hpp | 5 +- .../coroutines/detail/context_linux_x86.hpp | 18 +- .../hpx/coroutines/detail/context_posix.hpp | 5 +- .../detail/context_windows_fibers.hpp | 7 +- .../hpx/coroutines/detail/coroutine_impl.hpp | 10 +- .../hpx/coroutines/detail/coroutine_self.hpp | 5 + .../detail/coroutine_stackful_self.hpp | 4 +- .../detail/coroutine_stackful_self_direct.hpp | 77 +++++++ .../detail/coroutine_stackless_self.hpp | 20 +- .../include/hpx/coroutines/thread_enums.hpp | 83 +++++-- .../coroutines/src/detail/coroutine_impl.cpp | 68 +++++- libs/core/coroutines/src/thread_enums.cpp | 7 +- libs/core/debugging/src/print.cpp | 4 +- .../detail/async_launch_policy_dispatch.hpp | 133 +++++++---- .../hpx/execution/detail/future_exec.hpp | 22 +- .../execution/detail/post_policy_dispatch.hpp | 127 ++++++++--- .../detail/sync_launch_policy_dispatch.hpp | 25 ++- .../include/hpx/executors/dataflow.hpp | 10 +- libs/core/futures/CMakeLists.txt | 5 +- .../hpx/futures/detail/execute_thread.hpp | 15 ++ .../hpx/futures/detail/future_data.hpp | 84 ++++--- .../include/hpx/futures/futures_factory.hpp | 129 ++++++++++- .../hpx/futures/packaged_continuation.hpp | 7 +- .../include/hpx/futures/traits/is_future.hpp | 5 +- .../futures/src/detail/execute_thread.cpp | 210 ++++++++++++++++++ libs/core/futures/src/future_data.cpp | 92 ++++++-- libs/core/futures/tests/unit/CMakeLists.txt | 3 +- .../tests/unit/direct_scoped_execution.cpp | 87 ++++++++ .../src/init_runtime_local.cpp | 3 + .../hpx/resource_partitioner/partitioner.hpp | 3 + .../src/detail_partitioner.cpp | 2 + .../resource_partitioner/src/partitioner.cpp | 6 + .../tests/unit/cross_pool_injection.cpp | 97 ++++---- .../tests/unit/scheduler_binding_check.cpp | 6 +- .../hpx/runtime_local/runtime_local.hpp | 10 +- .../src/custom_exception_info.cpp | 140 +++++++----- libs/core/runtime_local/src/runtime_local.cpp | 11 + .../local_priority_queue_scheduler.hpp | 2 +- .../hpx/schedulers/queue_holder_thread.hpp | 4 +- .../shared_priority_queue_scheduler.hpp | 20 +- libs/core/thread_pools/CMakeLists.txt | 1 + .../thread_pools/scheduled_thread_pool.hpp | 2 +- .../scheduled_thread_pool_impl.hpp | 21 +- .../hpx/thread_pools/scheduling_loop.hpp | 29 +-- libs/core/threading_base/CMakeLists.txt | 2 +- .../hpx/threading_base/scheduler_base.hpp | 8 +- .../hpx/threading_base/thread_data.hpp | 50 ++++- .../threading_base/thread_data_stackful.hpp | 17 ++ .../hpx/threading_base/thread_pool_base.hpp | 2 +- libs/core/threading_base/src/create_work.cpp | 4 + .../threading_base/src/execution_agent.cpp | 2 + .../threading_base/src/set_thread_state.cpp | 3 +- .../src/set_thread_state_timed.cpp | 2 +- libs/core/threading_base/src/thread_data.cpp | 33 ++- .../threading_base/src/thread_helpers.cpp | 4 +- .../include/hpx/modules/threadmanager.hpp | 2 +- .../src/runtime_distributed.cpp | 41 ++++ 65 files changed, 1493 insertions(+), 391 deletions(-) create mode 100644 libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self_direct.hpp create mode 100644 libs/core/futures/include/hpx/futures/detail/execute_thread.hpp create mode 100644 libs/core/futures/src/detail/execute_thread.cpp create mode 100644 libs/core/futures/tests/unit/direct_scoped_execution.cpp diff --git a/.jenkins/cscs/env-clang-13.sh b/.jenkins/cscs/env-clang-13.sh index 1803b706501a..9d0d3a6f1138 100644 --- a/.jenkins/cscs/env-clang-13.sh +++ b/.jenkins/cscs/env-clang-13.sh @@ -30,6 +30,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" configure_extra_options+=" -DHPX_WITH_UNITY_BUILD=ON" +configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" # enable extra counters to verify everything compiles configure_extra_options+=" -DHPX_WITH_BACKGROUND_THREAD_COUNTERS=ON" diff --git a/.jenkins/lsu/batch.sh b/.jenkins/lsu/batch.sh index 64e4985ef554..f9bd7d3d3bf8 100755 --- a/.jenkins/lsu/batch.sh +++ b/.jenkins/lsu/batch.sh @@ -22,6 +22,7 @@ ulimit -l unlimited set +e ctest \ ${ctest_extra_args} \ + --output-on-failure \ -S ${src_dir}/.jenkins/lsu/ctest.cmake \ -DCTEST_CONFIGURE_EXTRA_OPTIONS="${configure_extra_options}" \ -DCTEST_BUILD_CONFIGURATION_NAME="${configuration_name_with_build_type}" \ diff --git a/.jenkins/lsu/env-gcc-9.sh b/.jenkins/lsu/env-gcc-9.sh index f78a0175e0c9..abdb272af42a 100644 --- a/.jenkins/lsu/env-gcc-9.sh +++ b/.jenkins/lsu/env-gcc-9.sh @@ -28,3 +28,5 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" + +configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" diff --git a/components/iostreams/src/server/output_stream.cpp b/components/iostreams/src/server/output_stream.cpp index cb53b1f335f1..dd0a519b27f8 100644 --- a/components/iostreams/src/server/output_stream.cpp +++ b/components/iostreams/src/server/output_stream.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -25,11 +26,12 @@ #include #include -namespace hpx { namespace iostreams { namespace detail { +namespace hpx::iostreams::detail { + void buffer::save(serialization::output_archive& ar, unsigned) const { - bool valid = (data_.get() && !data_->empty()); - ar& valid; + bool const valid = (data_.get() && !data_->empty()); + ar << valid; if (valid) { ar& data_; @@ -39,15 +41,15 @@ namespace hpx { namespace iostreams { namespace detail { void buffer::load(serialization::input_archive& ar, unsigned) { bool valid = false; - ar& valid; + ar >> valid; if (valid) { ar& data_; } } -}}} // namespace hpx::iostreams::detail +} // namespace hpx::iostreams::detail -namespace hpx { namespace iostreams { namespace server { +namespace hpx::iostreams::server { /////////////////////////////////////////////////////////////////////////// void output_stream::call_write_async(std::uint32_t locality_id, std::uint64_t count, detail::buffer const& in, hpx::id_type /*this_id*/) @@ -87,12 +89,13 @@ namespace hpx { namespace iostreams { namespace server { { // {{{ // Perform the IO in another OS thread. detail::buffer in(buf_in); - hpx::get_thread_pool("io_pool")->get_io_service().post(hpx::bind_front( - &output_stream::call_write_sync, this, locality_id, count, - std::ref(in), threads::thread_id_ref_type(threads::get_self_id()))); + hpx::get_thread_pool("io_pool")->get_io_service().post( + hpx::bind_front(&output_stream::call_write_sync, this, locality_id, + count, std::ref(in), + threads::thread_id_ref_type(threads::get_outer_self_id()))); // Sleep until the worker thread wakes us up. this_thread::suspend(threads::thread_schedule_state::suspended, "output_stream::write_sync"); } // }}} -}}} // namespace hpx::iostreams::server +} // namespace hpx::iostreams::server diff --git a/docs/sphinx/manual/hpx_runtime_and_resources.rst b/docs/sphinx/manual/hpx_runtime_and_resources.rst index e52b383fdd13..667c7ea7f535 100644 --- a/docs/sphinx/manual/hpx_runtime_and_resources.rst +++ b/docs/sphinx/manual/hpx_runtime_and_resources.rst @@ -42,12 +42,11 @@ turned on, work stealing is done from queues associated with the same NUMA domai first, only after that work is stolen from other NUMA domains. This scheduler is enabled at build time by default using the FIFO -(first-in-first-out) queing policy. This policy can be invoked using +(first-in-first-out) queueing policy. This policy can be invoked using :option:`--hpx:queuing`\ ``local-priority-fifo``. The scheduler can also be enabled using the LIFO (last-in-first-out) policy. This is not the default policy and must be invoked using the command line option -:option:`--hpx:queuing`\ -``=local-priority-lifo``. +:option:`--hpx:queuing`\ ``local-priority-lifo``. Static priority scheduling policy --------------------------------- @@ -63,7 +62,7 @@ robin fashion. There is no thread stealing in this policy. Local scheduling policy ----------------------- -* invoke using: :option:`--hpx:queuing`\ ``=local`` (or ``-ql``) +* invoke using: :option:`--hpx:queuing`\ ``local`` (or ``-ql``) * flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or ``HPX_THREAD_SCHEDULERS=local`` @@ -73,7 +72,7 @@ thread pulls its tasks (user threads). Static scheduling policy ------------------------ -* invoke using: :option:`--hpx:queuing`\ ``=static`` +* invoke using: :option:`--hpx:queuing`\ ``static`` * flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or ``HPX_THREAD_SCHEDULERS=static`` @@ -84,7 +83,7 @@ robin fashion. There is no thread stealing in this policy. Priority ABP scheduling policy ------------------------------ -* invoke using: :option:`--hpx:queuing`\ ``=abp-priority-fifo`` +* invoke using: :option:`--hpx:queuing`\ ``abp-priority-fifo`` * flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or ``HPX_THREAD_SCHEDULERS=abp-priority`` @@ -102,8 +101,8 @@ domain first, only after that work is stolen from other NUMA domains. This scheduler can be used with two underlying queuing policies (FIFO: first-in-first-out, and LIFO: last-in-first-out). In order to use the LIFO -policy use the command line option :option:`--hpx:queuing`\ -``=abp-priority-lifo``. +policy use the command line option +:option:`--hpx:queuing`\ ``=abp-priority-lifo``. .. Questions, concerns and notes: diff --git a/libs/core/async_base/include/hpx/async_base/scheduling_properties.hpp b/libs/core/async_base/include/hpx/async_base/scheduling_properties.hpp index 656d5182bb89..d37401954eee 100644 --- a/libs/core/async_base/include/hpx/async_base/scheduling_properties.hpp +++ b/libs/core/async_base/include/hpx/async_base/scheduling_properties.hpp @@ -64,6 +64,11 @@ namespace hpx::execution::experimental { { } with_priority{}; + template <> + struct is_scheduling_property : std::true_type + { + }; + inline constexpr struct get_priority_t final : hpx::functional::detail::tag_fallback { @@ -88,6 +93,11 @@ namespace hpx::execution::experimental { { } with_stacksize{}; + template <> + struct is_scheduling_property : std::true_type + { + }; + inline constexpr struct get_stacksize_t final : hpx::functional::detail::tag_fallback { @@ -112,6 +122,11 @@ namespace hpx::execution::experimental { { } with_hint{}; + template <> + struct is_scheduling_property : std::true_type + { + }; + inline constexpr struct get_hint_t final : hpx::functional::detail::tag_fallback { @@ -136,6 +151,11 @@ namespace hpx::execution::experimental { { } with_annotation{}; + template <> + struct is_scheduling_property : std::true_type + { + }; + inline constexpr struct get_annotation_t final : hpx::functional::detail::tag_fallback { diff --git a/libs/core/coroutines/CMakeLists.txt b/libs/core/coroutines/CMakeLists.txt index ca8560283b9b..2daf23eaa050 100644 --- a/libs/core/coroutines/CMakeLists.txt +++ b/libs/core/coroutines/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2020 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -14,6 +14,23 @@ hpx_option( MODULE COROUTINES ) +hpx_option( + HPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD + BOOL + "Futures attempt to run associated threads directly if those have not been started (default: OFF)" + OFF + CATEGORY "Thread Manager" + ADVANCED + MODULE COROUTINES +) + +if(HPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD) + hpx_add_config_define_namespace( + DEFINE HPX_COROUTINES_HAVE_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD + NAMESPACE COROUTINES + ) +endif() + set(coroutines_headers hpx/coroutines/coroutine.hpp hpx/coroutines/coroutine_fwd.hpp @@ -29,6 +46,7 @@ set(coroutines_headers hpx/coroutines/detail/coroutine_impl.hpp hpx/coroutines/detail/coroutine_self.hpp hpx/coroutines/detail/coroutine_stackful_self.hpp + hpx/coroutines/detail/coroutine_stackful_self_direct.hpp hpx/coroutines/detail/coroutine_stackless_self.hpp hpx/coroutines/detail/get_stack_pointer.hpp hpx/coroutines/detail/posix_utility.hpp diff --git a/libs/core/coroutines/include/hpx/coroutines/coroutine.hpp b/libs/core/coroutines/include/hpx/coroutines/coroutine.hpp index 0bc356698ba7..c4b03c7fb1da 100644 --- a/libs/core/coroutines/include/hpx/coroutines/coroutine.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/coroutine.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2006, Giovanni P. Deretta -// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // This code may be used under either of the following two licences: // @@ -151,6 +151,12 @@ namespace hpx::threads::coroutines { return impl_.result(); } + HPX_FORCEINLINE result_type invoke_directly(arg_type arg = arg_type()) + { + HPX_ASSERT(impl_.is_ready()); + return impl_.invoke_directly(arg); + } + bool is_ready() const noexcept { return impl_.is_ready(); diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/context_generic_context.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/context_generic_context.hpp index 9a1856327b73..bbea1d9080d5 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/context_generic_context.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/context_generic_context.hpp @@ -252,8 +252,11 @@ namespace hpx::threads::coroutines { return (std::numeric_limits::max)(); } #endif - void reset_stack() const + void reset_stack(bool direct_execution) { + if (direct_execution) + return; + if (ctx_) { #if defined(HPX_USE_POSIX_STACK_UTILITIES) diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/context_linux_x86.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/context_linux_x86.hpp index 5877c25c9c94..4a81f9879dce 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/context_linux_x86.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/context_linux_x86.hpp @@ -282,8 +282,8 @@ namespace hpx::threads::coroutines::detail::lx { using context_impl_base = x86_linux_context_impl_base; - // Create a context that on restore invokes Functor on - // a new stack. The stack size can be optionally specified. + // Create a context that on restore invokes Functor on a new stack. The + // stack size can be optionally specified. explicit x86_linux_context_impl(std::ptrdiff_t stack_size = -1) : m_stack_size(stack_size == -1 ? static_cast(default_stack_size) : @@ -365,8 +365,11 @@ namespace hpx::threads::coroutines::detail::lx { return m_stack_size; } - void reset_stack() + void reset_stack(bool direct_execution) { + if (direct_execution) + return; + HPX_ASSERT(m_stack); if (posix::reset_stack( m_stack, static_cast(m_stack_size))) @@ -379,7 +382,10 @@ namespace hpx::threads::coroutines::detail::lx { void rebind_stack() { - HPX_ASSERT(m_stack); + // directly executed coroutine, no need to allocate a stack + if (m_stack == nullptr) + return; + #if defined(HPX_HAVE_COROUTINE_COUNTERS) increment_stack_recycle_count(); #endif @@ -527,8 +533,8 @@ namespace hpx::threads::coroutines::detail::lx { #endif }; - // Free function. Saves the current context in @p from - // and restores the context in @p to. + // Free function. Saves the current context in @p from and restores the + // context in @p to. // @note This function is found by ADL. inline void swap_context(x86_linux_context_impl_base& from, x86_linux_context_impl_base const& to, default_hint) noexcept diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/context_posix.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/context_posix.hpp index 4a38b5325bad..d76d1a472dbe 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/context_posix.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/context_posix.hpp @@ -346,8 +346,11 @@ namespace hpx::threads::coroutines { return (std::numeric_limits::max)(); } #endif - void reset_stack() + void reset_stack(bool direct_execution) { + if (direct_execution) + return; + if (m_stack) { if (posix::reset_stack( diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/context_windows_fibers.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/context_windows_fibers.hpp index fb1e6edc5500..f3c4a04a9819 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/context_windows_fibers.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/context_windows_fibers.hpp @@ -35,10 +35,12 @@ #include #include #include +#include #include #include #include +#include #include #if defined(HPX_HAVE_COROUTINE_COUNTERS) @@ -279,7 +281,7 @@ namespace hpx::threads::coroutines { return stacksize_; } - static constexpr void reset_stack() noexcept {} + static constexpr void reset_stack(bool) noexcept {} #if defined(HPX_HAVE_COROUTINE_COUNTERS) void rebind_stack() noexcept @@ -294,7 +296,8 @@ namespace hpx::threads::coroutines { // https://stackoverflow.com/a/20930496/269943 static std::ptrdiff_t get_available_stack_space() noexcept { - MEMORY_BASIC_INFORMATION mbi = {}; // page range + MEMORY_BASIC_INFORMATION mbi = {}; // page range + std::memset(&mbi, '\0', sizeof(mbi)); VirtualQuery(&mbi, &mbi, sizeof(mbi)); // get range return reinterpret_cast(&mbi) - reinterpret_cast(mbi.AllocationBase); diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_impl.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_impl.hpp index c72859ef5b38..e6f1ff9e8f9c 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_impl.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_impl.hpp @@ -78,6 +78,10 @@ namespace hpx::threads::coroutines::detail { // execute the coroutine using normal context switching HPX_CORE_EXPORT void operator()() noexcept; + // execute the coroutine function directly in the context of the calling + // thread + HPX_CORE_EXPORT result_type invoke_directly(arg_type arg); + public: void bind_result(result_type const& res) noexcept { @@ -112,16 +116,18 @@ namespace hpx::threads::coroutines::detail { this->super_type::init(); } - void reset() + void reset(bool direct_execution) { // First reset the function and arguments + m_result = + result_type(thread_schedule_state::unknown, invalid_thread_id); m_arg = nullptr; m_fun.reset(); // Then reset the id and stack as they may be used by the // destructors of the thread function above this->super_type::reset(); - this->reset_stack(); + this->reset_stack(direct_execution); } void rebind(functor_type&& f, thread_id_type id) diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_self.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_self.hpp index ebfed04f8e7a..8737af27def7 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_self.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_self.hpp @@ -123,6 +123,11 @@ namespace hpx::threads::coroutines::detail { virtual thread_id_type get_thread_id() const noexcept = 0; + virtual thread_id_type get_outer_thread_id() const noexcept + { + return get_thread_id(); + } + virtual std::size_t get_thread_phase() const noexcept = 0; virtual std::ptrdiff_t get_available_stack_space() const noexcept = 0; diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self.hpp index 8819b21ab375..20d34a2c5541 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self.hpp @@ -22,7 +22,7 @@ namespace hpx::threads::coroutines::detail { { public: explicit coroutine_stackful_self( - impl_type* pimpl, coroutine_self* next_self = nullptr) noexcept + coroutine_impl* pimpl, coroutine_self* next_self = nullptr) noexcept : coroutine_self(next_self) , pimpl_(pimpl) { @@ -138,7 +138,7 @@ namespace hpx::threads::coroutines::detail { return pimpl_->get_continuation_recursion_count(); } - private: + protected: coroutine_impl* get_impl() noexcept override { return pimpl_; diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self_direct.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self_direct.hpp new file mode 100644 index 000000000000..b452b2bdb153 --- /dev/null +++ b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackful_self_direct.hpp @@ -0,0 +1,77 @@ +// Copyright (c) 2019-2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace hpx::threads::coroutines::detail { + + class coroutine_stackful_self_direct : public coroutine_stackful_self + { + public: + explicit coroutine_stackful_self_direct( + coroutine_impl* pimpl, coroutine_self* next_self) + : coroutine_stackful_self(pimpl, next_self) + , next_self_(next_self) + { + HPX_ASSERT(next_self_); + } + + // direct execution of a thread needs to use the executing context for + // yielding + arg_type yield_impl(result_type arg) override + { + return next_self_->yield_impl(arg); + } + + thread_id_type get_outer_thread_id() const noexcept override + { + return next_self_->get_outer_thread_id(); + } + +#if defined(HPX_HAVE_THREAD_PHASE_INFORMATION) + std::size_t get_thread_phase() const noexcept override + { + return next_self_->get_thread_phase(); + } +#endif + +#if defined(HPX_HAVE_THREADS_GET_STACK_POINTER) + // return the executing thread's available stack space + std::ptrdiff_t get_available_stack_space() const noexcept override + { + return next_self_->get_available_stack_space(); + } +#endif + + // return the executing thread's recursion count + std::size_t& get_continuation_recursion_count() noexcept override + { + return next_self_->get_continuation_recursion_count(); + } + + private: + // if we chain direct calls the executing thread needs to be inherited + // down + coroutine_impl* get_impl() noexcept override + { + return coroutine_accessor::get_impl(*next_self_); + } + + coroutine_self* next_self_; + }; +} // namespace hpx::threads::coroutines::detail diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackless_self.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackless_self.hpp index 903e911817ff..467ab84808f3 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackless_self.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/coroutine_stackless_self.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 Hartmut Kaiser +// Copyright (c) 2019-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -16,12 +16,12 @@ #include #include -namespace hpx ::threads::coroutines { +namespace hpx::threads::coroutines { class stackless_coroutine; } // namespace hpx::threads::coroutines -namespace hpx ::threads::coroutines::detail { +namespace hpx::threads::coroutines::detail { class coroutine_stackless_self : public coroutine_self { @@ -30,6 +30,7 @@ namespace hpx ::threads::coroutines::detail { : coroutine_self(nullptr) , pimpl_(pimpl) { + HPX_ASSERT(pimpl_); } arg_type yield_impl(result_type) override @@ -41,14 +42,12 @@ namespace hpx ::threads::coroutines::detail { thread_id_type get_thread_id() const noexcept override { - HPX_ASSERT(pimpl_); return pimpl_->get_thread_id(); } #if defined(HPX_HAVE_THREAD_PHASE_INFORMATION) std::size_t get_thread_phase() const noexcept override { - HPX_ASSERT(pimpl_); return pimpl_->get_thread_phase(); } #else @@ -65,47 +64,39 @@ namespace hpx ::threads::coroutines::detail { std::size_t get_thread_data() const noexcept override { - HPX_ASSERT(pimpl_); return pimpl_->get_thread_data(); } std::size_t set_thread_data(std::size_t data) noexcept override { - HPX_ASSERT(pimpl_); return pimpl_->set_thread_data(data); } #if defined(HPX_HAVE_LIBCDS) std::size_t get_libcds_data() const override { - HPX_ASSERT(pimpl_); return pimpl_->get_libcds_data(); } std::size_t set_libcds_data(std::size_t data) override { - HPX_ASSERT(pimpl_); return pimpl_->set_libcds_data(data); } std::size_t get_libcds_hazard_pointer_data() const override { - HPX_ASSERT(pimpl_); return pimpl_->get_libcds_hazard_pointer_data(); } std::size_t set_libcds_hazard_pointer_data(std::size_t data) override { - HPX_ASSERT(pimpl_); return pimpl_->set_libcds_hazard_pointer_data(data); } std::size_t get_libcds_dynamic_hazard_pointer_data() const override { - HPX_ASSERT(pimpl_); return pimpl_->get_libcds_dynamic_hazard_pointer_data(); } std::size_t set_libcds_dynamic_hazard_pointer_data( std::size_t data) override { - HPX_ASSERT(pimpl_); return pimpl_->set_libcds_dynamic_hazard_pointer_data(data); } #endif @@ -113,7 +104,6 @@ namespace hpx ::threads::coroutines::detail { #if defined(HPX_HAVE_THREAD_LOCAL_STORAGE) tss_storage* get_thread_tss_data() override { - HPX_ASSERT(pimpl_); return pimpl_->get_thread_tss_data(false); } #else @@ -126,7 +116,6 @@ namespace hpx ::threads::coroutines::detail { #if defined(HPX_HAVE_THREAD_LOCAL_STORAGE) tss_storage* get_or_create_thread_tss_data() override { - HPX_ASSERT(pimpl_); return pimpl_->get_thread_tss_data(true); } #else @@ -138,7 +127,6 @@ namespace hpx ::threads::coroutines::detail { std::size_t& get_continuation_recursion_count() noexcept override { - HPX_ASSERT(pimpl_); return pimpl_->get_continuation_recursion_count(); } diff --git a/libs/core/coroutines/include/hpx/coroutines/thread_enums.hpp b/libs/core/coroutines/include/hpx/coroutines/thread_enums.hpp index 0de9618fcefc..7abf67d5c3d6 100644 --- a/libs/core/coroutines/include/hpx/coroutines/thread_enums.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/thread_enums.hpp @@ -8,6 +8,7 @@ #pragma once +#include #include #include @@ -46,10 +47,12 @@ namespace hpx::threads { but allows to create a thread in pending state without scheduling it (internal, do not use) */ - pending_boost = 8 /*!< this is not a real thread state, + pending_boost = 8, /*!< this is not a real thread state, but allows to suspend a thread in pending state without high priority rescheduling */ + deleted = 9 /*!< thread has been stopped and was + deleted */ }; // clang-format on @@ -220,7 +223,7 @@ namespace hpx::threads { /// \enum thread_placement_hint /// /// The type of hint given to the scheduler related to thread placement - enum class thread_placement_hint : std::uint8_t + enum class thread_placement_hint : std::int8_t { /// No hint is specified. The implementation is free to chose what /// placement methods to use. @@ -254,7 +257,7 @@ namespace hpx::threads { /// /// The type of hint given to the scheduler related to whether it is ok to /// share the invoked function object between threads - enum class thread_sharing_hint : std::uint8_t + enum class thread_sharing_hint : std::int8_t { /// No hint is specified. The implementation is free to chose what /// sharing methods to use. @@ -273,16 +276,15 @@ namespace hpx::threads { constexpr bool do_not_share_function(thread_sharing_hint hint) noexcept { - return static_cast(hint) & - static_cast( + return static_cast(hint) & + static_cast( thread_sharing_hint::do_not_share_function); } constexpr bool do_not_combine_tasks(thread_sharing_hint hint) noexcept { - return static_cast(hint) & - static_cast( - thread_sharing_hint::do_not_combine_tasks); + return static_cast(hint) & + static_cast(thread_sharing_hint::do_not_combine_tasks); } constexpr thread_sharing_hint operator|( @@ -292,6 +294,40 @@ namespace hpx::threads { static_cast(lhs) | static_cast(rhs)); } + /////////////////////////////////////////////////////////////////////////// + /// \enum thread_placement_hint + /// + /// The type of hint given to the scheduler related running a thread as a + /// child directly in the context of the parent thread + enum class thread_execution_hint : std::int8_t + { + /// No hint is specified. Always run the thread in its own execution + /// environment. + none = 0, + + /// Attempt to run the thread in the execution context of the parent + /// thread. + run_as_child = 1, + }; + + constexpr bool run_as_child(thread_execution_hint hint) noexcept + { + return static_cast(hint) & + static_cast(thread_execution_hint::run_as_child); + } + + /////////////////////////////////////////////////////////////////////////// + /// Default value to use for runs-as-child mode (if \a true, then futures + /// will attempt to execute associated threads directly if they have not + /// started running). +#if defined(HPX_COROUTINES_HAVE_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD) + inline constexpr thread_execution_hint default_runs_as_child_hint = + thread_execution_hint::run_as_child; +#else + inline constexpr thread_execution_hint default_runs_as_child_hint = + thread_execution_hint::none; +#endif + /////////////////////////////////////////////////////////////////////////// /// \brief A hint given to a scheduler to guide where a task should be /// scheduled. @@ -306,6 +342,8 @@ namespace hpx::threads { static_cast(thread_placement_hint::none)) , sharing_mode_bits( static_cast(thread_sharing_hint::none)) + , runs_as_child_mode_bits( + static_cast(default_runs_as_child_hint)) { } @@ -313,11 +351,13 @@ namespace hpx::threads { /// given hint as the local thread number. constexpr explicit thread_schedule_hint(std::int16_t thread_hint, thread_placement_hint placement = thread_placement_hint::none, + thread_execution_hint runs_as_child = default_runs_as_child_hint, thread_sharing_hint sharing = thread_sharing_hint::none) noexcept : hint(thread_hint) , mode(thread_schedule_hint_mode::thread) , placement_mode_bits(static_cast(placement)) , sharing_mode_bits(static_cast(sharing)) + , runs_as_child_mode_bits(static_cast(runs_as_child)) { } @@ -326,11 +366,13 @@ namespace hpx::threads { constexpr thread_schedule_hint(thread_schedule_hint_mode mode, std::int16_t hint, thread_placement_hint placement = thread_placement_hint::none, + thread_execution_hint runs_as_child = default_runs_as_child_hint, thread_sharing_hint sharing = thread_sharing_hint::none) noexcept : hint(hint) , mode(mode) , placement_mode_bits(static_cast(placement)) , sharing_mode_bits(static_cast(sharing)) + , runs_as_child_mode_bits(static_cast(runs_as_child)) { } @@ -340,7 +382,8 @@ namespace hpx::threads { { return mode == rhs.mode && hint == rhs.hint && placement_mode() == rhs.placement_mode() && - sharing_mode() == rhs.sharing_mode(); + sharing_mode() == rhs.sharing_mode() && + runs_as_child_mode() == rhs.runs_as_child_mode(); } constexpr bool operator!=( @@ -359,7 +402,7 @@ namespace hpx::threads { } void placement_mode(thread_placement_hint bits) noexcept { - placement_mode_bits = static_cast(bits); + placement_mode_bits = static_cast(bits); } [[nodiscard]] constexpr thread_sharing_hint sharing_mode() @@ -369,7 +412,17 @@ namespace hpx::threads { } void sharing_mode(thread_sharing_hint bits) noexcept { - sharing_mode_bits = static_cast(bits); + sharing_mode_bits = static_cast(bits); + } + + [[nodiscard]] constexpr thread_execution_hint runs_as_child_mode() + const noexcept + { + return static_cast(runs_as_child_mode_bits); + } + void runs_as_child_mode(thread_execution_hint bits) noexcept + { + runs_as_child_mode_bits = static_cast(bits); } /// The hint associated with the mode. The interpretation of this hint @@ -380,9 +433,13 @@ namespace hpx::threads { thread_schedule_hint_mode mode = thread_schedule_hint_mode::none; /// The mode of the desired thread placement. - std::int8_t placement_mode_bits : 6; + std::uint8_t placement_mode_bits : 5; /// The mode of the desired sharing hint - std::int8_t sharing_mode_bits : 2; + std::uint8_t sharing_mode_bits : 2; + + /// The thread will run as a child directly in the context of the + /// current thread + std::uint8_t runs_as_child_mode_bits : 1; }; } // namespace hpx::threads diff --git a/libs/core/coroutines/src/detail/coroutine_impl.cpp b/libs/core/coroutines/src/detail/coroutine_impl.cpp index b1ad0f300166..66d1490ac3f8 100644 --- a/libs/core/coroutines/src/detail/coroutine_impl.cpp +++ b/libs/core/coroutines/src/detail/coroutine_impl.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2006, Giovanni P. Deretta -// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // This code may be used under either of the following two licences: // @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -51,9 +52,10 @@ namespace hpx::threads::coroutines::detail { void coroutine_impl::operator()() noexcept { + using context_state = super_type::context_state; using context_exit_status = super_type::context_exit_status; - context_exit_status status = - super_type::context_exit_status::not_exited; + + context_exit_status status = context_exit_status::not_exited; // yield value once the thread function has finished executing result_type result_last( @@ -75,26 +77,76 @@ namespace hpx::threads::coroutines::detail { result_last = m_fun(*this->args()); HPX_ASSERT( result_last.first == thread_schedule_state::terminated); - status = super_type::context_exit_status::exited_return; + status = context_exit_status::exited_return; } catch (...) { - status = super_type::context_exit_status::exited_abnormally; + status = context_exit_status::exited_abnormally; tinfo = std::current_exception(); } // Reset early as the destructors may still yield. this->reset_tss(); - this->reset(); + this->reset(false); // return value to other side of the fence this->bind_result(result_last); } this->do_return(status, HPX_MOVE(tinfo)); - } while (this->m_state == super_type::context_state::running); + } while (this->m_state == context_state::running); // should not get here, never - HPX_ASSERT(this->m_state == super_type::context_state::running); + HPX_ASSERT(this->m_state == context_state::running); + } + + // execute the coroutine function directly in the context of the calling + // thread + coroutine_impl::result_type coroutine_impl::invoke_directly( + coroutine_impl::arg_type arg) + { + using context_state = super_type::context_state; + using context_exit_status = super_type::context_exit_status; + + context_exit_status status = context_exit_status::not_exited; + + result_type result_last( + thread_schedule_state::unknown, invalid_thread_id); + + this->m_state = context_state::running; + + std::exception_ptr tinfo; + { + coroutine_self* old_self = coroutine_self::get_self(); + coroutine_stackful_self_direct self(this, old_self); + reset_self_on_exit on_exit(&self, old_self); + try + { + this->m_result = this->m_fun(arg); + HPX_ASSERT( + this->m_result.first == thread_schedule_state::terminated); + status = context_exit_status::exited_return; + } + catch (...) + { + status = context_exit_status::exited_abnormally; + this->m_type_info = std::current_exception(); + } + + this->reset_tss(); + this->reset(true); + + this->bind_result(result_last); + } + + HPX_ASSERT(this->m_state == context_state::running); + this->m_state = context_state::exited; + this->m_exit_status = status; + + if (status == context_exit_status::exited_abnormally) + { + std::rethrow_exception(this->m_type_info); + } + return this->m_result; } } // namespace hpx::threads::coroutines::detail diff --git a/libs/core/coroutines/src/thread_enums.cpp b/libs/core/coroutines/src/thread_enums.cpp index 6dc027fcc2c1..53235d6d4a69 100644 --- a/libs/core/coroutines/src/thread_enums.cpp +++ b/libs/core/coroutines/src/thread_enums.cpp @@ -29,7 +29,8 @@ namespace hpx::threads { "terminated", "staged", "pending_do_not_schedule", - "pending_boost" + "pending_boost", + "deleted" }; // clang-format on @@ -37,8 +38,8 @@ namespace hpx::threads { char const* get_thread_state_name(thread_schedule_state state) noexcept { - if (state < thread_schedule_state::active || - state > thread_schedule_state::pending_boost) + if (state < thread_schedule_state::unknown || + state > thread_schedule_state::deleted) { return "unknown"; } diff --git a/libs/core/debugging/src/print.cpp b/libs/core/debugging/src/print.cpp index cf1b201382af..3d7cf5da2aa0 100644 --- a/libs/core/debugging/src/print.cpp +++ b/libs/core/debugging/src/print.cpp @@ -257,7 +257,7 @@ namespace hpx::debug { [[nodiscard]] char const* hostname_print_helper::get_hostname() const { static bool initialized = false; - static char hostname_[20] = {'\0'}; + static char hostname_[32] = {'\0'}; if (!initialized) { initialized = true; @@ -265,7 +265,7 @@ namespace hpx::debug { gethostname(hostname_, static_cast(12)); #endif int const rank = guess_rank(); - if (rank != -1) + if (rank >= 0) { #if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 110000 #pragma GCC diagnostic push diff --git a/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp index b97dd8862701..3af112f5c98b 100644 --- a/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const&, F&& f, Ts&&... ts) + call(Policy&&, F&& f, Ts&&... ts) { return call_sync(HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -77,7 +78,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const&, hpx::threads::thread_description const& desc, F&& f, + call(Policy&&, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { auto ann = hpx::scoped_annotation(desc.get_description()); @@ -88,7 +89,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const&, hpx::threads::thread_description const& desc, + call(Policy&&, hpx::threads::thread_description const& desc, threads::thread_pool_base*, F&& f, Ts&&... ts) { auto ann = hpx::scoped_annotation(desc.get_description()); @@ -103,7 +104,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const&, F&& f, Ts&&... ts) + call(Policy&&, F&& f, Ts&&... ts) { using result_type = util::detail::invoke_deferred_result_t; @@ -117,10 +118,10 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, hpx::threads::thread_description const& desc, + call(Policy&& policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - return call(policy, + return call(HPX_FORWARD(Policy, policy), hpx::annotated_function( HPX_FORWARD(F, f), desc.get_description()), HPX_FORWARD(Ts, ts)...); @@ -130,10 +131,10 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, hpx::threads::thread_description const& desc, + call(Policy&& policy, hpx::threads::thread_description const& desc, threads::thread_pool_base*, F&& f, Ts&&... ts) { - return call(policy, + return call(HPX_FORWARD(Policy, policy), hpx::annotated_function( HPX_FORWARD(F, f), desc.get_description()), HPX_FORWARD(Ts, ts)...); @@ -147,7 +148,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, hpx::threads::thread_description const& desc, + call(Policy&& policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { HPX_ASSERT(pool); @@ -155,19 +156,43 @@ namespace hpx::detail { using result_type = util::detail::invoke_deferred_result_t; + // if one of the arguments is a future we play it conservatively + auto hint = policy.hint(); + if constexpr (hpx::traits::is_future_any_v...>) + { + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } + lcos::local::futures_factory p( util::deferred_call(HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...)); threads::thread_id_ref_type tid = - p.post(pool, desc.get_description(), policy); + p.post(pool, desc.get_description(), HPX_MOVE(policy)); if (tid) { - // keep thread alive, if needed - auto&& result = p.get_future(); - traits::detail::get_shared_state(result)->set_on_completed( - [tid = HPX_MOVE(tid)]() { (void) tid; }); - return HPX_MOVE(result); + auto runs_as_child = hint.runs_as_child_mode(); + if (runs_as_child == + hpx::threads::thread_execution_hint::run_as_child && + !pool->get_scheduler()->supports_direct_execution()) + { + runs_as_child = hpx::threads::thread_execution_hint::none; + } + + if (runs_as_child == hpx::threads::thread_execution_hint::none) + { + // keep thread alive, if needed + auto result = p.get_future(); + traits::detail::get_shared_state(result)->set_on_completed( + [tid = HPX_MOVE(tid)]() { (void) tid; }); + return result; + } } return p.get_future(); @@ -177,10 +202,10 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, hpx::threads::thread_description const& desc, + call(Policy&& policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - return call(policy, desc, + return call(HPX_FORWARD(Policy, policy), desc, threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -189,10 +214,10 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, F&& f, Ts&&... ts) + call(Policy&& policy, F&& f, Ts&&... ts) { hpx::threads::thread_description desc(f); - return call(policy, desc, + return call(HPX_FORWARD(Policy, policy), desc, threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -205,14 +230,27 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, hpx::threads::thread_description const& desc, + call(Policy&& policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { - HPX_ASSERT(pool); + HPX_ASSERT(pool != nullptr); using result_type = util::detail::invoke_deferred_result_t; + // if one of the arguments is a future we play it conservatively + auto hint = policy.hint(); + if constexpr (hpx::traits::is_future_any_v...>) + { + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } + lcos::local::futures_factory p( util::deferred_call(HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...)); @@ -230,11 +268,22 @@ namespace hpx::detail { threads::thread_schedule_state::pending, tid.noref(), desc.get_description()); - // keep thread alive, if needed - auto&& result = p.get_future(); - traits::detail::get_shared_state(result)->set_on_completed( - [tid = HPX_MOVE(tid)]() { (void) tid; }); - return HPX_MOVE(result); + auto runs_as_child = hint.runs_as_child_mode(); + if (runs_as_child == + hpx::threads::thread_execution_hint::run_as_child && + !pool->get_scheduler()->supports_direct_execution()) + { + runs_as_child = hpx::threads::thread_execution_hint::none; + } + + if (runs_as_child == hpx::threads::thread_execution_hint::none) + { + // keep thread alive, if needed + auto result = p.get_future(); + traits::detail::get_shared_state(result)->set_on_completed( + [tid = HPX_MOVE(tid)]() { (void) tid; }); + return result; + } } return p.get_future(); @@ -244,10 +293,10 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, hpx::threads::thread_description const& desc, + call(Policy&& policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - return call(policy, desc, + return call(HPX_FORWARD(Policy, policy), desc, threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -256,10 +305,10 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy const& policy, F&& f, Ts&&... ts) + call(Policy&& policy, F&& f, Ts&&... ts) { hpx::threads::thread_description desc(f); - return call(policy, desc, + return call(HPX_FORWARD(Policy, policy), desc, threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -276,27 +325,33 @@ namespace hpx::detail { call(launch policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { + if (policy == launch::async) + { + return async_launch_policy_dispatch< + hpx::launch::async_policy>::call(HPX_MOVE(policy), desc, + pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + } if (policy == launch::sync) { return async_launch_policy_dispatch< - hpx::launch::sync_policy>::call(policy, desc, pool, - HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + hpx::launch::sync_policy>::call(HPX_MOVE(policy), desc, + pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } if (policy == launch::deferred) { return async_launch_policy_dispatch< - hpx::launch::deferred_policy>::call(policy, desc, pool, - HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + hpx::launch::deferred_policy>::call(HPX_MOVE(policy), desc, + pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } if (policy == launch::fork) { return async_launch_policy_dispatch< - hpx::launch::fork_policy>::call(policy, desc, pool, - HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + hpx::launch::fork_policy>::call(HPX_MOVE(policy), desc, + pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } return async_launch_policy_dispatch< - hpx::launch::async_policy>::call(policy, desc, pool, + hpx::launch::async_policy>::call(HPX_MOVE(policy), desc, pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -307,7 +362,7 @@ namespace hpx::detail { call(launch policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - return call(policy, desc, + return call(HPX_MOVE(policy), desc, threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } @@ -319,7 +374,7 @@ namespace hpx::detail { call(launch policy, F&& f, Ts&&... ts) { hpx::threads::thread_description desc(f); - return call(policy, desc, + return call(HPX_MOVE(policy), desc, threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } diff --git a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp index c1445ed2c3a1..32a4c9953075 100644 --- a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp +++ b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -25,6 +25,9 @@ #include #include #include +#include +#include +#include #include #include @@ -150,10 +153,17 @@ namespace hpx::lcos::detail { struct post_policy_spawner { template - void operator()(F&& f, hpx::threads::thread_description desc) const + threads::thread_id_ref_type operator()( + F&& f, hpx::threads::thread_description desc) const { - hpx::detail::post_policy_dispatch::call( - hpx::launch::async, desc, HPX_FORWARD(F, f)); + threads::thread_init_data data( + threads::make_thread_function_nullary(HPX_FORWARD(F, f)), + HPX_MOVE(desc), threads::thread_priority::default_, + threads::thread_schedule_hint(), + threads::thread_stacksize::default_, + threads::thread_schedule_state::pending); + + return threads::register_thread(data); } }; @@ -163,9 +173,11 @@ namespace hpx::lcos::detail { Executor exec; template - void operator()(F&& f, hpx::threads::thread_description) const + threads::thread_id_ref_type operator()( + F&& f, hpx::threads::thread_description) const { hpx::parallel::execution::post(exec, HPX_FORWARD(F, f)); + return threads::invalid_thread_id; } }; diff --git a/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp index 1c85f81da120..bfd014d447b4 100644 --- a/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -32,25 +33,38 @@ namespace hpx::detail { struct post_policy_dispatch { template - static void call(Policy const& policy, + static void call(Policy policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { + auto hint = policy.hint(); + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + if (!pool->get_scheduler()->supports_direct_execution()) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } + threads::thread_init_data data( threads::make_thread_function_nullary(hpx::util::deferred_call( HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...)), - desc, policy.priority(), policy.hint(), policy.stacksize(), + desc, policy.priority(), hint, policy.stacksize(), threads::thread_schedule_state::pending); threads::register_work(data, pool); } template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - call(policy, desc, threads::detail::get_self_or_default_pool(), - HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + call(HPX_FORWARD(Policy, policy), desc, + threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } }; @@ -58,16 +72,30 @@ namespace hpx::detail { struct post_policy_dispatch { template - static void call(Policy const& policy, + static void call(Policy policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { + auto hint = policy.hint(); + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + if (!pool->get_scheduler()->supports_direct_execution()) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } + threads::thread_init_data data( threads::make_thread_function_nullary(hpx::util::deferred_call( HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...)), desc, policy.priority(), threads::thread_schedule_hint( - static_cast(get_worker_thread_num())), + threads::thread_schedule_hint_mode::thread, + static_cast(get_worker_thread_num()), + hint.placement_mode(), hint.runs_as_child_mode()), policy.stacksize(), threads::thread_schedule_state::pending_do_not_schedule, true); @@ -88,11 +116,12 @@ namespace hpx::detail { } template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - call(policy, desc, threads::detail::get_self_or_default_pool(), - HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + call(HPX_FORWARD(Policy, policy), desc, + threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } }; @@ -100,20 +129,22 @@ namespace hpx::detail { struct post_policy_dispatch { template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const&, threads::thread_pool_base*, F&& f, Ts&&... ts) { hpx::detail::sync_launch_policy_dispatch::call( - policy, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + HPX_FORWARD(Policy, policy), HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const&, F&& f, Ts&&... ts) { hpx::detail::sync_launch_policy_dispatch::call( - policy, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + HPX_FORWARD(Policy, policy), HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } }; @@ -121,22 +152,22 @@ namespace hpx::detail { struct post_policy_dispatch { template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const&, threads::thread_pool_base*, F&& f, Ts&&... ts) { hpx::detail::sync_launch_policy_dispatch< - launch::deferred_policy>::call(policy, HPX_FORWARD(F, f), - HPX_FORWARD(Ts, ts)...); + launch::deferred_policy>::call(HPX_FORWARD(Policy, policy), + HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const&, F&& f, Ts&&... ts) { hpx::detail::sync_launch_policy_dispatch< - launch::deferred_policy>::call(policy, HPX_FORWARD(F, f), - HPX_FORWARD(Ts, ts)...); + launch::deferred_policy>::call(HPX_FORWARD(Policy, policy), + HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } }; @@ -144,39 +175,69 @@ namespace hpx::detail { struct post_policy_dispatch { template - static void call(Policy const& policy, + static void call(Policy policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { + HPX_ASSERT(pool != nullptr); + + auto hint = policy.hint(); + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + if (!pool->get_scheduler()->supports_direct_execution()) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } + if (policy == launch::sync) { - post_policy_dispatch::call(policy, desc, - pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + auto mod_policy = launch::sync_policy( + policy.priority(), policy.stacksize(), hint); + + post_policy_dispatch::call( + HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } else if (policy == launch::deferred) { - // execute synchronously - post_policy_dispatch::call(policy, - desc, pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + auto mod_policy = launch::deferred_policy( + policy.priority(), policy.stacksize(), hint); + + post_policy_dispatch::call( + HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } else if (policy == launch::fork) { - post_policy_dispatch::call(policy, desc, - pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + auto mod_policy = launch::fork_policy( + policy.priority(), policy.stacksize(), hint); + + post_policy_dispatch::call( + HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } else { - post_policy_dispatch::call(policy, desc, - pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + auto mod_policy = launch::async_policy( + policy.priority(), policy.stacksize(), hint); + + post_policy_dispatch::call( + HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } } template - static void call(Policy const& policy, + static void call(Policy&& policy, hpx::threads::thread_description const& desc, F&& f, Ts&&... ts) { - call(policy, desc, threads::detail::get_self_or_default_pool(), - HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + call(HPX_FORWARD(Policy, policy), desc, + threads::detail::get_self_or_default_pool(), HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } }; } // namespace hpx::detail diff --git a/libs/core/execution/include/hpx/execution/detail/sync_launch_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/sync_launch_policy_dispatch.hpp index 90cf3acbc68d..4261788a86c3 100644 --- a/libs/core/execution/include/hpx/execution/detail/sync_launch_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/sync_launch_policy_dispatch.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -29,7 +30,7 @@ namespace hpx::detail { { // launch::sync execute inline template - HPX_FORCEINLINE static decltype(auto) call(Policy, F&& f, Ts&&... ts) + HPX_FORCEINLINE static decltype(auto) call(Policy&&, F&& f, Ts&&... ts) { try { @@ -51,7 +52,7 @@ namespace hpx::detail { { // launch::deferred execute inline template - HPX_FORCEINLINE static decltype(auto) call(Policy, F&& f, Ts&&... ts) + HPX_FORCEINLINE static decltype(auto) call(Policy&&, F&& f, Ts&&... ts) { try { @@ -82,13 +83,14 @@ namespace hpx::detail { if (policy == launch::sync) { return sync_launch_policy_dispatch::call( - policy, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); + HPX_MOVE(policy), HPX_FORWARD(F, f), + HPX_FORWARD(Ts, ts)...); } else if (policy == launch::deferred) { return sync_launch_policy_dispatch< - launch::deferred_policy>::call(policy, HPX_FORWARD(F, f), - HPX_FORWARD(Ts, ts)...); + launch::deferred_policy>::call(HPX_MOVE(policy), + HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } lcos::local::futures_factory p( @@ -96,6 +98,19 @@ namespace hpx::detail { if (hpx::detail::has_async_policy(policy)) { + // if one of the arguments is a future we play it conservatively + if constexpr (hpx::traits::is_future_any_v...>) + { + auto hint = policy.hint(); + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } + threads::thread_id_ref_type tid = p.post("sync_launch_policy_dispatch", policy, policy.priority()); diff --git a/libs/core/executors/include/hpx/executors/dataflow.hpp b/libs/core/executors/include/hpx/executors/dataflow.hpp index 51ac412acf48..38d5b696497e 100644 --- a/libs/core/executors/include/hpx/executors/dataflow.hpp +++ b/libs/core/executors/include/hpx/executors/dataflow.hpp @@ -348,13 +348,9 @@ namespace hpx::lcos::detail { // clang-format on HPX_FORCEINLINE void finalize(Executor&& exec, Futures_&& futures) { -#if defined(HPX_CUDA_VERSION) - std::forward(exec) -#else - HPX_FORWARD(Executor, exec) -#endif - .dataflow_finalize( - this, HPX_MOVE(func_), HPX_FORWARD(Futures_, futures)); + // uses std::forward for HPX_CUDA_VERSION + std::forward(exec).dataflow_finalize( + this, HPX_MOVE(func_), HPX_FORWARD(Futures_, futures)); } public: diff --git a/libs/core/futures/CMakeLists.txt b/libs/core/futures/CMakeLists.txt index 2bcc5cf132fe..f914351c40d4 100644 --- a/libs/core/futures/CMakeLists.txt +++ b/libs/core/futures/CMakeLists.txt @@ -7,6 +7,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(futures_headers + hpx/futures/detail/execute_thread.hpp hpx/futures/future.hpp hpx/futures/future_fwd.hpp hpx/futures/futures_factory.hpp @@ -51,7 +52,7 @@ set(futures_compat_headers ) # cmake-format: on -set(futures_sources future_data.cpp) +set(futures_sources detail/execute_thread.cpp future_data.cpp) include(HPX_AddModule) add_hpx_module( @@ -69,10 +70,12 @@ add_hpx_module( hpx_config hpx_errors hpx_functional + hpx_logging hpx_memory hpx_serialization hpx_synchronization hpx_timing + hpx_threading_base hpx_type_support CMAKE_SUBDIRS examples tests ) diff --git a/libs/core/futures/include/hpx/futures/detail/execute_thread.hpp b/libs/core/futures/include/hpx/futures/detail/execute_thread.hpp new file mode 100644 index 000000000000..30567b1cbfb4 --- /dev/null +++ b/libs/core/futures/include/hpx/futures/detail/execute_thread.hpp @@ -0,0 +1,15 @@ +// Copyright (c) 2019-2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +namespace hpx::threads::detail { + + HPX_CORE_EXPORT bool execute_thread(thread_id_ref_type thrd); +} // namespace hpx::threads::detail diff --git a/libs/core/futures/include/hpx/futures/detail/future_data.hpp b/libs/core/futures/include/hpx/futures/detail/future_data.hpp index 32fd7747c4a8..555504e54182 100644 --- a/libs/core/futures/include/hpx/futures/detail/future_data.hpp +++ b/libs/core/futures/include/hpx/futures/detail/future_data.hpp @@ -225,12 +225,14 @@ namespace hpx::lcos::detail { future_data_base() noexcept : state_(empty) + , runs_child_(threads::invalid_thread_id) { } explicit future_data_base(init_no_addref no_addref) noexcept : future_data_refcnt_base(no_addref) , state_(empty) + , runs_child_(threads::invalid_thread_id) { } @@ -318,12 +320,17 @@ namespace hpx::lcos::detail { virtual std::exception_ptr get_exception_ptr() const = 0; + protected: + // try to perform scoped execution of the associated thread (if any) + bool execute_thread(); + protected: mutable mutex_type mtx_; std::atomic state_; // current state completed_callback_vector_type on_completed_; local::detail::condition_variable cond_; // threads waiting in read + threads::thread_id_ref_type runs_child_; }; template @@ -472,6 +479,10 @@ namespace hpx::lcos::detail { "data has already been set for this future"); } + // reset runs_child_ thread id to avoid keeping the thread + // alive as long as the future + this->base_type::runs_child_.reset(); + // 26111: Caller failing to release lock 'this->mtx_' // 26115: Failing to release lock 'this->mtx_' // 26800: Use of a moved from object 'l' @@ -545,6 +556,10 @@ namespace hpx::lcos::detail { "data has already been set for this future"); } + // reset runs_child_ thread id to avoid keeping the thread + // alive as long as the future + this->base_type::runs_child_.reset(); + // 26111: Caller failing to release lock 'this->mtx_' // 26115: Failing to release lock 'this->mtx_' // 26800: Use of a moved from object 'l' @@ -827,10 +842,15 @@ namespace hpx::lcos::detail { using base_type::mtx_; public: - task_base() = default; + task_base() + : base_type() + , started_(false) + { + } explicit task_base(init_no_addref no_addref) noexcept : base_type(no_addref) + , started_(false) { } @@ -840,16 +860,22 @@ namespace hpx::lcos::detail { { this->do_run(); } + + // attempt to directly execute thread + this->execute_thread(); } // retrieving the value - result_type* get_result(error_code& ec = throws) override + util::unused_type* get_result_void(error_code& ec = throws) override { if (!started_test_and_set()) { this->do_run(); } - return this->future_data::get_result(ec); + + // attempt to directly execute thread + this->execute_thread(); + return this->base_type::get_result_void(ec); } // wait support @@ -859,57 +885,50 @@ namespace hpx::lcos::detail { { this->do_run(); } - return this->future_data::wait(ec); + + // attempt to directly execute thread + this->execute_thread(); + return this->base_type::wait(ec); } hpx::future_status wait_until( std::chrono::steady_clock::time_point const& abs_time, error_code& ec = throws) override { - if (!started_test()) + if (!started_test_and_set()) { - return hpx::future_status::deferred; //-V110 + this->do_run(); } - return this->future_data::wait_until(abs_time, ec); + + // attempt to directly execute thread + this->execute_thread(); + return this->base_type::wait_until(abs_time, ec); } - private: + protected: bool started_test() const noexcept { - std::lock_guard l(mtx_); - return started_; + return started_.load(std::memory_order_acquire); } - template - bool started_test_and_set_locked(Lock& l) + // returns whether this task was started before + bool started_test_and_set() { - HPX_ASSERT_OWNS_LOCK(l); - if (started_) + if (!started_.load(std::memory_order_relaxed)) { - return true; + return started_.exchange(true, std::memory_order_release); } - started_ = true; - return false; - } - - protected: - bool started_test_and_set() - { - std::lock_guard l(mtx_); - return started_test_and_set_locked(l); + return true; } void check_started() { - std::unique_lock l(mtx_); - if (started_) + if (started_test_and_set()) { - l.unlock(); HPX_THROW_EXCEPTION(hpx::error::task_already_started, "task_base::check_started", "this task has already been started"); } - started_ = true; } public: @@ -953,7 +972,7 @@ namespace hpx::lcos::detail { } protected: - bool started_ = false; + std::atomic started_; }; /////////////////////////////////////////////////////////////////////////// @@ -1030,7 +1049,7 @@ namespace hpx::lcos::detail { std::unique_lock l(mtx_); hpx::detail::try_catch_exception_ptr( [&]() { - if (!this->started_) + if (!this->started_test_and_set()) { HPX_THROW_THREAD_INTERRUPTED_EXCEPTION(); } @@ -1040,13 +1059,12 @@ namespace hpx::lcos::detail { return; // nothing we can do } + std::unique_lock l(mtx_); if (id_ != threads::invalid_thread_id) { // interrupt the executing thread threads::interrupt_thread(id_); - this->started_ = true; - l.unlock(); this->set_error(hpx::error::future_cancelled, "task_base::cancel", @@ -1062,7 +1080,7 @@ namespace hpx::lcos::detail { } }, [&](std::exception_ptr ep) { - this->started_ = true; + HPX_ASSERT(this->started_test()); this->set_exception(ep); std::rethrow_exception(HPX_MOVE(ep)); }); diff --git a/libs/core/futures/include/hpx/futures/futures_factory.hpp b/libs/core/futures/include/hpx/futures/futures_factory.hpp index c407198e140d..f61f0000e441 100644 --- a/libs/core/futures/include/hpx/futures/futures_factory.hpp +++ b/libs/core/futures/include/hpx/futures/futures_factory.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -70,6 +71,13 @@ namespace hpx::lcos::local { { } + task_object(task_object const&) = delete; + task_object(task_object&&) = delete; + task_object& operator=(task_object const&) = delete; + task_object& operator=(task_object&&) = delete; + + ~task_object() = default; + void do_run() noexcept override { hpx::intrusive_ptr this_(this); @@ -95,7 +103,22 @@ namespace hpx::lcos::local { threads::thread_id_ref_type post(threads::thread_pool_base* pool, char const* annotation, launch policy, error_code& ec) override { - this->check_started(); + if (this->started_test_and_set()) + { + return threads::invalid_thread_id; + } + + auto hint = policy.hint(); + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + if (!pool->get_scheduler()->supports_direct_execution()) + { + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); + } + } hpx::intrusive_ptr this_(this); if (policy == launch::fork) @@ -107,14 +130,51 @@ namespace hpx::lcos::local { threads::thread_description(f_, annotation), policy.priority(), threads::thread_schedule_hint( - static_cast(get_worker_thread_num())), + hpx::threads::thread_schedule_hint_mode::thread, + static_cast(get_worker_thread_num()), + hint.placement_mode(), hint.runs_as_child_mode()), policy.stacksize(), threads::thread_schedule_state::pending_do_not_schedule, true); + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + HPX_ASSERT( + this->runs_child_ == threads::invalid_thread_id); + this->runs_child_ = + threads::register_thread(data, pool, ec); + return this->runs_child_; + } + return threads::register_thread(data, pool, ec); } + if (hint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) + { + // create the thread without running it + threads::thread_init_data data( + threads::make_thread_function_nullary( + util::deferred_call( + &base_type::run_impl, std::move(this_))), + threads::thread_description(f_, annotation), + policy.priority(), policy.hint(), policy.stacksize(), + threads::thread_schedule_state::suspended, true); + + HPX_ASSERT(this->runs_child_ == threads::invalid_thread_id); + this->runs_child_ = + threads::register_thread(data, pool, ec); + + // now run the thread + threads::set_thread_state(this->runs_child_.noref(), + threads::thread_schedule_state::pending, + threads::thread_restart_state::signaled, + policy.priority(), true, ec); + + return this->runs_child_; + } + threads::thread_init_data data( threads::make_thread_function_nullary(util::deferred_call( &base_type::run_impl, HPX_MOVE(this_))), @@ -163,6 +223,14 @@ namespace hpx::lcos::local { { } + task_object_allocator(task_object_allocator const&) = delete; + task_object_allocator(task_object_allocator&&) = delete; + task_object_allocator& operator=( + task_object_allocator const&) = delete; + task_object_allocator& operator=(task_object_allocator&&) = delete; + + ~task_object_allocator() = default; + private: void destroy() noexcept override { @@ -230,6 +298,13 @@ namespace hpx::lcos::local { { } + task_object(task_object const&) = delete; + task_object(task_object&&) = delete; + task_object& operator=(task_object const&) = delete; + task_object& operator=(task_object&&) = delete; + + ~task_object() = default; + protected: // run in a separate thread threads::thread_id_ref_type post(threads::thread_pool_base* pool, @@ -237,13 +312,14 @@ namespace hpx::lcos::local { { if (exec_) { - this->check_started(); - - hpx::intrusive_ptr this_(this); - parallel::execution::post(*exec_, - util::deferred_call( - &base_type::run_impl, HPX_MOVE(this_)), - exec_->get_schedulehint(), annotation); + if (!this->started_test_and_set()) + { + hpx::intrusive_ptr this_(this); + parallel::execution::post(*exec_, + util::deferred_call( + &base_type::run_impl, HPX_MOVE(this_)), + exec_->get_schedulehint(), annotation); + } return threads::invalid_thread_id; } @@ -284,6 +360,15 @@ namespace hpx::lcos::local { : base_type(no_addref, HPX_MOVE(f)) { } + + cancelable_task_object(cancelable_task_object const&) = delete; + cancelable_task_object(cancelable_task_object&&) = delete; + cancelable_task_object& operator=( + cancelable_task_object const&) = delete; + cancelable_task_object& operator=( + cancelable_task_object&&) = delete; + + ~cancelable_task_object() = default; }; template @@ -325,6 +410,17 @@ namespace hpx::lcos::local { { } + cancelable_task_object_allocator( + cancelable_task_object_allocator const&) = delete; + cancelable_task_object_allocator( + cancelable_task_object_allocator&&) = delete; + cancelable_task_object_allocator& operator=( + cancelable_task_object_allocator const&) = delete; + cancelable_task_object_allocator& operator=( + cancelable_task_object_allocator&&) = delete; + + ~cancelable_task_object_allocator() = default; + private: void destroy() noexcept override { @@ -389,6 +485,15 @@ namespace hpx::lcos::local { : base_type(exec, no_addref, HPX_MOVE(f)) { } + + cancelable_task_object(cancelable_task_object const&) = delete; + cancelable_task_object(cancelable_task_object&&) = delete; + cancelable_task_object& operator=( + cancelable_task_object const&) = delete; + cancelable_task_object& operator=( + cancelable_task_object&&) = delete; + + ~cancelable_task_object() = default; }; } // namespace detail } // namespace hpx::lcos::local @@ -722,7 +827,7 @@ namespace hpx::lcos::local { launch policy = launch::async, error_code& ec = throws) const { return post(threads::detail::get_self_or_default_pool(), annotation, - policy, ec); + HPX_MOVE(policy), ec); } threads::thread_id_ref_type post(threads::thread_pool_base* pool, @@ -736,7 +841,7 @@ namespace hpx::lcos::local { "futures_factory invalid (has it been moved?)"); return threads::invalid_thread_id; } - return task_->post(pool, annotation, policy, ec); + return task_->post(pool, annotation, HPX_MOVE(policy), ec); } // This is the same as get_future, except that it moves the diff --git a/libs/core/futures/include/hpx/futures/packaged_continuation.hpp b/libs/core/futures/include/hpx/futures/packaged_continuation.hpp index 1fd4d00cd342..0130bb7f55f6 100644 --- a/libs/core/futures/include/hpx/futures/packaged_continuation.hpp +++ b/libs/core/futures/include/hpx/futures/packaged_continuation.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -225,12 +226,12 @@ namespace hpx::lcos::detail { { ensure_started(); + HPX_ASSERT(!this->runs_child_); + hpx::intrusive_ptr this_(this); hpx::threads::thread_description desc(f_, "async"); - - spawner( + this->runs_child_ = spawner( [this_ = HPX_MOVE(this_), f = HPX_MOVE(f)]() mutable -> void { - reset_id r(*this_); this_->template run_impl(HPX_MOVE(f)); }, desc); diff --git a/libs/core/futures/include/hpx/futures/traits/is_future.hpp b/libs/core/futures/include/hpx/futures/traits/is_future.hpp index 5c0941f7a1c1..feaabafa8db9 100644 --- a/libs/core/futures/include/hpx/futures/traits/is_future.hpp +++ b/libs/core/futures/include/hpx/futures/traits/is_future.hpp @@ -66,13 +66,12 @@ namespace hpx::traits { /////////////////////////////////////////////////////////////////////////// template - struct is_future_any : hpx::util::any_of...> - { - }; + using is_future_any = hpx::util::any_of...>; template inline constexpr bool is_future_any_v = is_future_any::value; + /////////////////////////////////////////////////////////////////////////// template struct is_ref_wrapped_future : std::false_type { diff --git a/libs/core/futures/src/detail/execute_thread.cpp b/libs/core/futures/src/detail/execute_thread.cpp new file mode 100644 index 000000000000..7bcd763eca72 --- /dev/null +++ b/libs/core/futures/src/detail/execute_thread.cpp @@ -0,0 +1,210 @@ +// Copyright (c) 2019-2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace hpx::threads::detail { + +#if !defined(HPX_HAVE_THREADS_GET_STACK_POINTER) + /////////////////////////////////////////////////////////////////////////// + // reuse the continuation recursion count here as well + struct execute_thread_recursion_count + { + execute_thread_recursion_count() noexcept + : count_(threads::get_continuation_recursion_count()) + { + ++count_; + } + ~execute_thread_recursion_count() noexcept + { + --count_; + } + + std::size_t& count_; + }; +#endif + + // make sure thread invocation does not recurse deeper than allowed + HPX_FORCEINLINE coroutine_type::result_type handle_execute_thread( + thread_id_type const& thrd) + { + auto* thrdptr = get_thread_id_data(thrd); + + // We need to run the completion on a new thread + HPX_ASSERT(nullptr != hpx::threads::get_self_ptr()); + +#if defined(HPX_HAVE_THREADS_GET_STACK_POINTER) + bool recurse_asynchronously = + !this_thread::has_sufficient_stack_space(); +#else + execute_thread_recursion_count cnt; + bool recurse_asynchronously = + cnt.count_ > HPX_CONTINUATION_MAX_RECURSION_DEPTH; +#endif + if (!recurse_asynchronously) + { + // directly execute continuation on this thread + return thrdptr->invoke_directly(); + } + + LTM_(error).format( + "handle_execute_thread: couldn't directly execute thread({}), " + "description({})", + thrdptr, thrdptr->get_description()); + + return {thread_schedule_state::pending, invalid_thread_id}; + } + + bool execute_thread(thread_id_ref_type thrd) + { + auto* thrdptr = get_thread_id_data(thrd); + thread_state state = thrdptr->get_state(); + thread_schedule_state state_val = state.state(); + + // the given thread can be executed inline if its state is 'pending' + // (i.e. not running and not finished running) + if (state_val != thread_schedule_state::pending) + { + return false; + } + + bool reschedule = false; + + // don't directly run any threads that have started running 'normally' + // and were suspended afterwards + if (thrdptr->runs_as_child()) + { + LTM_(error).format( + "execute_thread: attempting to directly execute thread({}), " + "description({}), runs_as_child({})", + thrdptr, thrdptr->get_description(), + thrdptr->runs_as_child(std::memory_order_relaxed)); + + // tries to set state to active (only if state is still the same as + // 'state') + switch_status thrd_stat(thrd, state); + if (HPX_UNLIKELY(!thrd_stat.is_valid())) + { + // state change failed + LTM_(error).format( + "execute_thread: couldn't directly execute " + "thread({}), description({}), state change failed", + thrdptr, thrdptr->get_description()); + + // switch_status will not reset thread state + return false; + } + + HPX_ASSERT( + thrdptr->get_state().state() == thread_schedule_state::active); + + if (HPX_UNLIKELY( + thrd_stat.get_previous() != thread_schedule_state::pending)) + { + // thread was not pending + LTM_(error).format( + "execute_thread: couldn't directly execute " + "thread({}), description({}), thread not pending", + thrdptr, thrdptr->get_description()); + + // switch_status will reset state to what it was before + return false; + } + + // check again, making sure the state has not changed in the mean + // time + if (thrdptr->runs_as_child()) + { +#if defined(HPX_HAVE_APEX) + // get the APEX data pointer, in case we are resuming the thread + // and have to restore any leaf timers from direct actions, etc. + util::external_timer::scoped_timer profiler( + thrdptr->get_timer_data()); + + thrd_stat = handle_execute_thread(thrd.noref()); + + thread_schedule_state s = thrd_stat.get_previous(); + if (s == thread_schedule_state::terminated || + s == thread_schedule_state::deleted) + { + profiler.stop(); + + // just in case, clean up the now dead pointer. + thrdptr->set_timer_data(nullptr); + } + else + { + profiler.yield(); + } +#else + thrd_stat = handle_execute_thread(thrd.noref()); +#endif + } + else + { + // reschedule thread as it could have been dropped on the floor + // by the scheduler while the status was set to active here + reschedule = true; + } + + // store and retrieve the new state in the thread + if (HPX_LIKELY(thrd_stat.store_state(state))) + { + // direct execution doesn't support specifying the next + // thread to execute + HPX_ASSERT(thrd_stat.get_next_thread() == nullptr); + + state_val = state.state(); + if (state_val == thread_schedule_state::pending) + { + // explicitly reschedule thread as it was not executed + // directly + reschedule = true; + } + } + + LTM_(error).format("execute_thread: directly executed thread({}), " + "description({}), returned state({})", + thrdptr, thrdptr->get_description(), state_val); + + // any exception thrown from the thread will reset its state at this + // point + } + + if (reschedule) + { + LTM_(error).format( + "execute_thread: rescheduling thread after failing to directly " + "execute thread({}), description({})", + thrdptr, thrdptr->get_description()); + + set_thread_state(thrd.noref(), thread_schedule_state::pending, + thread_restart_state::signaled); + auto* scheduler = thrdptr->get_scheduler_base(); + scheduler->schedule_thread_last(thrd, thread_schedule_hint()); + } + + HPX_ASSERT(state_val != thread_schedule_state::terminated); + return state_val == thread_schedule_state::deleted; + } +} // namespace hpx::threads::detail diff --git a/libs/core/futures/src/future_data.cpp b/libs/core/futures/src/future_data.cpp index 220e337d2020..4d82818ee419 100644 --- a/libs/core/futures/src/future_data.cpp +++ b/libs/core/futures/src/future_data.cpp @@ -8,13 +8,16 @@ #include #include +#include #include #include #include #include #include +#include #include #include +#include #include #include @@ -95,8 +98,68 @@ namespace hpx::lcos::detail { } /////////////////////////////////////////////////////////////////////////// - future_data_base::~future_data_base() = - default; + future_data_base::~future_data_base() + { + if (runs_child_ != threads::invalid_thread_id) + { + auto* thrd = get_thread_id_data(runs_child_); + (void) thrd; + LTM_(debug).format( + "task_object::~task_object({}), description({}): " + "destroy runs_as_child thread", + thrd, thrd->get_description(), thrd->get_thread_phase()); + + runs_child_ = threads::invalid_thread_id; + } + } + + // try to performed scoped execution of the associated thread (if any) + bool future_data_base::execute_thread() + { + // we try to directly execute the thread exactly once + if (!runs_child_) + { + return false; + } + + auto state = this->state_.load(std::memory_order_acquire); + if (state != this->empty) + { + return false; + } + + // this thread would block on the future + threads::thread_id_ref_type runs_child = runs_child_; + + auto* thrd = get_thread_id_data(runs_child); + HPX_UNUSED(thrd); // might be unused + + LTM_(debug).format("task_object::get_result_void: attempting to " + "directly execute child({}), description({})", + thrd, thrd->get_description()); + + if (threads::detail::execute_thread(HPX_MOVE(runs_child))) + { + // don't try running this twice + runs_child_.reset(); + + LTM_(debug).format("task_object::get_result_void: successfully " + "directly executed child({}), description({})", + thrd, thrd->get_description()); + + // thread terminated, mark as being destroyed + HPX_ASSERT(thrd->get_state().state() == + threads::thread_schedule_state::deleted); + + return true; + } + + LTM_(debug).format("task_object::get_result_void: failed to " + "directly execute child({}), description({})", + thrd, thrd->get_description()); + + return false; + } static util::unused_type unused_; @@ -111,8 +174,8 @@ namespace hpx::lcos::detail { return nullptr; } - // No locking is required. Once a future has been made ready, which - // is a postcondition of wait, either: + // No locking is required. Once a future has been made ready, which is a + // postcondition of wait, either: // // - there is only one writer (future), or // - there are multiple readers only (shared_future, lock hurts @@ -139,16 +202,15 @@ namespace hpx::lcos::detail { return nullptr; } - // the thread has been re-activated by one of the actions - // supported by this promise (see promise::set_event - // and promise::set_exception). + // the thread has been re-activated by one of the actions supported by + // this promise (see promise::set_event and promise::set_exception). if (s == exception) { auto const* exception_ptr = static_cast(storage); - // an error has been reported in the meantime, throw or set - // the error code + // an error has been reported in the meantime, throw or set the + // error code if (&ec == &throws) { std::rethrow_exception(*exception_ptr); @@ -171,8 +233,8 @@ namespace hpx::lcos::detail { on_completed(); }, [&](std::exception_ptr ep) { - // If the completion handler throws an exception, there's nothing - // we can do, report the exception and terminate. + // If the completion handler throws an exception, there's + // nothing we can do, report the exception and terminate. if (run_on_completed_error_handler) { run_on_completed_error_handler(HPX_MOVE(ep)); @@ -193,15 +255,14 @@ namespace hpx::lcos::detail { } } - // make sure continuation invocation does not recurse deeper than - // allowed + // make sure continuation invocation does not recurse deeper than allowed template void future_data_base::handle_on_completed( Callback&& on_completed) { - // We need to run the completion on a new thread if we are on a - // non HPX thread. + // We need to run the completion on a new thread if we are on a non HPX + // thread. #if defined(HPX_HAVE_THREADS_GET_STACK_POINTER) bool recurse_asynchronously = !this_thread::has_sufficient_stack_space(); @@ -219,7 +280,6 @@ namespace hpx::lcos::detail { else { // re-spawn continuation on a new thread - hpx::detail::try_catch_exception_ptr( [&]() { // clang-format off diff --git a/libs/core/futures/tests/unit/CMakeLists.txt b/libs/core/futures/tests/unit/CMakeLists.txt index ecebaf5af73a..e4e818c67db2 100644 --- a/libs/core/futures/tests/unit/CMakeLists.txt +++ b/libs/core/futures/tests/unit/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2007-2017 Hartmut Kaiser +# Copyright (c) 2007-2023 Hartmut Kaiser # Copyright (c) 2011-2012 Bryce Adelstein-Lelbach # # SPDX-License-Identifier: BSL-1.0 @@ -6,6 +6,7 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) set(tests + direct_scoped_execution future future_ref future_then diff --git a/libs/core/futures/tests/unit/direct_scoped_execution.cpp b/libs/core/futures/tests/unit/direct_scoped_execution.cpp new file mode 100644 index 000000000000..ba0205d47626 --- /dev/null +++ b/libs/core/futures/tests/unit/direct_scoped_execution.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2011 Bryce Lelbach +// Copyright (c) 2021 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +bool use_scoping = true; + +/////////////////////////////////////////////////////////////////////////////// +std::uint64_t fibonacci(std::uint64_t n) +{ + if (n < 2) + return n; + + hpx::threads::thread_schedule_hint hint; + hint.runs_as_child_mode(use_scoping ? + hpx::threads::thread_execution_hint::run_as_child : + hpx::threads::thread_execution_hint::none); + + auto exec = hpx::execution::experimental::with_hint( + hpx::execution::parallel_executor{}, hint); + + hpx::future n1 = hpx::async(exec, fibonacci, n - 1); + std::uint64_t const n2 = fibonacci(n - 2); + + // wait for the Futures to return their values + return n1.get() + n2; +} + +/////////////////////////////////////////////////////////////////////////////// +int hpx_main(hpx::program_options::variables_map& vm) +{ + // extract command line argument, i.e. fib(N) + std::uint64_t const n = vm["n-value"].as(); + + use_scoping = vm.count("non-scoped") == 0; + + { + // Keep track of the time required to execute. + hpx::chrono::high_resolution_timer const t; + + std::uint64_t const r = fibonacci(n); + + char const* fmt = "fibonacci({1}) == {2}\nelapsed time: {3} [s]\n"; + hpx::util::format_to(std::cout, fmt, n, r, t.elapsed()); + } + + return hpx::finalize(); // Handles HPX shutdown +} + +/////////////////////////////////////////////////////////////////////////////// +int main(int argc, char* argv[]) +{ + // Configure application-specific options + hpx::program_options::options_description desc_commandline( + "Usage: " HPX_APPLICATION_STRING " [options]"); + + // clang-format off + desc_commandline.add_options() + ("n-value", + hpx::program_options::value()->default_value(10), + "n value for the Fibonacci function") + ("non-scoped", "run created threads without scoping"); + // clang-format on + + // use LIFO scheduler + hpx::init_params params; + params.desc_cmdline = desc_commandline; + + // Initialize and run HPX + hpx::init(argc, argv, params); + + return hpx::util::report_errors(); +} diff --git a/libs/core/init_runtime_local/src/init_runtime_local.cpp b/libs/core/init_runtime_local/src/init_runtime_local.cpp index 4fd1bbcd70aa..3ddcd74bf3b1 100644 --- a/libs/core/init_runtime_local/src/init_runtime_local.cpp +++ b/libs/core/init_runtime_local/src/init_runtime_local.cpp @@ -500,6 +500,9 @@ namespace hpx { return result; } + rp.assign_cores(hpx::util::get_entry_as( + cmdline.rtcfg_, "hpx.first_used_core", 0)); + // If thread_pools initialization in user main if (params.rp_callback) { diff --git a/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner.hpp b/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner.hpp index f26e1907010a..da52bd4160fe 100644 --- a/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner.hpp +++ b/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner.hpp @@ -205,6 +205,9 @@ namespace hpx::resource { // resource partitioner called in hpx_init HPX_CORE_EXPORT void configure_pools(); + // Local runtime only: initialize first_core and pus_needed + HPX_CORE_EXPORT void assign_cores(std::size_t first_core); + private: detail::partitioner& partitioner_; }; diff --git a/libs/core/resource_partitioner/src/detail_partitioner.cpp b/libs/core/resource_partitioner/src/detail_partitioner.cpp index 245fa329267f..68fb708814e7 100644 --- a/libs/core/resource_partitioner/src/detail_partitioner.cpp +++ b/libs/core/resource_partitioner/src/detail_partitioner.cpp @@ -948,6 +948,8 @@ namespace hpx::resource::detail { affinity_data_ = affinity_data; fill_topology_vectors(); + + pus_needed_ = assign_cores(0); } scheduler_function partitioner::get_pool_creator(std::size_t index) const diff --git a/libs/core/resource_partitioner/src/partitioner.cpp b/libs/core/resource_partitioner/src/partitioner.cpp index fb9f66d315dd..eaf1790d5277 100644 --- a/libs/core/resource_partitioner/src/partitioner.cpp +++ b/libs/core/resource_partitioner/src/partitioner.cpp @@ -228,4 +228,10 @@ namespace hpx::resource { { partitioner_.configure_pools(); } + + // Local runtime only: initialize first_core and pus_needed + void partitioner::assign_cores(std::size_t first_core) + { + partitioner_.assign_cores(first_core); + } } // namespace hpx::resource diff --git a/libs/core/resource_partitioner/tests/unit/cross_pool_injection.cpp b/libs/core/resource_partitioner/tests/unit/cross_pool_injection.cpp index a8042d88fb9c..a86e064014b4 100644 --- a/libs/core/resource_partitioner/tests/unit/cross_pool_injection.cpp +++ b/libs/core/resource_partitioner/tests/unit/cross_pool_injection.cpp @@ -35,11 +35,9 @@ std::size_t num_pools = 0; // dummy function we will call using async void dummy_task(std::size_t n) { - // no other work can take place on this thread whilst it sleeps - std::this_thread::sleep_for(std::chrono::milliseconds(n)); - // - for (std::size_t i(0); i < n; ++i) + if (n != 0) { + hpx::this_thread::sleep_for(std::chrono::milliseconds(n)); } } @@ -94,10 +92,13 @@ int hpx_main() hpx::threads::thread_priority::default_); } - // randomly create tasks that run on a random pool - // attach continuations to them that run on different - // random pools + // randomly create tasks that run on a random pool attach continuations to + // them that run on different random pools int const loops = 1000; + + std::vector> futures; + futures.resize(loops); + // std::cout << "1: Starting HP " << loops << std::endl; std::atomic counter(loops); @@ -109,15 +110,14 @@ int hpx_main() auto& exec_1 = HP_executors[random_pool_1]; auto& exec_2 = HP_executors[random_pool_2]; auto f1 = hpx::async(exec_1, &dummy_task, 0); - auto f2 = f1.then(exec_2, [=, &counter](hpx::future&&) { + futures[i] = f1.then(exec_2, [=, &counter](hpx::future&&) { dummy_task(0); --counter; }); } - do - { - hpx::this_thread::yield(); - } while (counter > 0); + + hpx::wait_all(futures); + HPX_TEST(counter == 0); std::cout << "2: Starting NP " << loops << std::endl; counter = loops; @@ -129,15 +129,14 @@ int hpx_main() auto& exec_3 = NP_executors[random_pool_1]; auto& exec_4 = NP_executors[random_pool_2]; auto f3 = hpx::async(exec_3, &dummy_task, 0); - auto f4 = f3.then(exec_4, [=, &counter](hpx::future&&) { + futures[i] = f3.then(exec_4, [=, &counter](hpx::future&&) { dummy_task(0); --counter; }); } - do - { - hpx::this_thread::yield(); - } while (counter > 0); + + hpx::wait_all(futures); + HPX_TEST(counter == 0); std::cout << "3: Starting HP->NP " << loops << std::endl; counter = loops; @@ -149,15 +148,14 @@ int hpx_main() auto& exec_5 = HP_executors[random_pool_1]; auto& exec_6 = NP_executors[random_pool_2]; auto f5 = hpx::async(exec_5, &dummy_task, 0); - auto f6 = f5.then(exec_6, [=, &counter](hpx::future&&) { + futures[i] = f5.then(exec_6, [=, &counter](hpx::future&&) { dummy_task(0); --counter; }); } - do - { - hpx::this_thread::yield(); - } while (counter > 0); + + hpx::wait_all(futures); + HPX_TEST(counter == 0); std::cout << "4: Starting NP->HP " << loops << std::endl; counter = loops; @@ -169,15 +167,14 @@ int hpx_main() auto& exec_7 = NP_executors[random_pool_1]; auto& exec_8 = HP_executors[random_pool_2]; auto f7 = hpx::async(exec_7, &dummy_task, 0); - auto f8 = f7.then(exec_8, [=, &counter](hpx::future&&) { + futures[i] = f7.then(exec_8, [=, &counter](hpx::future&&) { dummy_task(0); --counter; }); } - do - { - hpx::this_thread::yield(); - } while (counter > 0); + + hpx::wait_all(futures); + HPX_TEST(counter == 0); std::cout << "5: Starting suspending " << loops << std::endl; counter = loops; @@ -189,19 +186,23 @@ int hpx_main() auto& exec_7 = NP_executors[random_pool_1]; auto& exec_8 = HP_executors[random_pool_2]; // random delay up to 5 milliseconds - std::size_t delay = st_rand(0, 5); - auto f7 = hpx::async(exec_7, &dummy_task, delay); - auto f8 = hpx::async(exec_8, [f7(std::move(f7)), &counter]() mutable { - // if f7 is not ready then f8 will suspend itself on get - f7.get(); - dummy_task(0); - --counter; - }); + //std::size_t delay = st_rand(0, 5); + auto f7 = hpx::async(exec_7, &dummy_task, 0); + + // we pass f7 as an argument to prevent direct execution from happening + futures[i] = hpx::async( + exec_8, + [&counter](auto&& f7) mutable { + // if f7 is not ready then f8 will suspend itself on get + f7.get(); + dummy_task(0); + --counter; + }, + std::move(f7)); } - do - { - hpx::this_thread::yield(); - } while (counter > 0); + + hpx::wait_all(futures); + HPX_TEST(counter == 0); return hpx::local::finalize(); } @@ -216,7 +217,7 @@ void init_resource_partitioner_handler(hpx::resource::partitioner& rp, rp.set_default_pool_name("pool-0"); auto seed = std::time(nullptr); - std::srand(seed); + std::srand((unsigned int) seed); std::cout << "Random seed " << seed << std::endl; // create N pools @@ -224,6 +225,7 @@ void init_resource_partitioner_handler(hpx::resource::partitioner& rp, std::string pool_name; std::size_t threads_remaining = max_threads; std::size_t threads_in_pool = 0; + // create pools randomly and add a random number of PUs to each pool for (hpx::resource::numa_domain const& d : rp.numa_domains()) { @@ -234,12 +236,15 @@ void init_resource_partitioner_handler(hpx::resource::partitioner& rp, if (threads_in_pool == 0) { // pick a random number of threads less than the max - threads_in_pool = 1 + - st_rand( - 0, ((std::max)(std::size_t(1), max_threads / 2))); + threads_in_pool = + (std::max)(std::size_t(1), st_rand(0, max_threads / 2)); pool_name = "pool-" + std::to_string(num_pools); rp.create_thread_pool(pool_name, policy); num_pools++; + + std::cout << "Created pool: " << pool_name << "\n"; + std::cout << " threads_in_pool: " << threads_in_pool + << "\n"; } std::cout << "Added pu " << p.id() << " to " << pool_name << "\n"; @@ -247,7 +252,11 @@ void init_resource_partitioner_handler(hpx::resource::partitioner& rp, threads_in_pool--; if (threads_remaining-- == 0) { - std::cerr << "This should not happen!" << std::endl; + std::cerr << "This should not happen!\n"; + std::cerr << " max_threads: " << max_threads << "\n"; + std::cerr << " num_pools: " << num_pools << "\n"; + std::cerr << " threads_in_pool: " << threads_in_pool + << "\n"; } } } diff --git a/libs/core/resource_partitioner/tests/unit/scheduler_binding_check.cpp b/libs/core/resource_partitioner/tests/unit/scheduler_binding_check.cpp index 1db907303f7e..7c483d49ebad 100644 --- a/libs/core/resource_partitioner/tests/unit/scheduler_binding_check.cpp +++ b/libs/core/resource_partitioner/tests/unit/scheduler_binding_check.cpp @@ -61,14 +61,16 @@ void threadLoop() HPX_TEST_EQ(thread_actual, thread_expected); }; - std::size_t threads = hpx::get_num_worker_threads(); // launch tasks on threads using numbering 0,1,2,3...0,1,2,3 + std::size_t threads = hpx::get_num_worker_threads(); for (std::size_t i = 0; i < iterations; ++i) { auto exec = hpx::execution::parallel_executor( hpx::threads::thread_priority::bound, hpx::threads::thread_stacksize::default_, - hpx::threads::thread_schedule_hint(std::int16_t(i % threads))); + hpx::threads::thread_schedule_hint( + hpx::threads::thread_schedule_hint_mode::thread, + std::int16_t(i % threads))); hpx::async(exec, f, i, (i % threads)).get(); } diff --git a/libs/core/runtime_local/include/hpx/runtime_local/runtime_local.hpp b/libs/core/runtime_local/include/hpx/runtime_local/runtime_local.hpp index 96383a9ccb75..801c8a859972 100644 --- a/libs/core/runtime_local/include/hpx/runtime_local/runtime_local.hpp +++ b/libs/core/runtime_local/include/hpx/runtime_local/runtime_local.hpp @@ -378,15 +378,9 @@ namespace hpx { virtual std::string get_locality_name() const; - virtual std::uint32_t assign_cores(std::string const&, std::uint32_t) - { - return std::uint32_t(-1); - } + virtual std::uint32_t assign_cores(std::string const&, std::uint32_t); - virtual std::uint32_t assign_cores() - { - return std::uint32_t(-1); - } + virtual std::uint32_t assign_cores(); hpx::program_options::options_description const& get_app_options() const { diff --git a/libs/core/runtime_local/src/custom_exception_info.cpp b/libs/core/runtime_local/src/custom_exception_info.cpp index b2f9594c3d0a..588ee1b6e0f4 100644 --- a/libs/core/runtime_local/src/custom_exception_info.cpp +++ b/libs/core/runtime_local/src/custom_exception_info.cpp @@ -75,9 +75,11 @@ namespace hpx { { strm << full_build_string(); - std::string const* env = xi.get(); - if (env && !env->empty()) + if (std::string const* env = xi.get(); + env && !env->empty()) + { strm << "{env}: " << *env; + } } if (verbosity >= 1) @@ -90,10 +92,11 @@ namespace hpx { strm << "{stack-trace}: " << *back_trace << "\n"; } - std::uint32_t const* locality = - xi.get(); - if (locality) + if (std::uint32_t const* locality = + xi.get()) + { strm << "{locality-id}: " << *locality << "\n"; + } std::string const* hostname_ = xi.get(); @@ -106,14 +109,15 @@ namespace hpx { bool thread_info = false; char const* const thread_prefix = "{os-thread}: "; - std::size_t const* shepherd = xi.get(); - if (shepherd && std::size_t(-1) != *shepherd) + if (std::size_t const* shepherd = + xi.get(); + shepherd && static_cast(-1) != *shepherd) { strm << thread_prefix << *shepherd; thread_info = true; } - std::string thread_name = hpx::get_thread_name(); + std::string const thread_name = hpx::get_thread_name(); if (!thread_info) strm << thread_prefix; else @@ -133,31 +137,28 @@ namespace hpx { if (thread_description && !thread_description->empty()) strm << "{thread-description}: " << *thread_description << "\n"; - std::string const* state = xi.get(); - if (state) + if (std::string const* state = xi.get()) strm << "{state}: " << *state << "\n"; - std::string const* auxinfo = xi.get(); - if (auxinfo) + if (std::string const* auxinfo = + xi.get()) + { strm << "{auxinfo}: " << *auxinfo << "\n"; + } } - std::string const* file = xi.get(); - if (file) + if (std::string const* file = xi.get()) strm << "{file}: " << *file << "\n"; - long const* line = xi.get(); - if (line) + if (long const* line = xi.get()) strm << "{line}: " << *line << "\n"; - std::string const* function = xi.get(); - if (function) + if (std::string const* function = xi.get()) strm << "{function}: " << *function << "\n"; // Try a cast to std::exception - this should handle boost.system // error codes in addition to the standard library exceptions. - std::exception const* se = dynamic_cast(&xi); - if (se) + if (auto const* se = dynamic_cast(&xi)) strm << "{what}: " << se->what() << "\n"; return strm.str(); @@ -174,14 +175,13 @@ namespace hpx::util { #if defined(HPX_HAVE_STACKTRACES) if (frames_no == 0) { - return std::string(); + return {}; } backtrace bt(frames_no); // avoid infinite recursion on handling errors - auto* self = threads::get_self_ptr(); - if (nullptr == self || + if (auto const* self = threads::get_self_ptr(); nullptr == self || self->get_thread_id() == threads::invalid_thread_id) { return bt.trace(); @@ -191,7 +191,7 @@ namespace hpx::util { [&bt]() { return bt.trace(); }); error_code ec(throwmode::lightweight); - threads::thread_id_ref_type tid = + threads::thread_id_ref_type const tid = p.post("hpx::util::trace_on_new_stack", launch::fork_policy(threads::thread_priority::default_, threads::thread_stacksize::medium), @@ -395,20 +395,20 @@ namespace hpx::detail { hpx::exception_info custom_exception_info(std::string const& func, std::string const& file, long line, std::string const& auxinfo) { - std::int64_t pid = ::getpid(); + std::int64_t const pid = ::getpid(); - std::size_t const trace_depth = + auto const trace_depth = util::from_string(get_config_entry( "hpx.trace_depth", HPX_HAVE_THREAD_BACKTRACE_DEPTH)); - std::string back_trace(hpx::util::trace_on_new_stack(trace_depth)); + std::string const back_trace( + hpx::util::trace_on_new_stack(trace_depth)); std::string state_name("not running"); std::string hostname; - hpx::runtime* rt = get_runtime_ptr(); - if (rt) + if (hpx::runtime const* rt = get_runtime_ptr()) { - state rts_state = rt->get_state(); + state const rts_state = rt->get_state(); state_name = get_runtime_state_name(rts_state); if (rts_state >= state::initialized && rts_state < state::stopped) @@ -420,13 +420,13 @@ namespace hpx::detail { // if this is not a HPX thread we do not need to query neither for // the shepherd thread nor for the thread id error_code ec(throwmode::lightweight); - std::uint32_t node = get_locality_id(ec); + std::uint32_t const node = get_locality_id(ec); - std::size_t shepherd = std::size_t(-1); + auto shepherd = static_cast(-1); threads::thread_id_type thread_id; threads::thread_description thread_name; - threads::thread_self* self = threads::get_self_ptr(); + threads::thread_self const* self = threads::get_self_ptr(); if (nullptr != self) { if (threads::threadmanager_is(hpx::state::running)) @@ -436,8 +436,8 @@ namespace hpx::detail { thread_name = threads::get_thread_description(thread_id); } - std::string env(get_execution_environment()); - std::string config(configuration_string()); + std::string const env(get_execution_environment()); + std::string const config(configuration_string()); return hpx::exception_info().set( hpx::detail::throw_stacktrace(back_trace), @@ -463,15 +463,17 @@ namespace hpx { std::string const* hostname_ = xi.get(); if (hostname_ && !hostname_->empty()) return *hostname_; - return std::string(); + return {}; } /// Return the locality where the exception was thrown. std::uint32_t get_error_locality_id(hpx::exception_info const& xi) { - std::uint32_t const* locality = xi.get(); - if (locality) + if (std::uint32_t const* locality = + xi.get()) + { return *locality; + } // same as naming::invalid_locality_id return ~static_cast(0); @@ -481,8 +483,7 @@ namespace hpx { /// exception was thrown. std::int64_t get_error_process_id(hpx::exception_info const& xi) { - std::int64_t const* pid_ = xi.get(); - if (pid_) + if (std::int64_t const* pid_ = xi.get()) return *pid_; return -1; } @@ -491,9 +492,11 @@ namespace hpx { /// was thrown. std::string get_error_env(hpx::exception_info const& xi) { - std::string const* env = xi.get(); - if (env && !env->empty()) + if (std::string const* env = xi.get(); + env && !env->empty()) + { return *env; + } return ""; } @@ -501,61 +504,76 @@ namespace hpx { /// Return the stack backtrace at the point the exception was thrown. std::string get_error_backtrace(hpx::exception_info const& xi) { - std::string const* back_trace = xi.get(); - if (back_trace && !back_trace->empty()) + if (std::string const* back_trace = + xi.get(); + back_trace && !back_trace->empty()) + { return *back_trace; + } - return std::string(); + return {}; } /// Return the sequence number of the OS-thread used to execute HPX-threads /// from which the exception was thrown. std::size_t get_error_os_thread(hpx::exception_info const& xi) { - std::size_t const* shepherd = xi.get(); - if (shepherd && std::size_t(-1) != *shepherd) + if (std::size_t const* shepherd = xi.get(); + shepherd && static_cast(-1) != *shepherd) + { return *shepherd; - return std::size_t(-1); + } + return static_cast(-1); } /// Return the unique thread id of the HPX-thread from which the exception /// was thrown. std::size_t get_error_thread_id(hpx::exception_info const& xi) { - std::size_t const* thread_id = xi.get(); - if (thread_id && *thread_id) + if (std::size_t const* thread_id = + xi.get(); + thread_id && *thread_id) + { return *thread_id; - return std::size_t(-1); + } + return static_cast(-1); } /// Return any addition thread description of the HPX-thread from which the /// exception was thrown. std::string get_error_thread_description(hpx::exception_info const& xi) { - std::string const* thread_description = - xi.get(); - if (thread_description && !thread_description->empty()) + if (std::string const* thread_description = + xi.get(); + thread_description && !thread_description->empty()) + { return *thread_description; - return std::string(); + } + return {}; } /// Return the HPX configuration information point from which the /// exception was thrown. std::string get_error_config(hpx::exception_info const& xi) { - std::string const* config_info = xi.get(); - if (config_info && !config_info->empty()) + if (std::string const* config_info = + xi.get(); + config_info && !config_info->empty()) + { return *config_info; - return std::string(); + } + return {}; } /// Return the HPX runtime state information at which the exception was /// thrown. std::string get_error_state(hpx::exception_info const& xi) { - std::string const* state_info = xi.get(); - if (state_info && !state_info->empty()) + if (std::string const* state_info = xi.get(); + state_info && !state_info->empty()) + { return *state_info; - return std::string(); + } + return {}; } } // namespace hpx diff --git a/libs/core/runtime_local/src/runtime_local.cpp b/libs/core/runtime_local/src/runtime_local.cpp index b002181f7a8c..94626ecfb40c 100644 --- a/libs/core/runtime_local/src/runtime_local.cpp +++ b/libs/core/runtime_local/src/runtime_local.cpp @@ -651,6 +651,17 @@ namespace hpx { return "console"; } + std::uint32_t runtime::assign_cores(std::string const&, std::uint32_t) + { + return 0; + } + + std::uint32_t runtime::assign_cores() + { + return static_cast( + hpx::resource::get_partitioner().assign_cores(0)); + } + /////////////////////////////////////////////////////////////////////////// threads::policies::callback_notifier::on_startstop_type get_thread_on_start_func() diff --git a/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp index bd754280f0f5..19b7010173fc 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp @@ -1632,7 +1632,7 @@ namespace hpx::threads::policies { queues_[num_thread].data_->on_error(num_thread, e); } - void reset_thread_distribution() override + void reset_thread_distribution() noexcept override { curr_queue_.store(0, std::memory_order_release); } diff --git a/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp b/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp index ec9e4887bc4f..4837cbe5623f 100644 --- a/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp @@ -392,7 +392,9 @@ namespace hpx::threads::policies { void create_thread(thread_init_data& data, thread_id_ref_type* tid, std::size_t thread_num, error_code& ec) { - if (thread_num != thread_num_) + if (thread_num != thread_num_ && + (data.initial_state == thread_schedule_state::pending || + data.initial_state == thread_schedule_state::pending_boost)) { data.run_now = false; } diff --git a/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp index 9aed2430261e..d37ffb9c915b 100644 --- a/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp @@ -299,6 +299,11 @@ namespace hpx::threads::policies { // safety check that task was created by this thread/scheduler HPX_ASSERT(data.scheduler_base == this); + // this scheduler assumes thread bindings for the scheduled + // threads in certain ways preventing direct execution + HPX_ASSERT(data.schedulehint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::none); + std::size_t const local_num = local_thread_number(); std::size_t thread_num = local_num; @@ -327,6 +332,7 @@ namespace hpx::threads::policies { , "parent offset", parent_pool_->get_thread_offset() , parent_pool_->get_pool_name()); // clang-format on + // This is a task being injected from a thread on another // pool - we can schedule on any thread available thread_num = numa_holder_[0].thread_queue(0)->worker_next( @@ -408,11 +414,15 @@ namespace hpx::threads::policies { "Invalid schedule hint mode: {}", static_cast(data.schedulehint.mode)); } + // we do not allow threads created on other queues to 'run now' // as this causes cross-thread allocations and map accesses - if (local_num != thread_num) + if (local_num != thread_num && + (data.initial_state == thread_schedule_state::pending || + data.initial_state == thread_schedule_state::pending_boost)) { data.run_now = false; + // clang-format off spq_deb.debug(debug::str<>("create_thread") , "pool", parent_pool_->get_pool_name() @@ -705,7 +715,7 @@ namespace hpx::threads::policies { auto msg = spq_deb.declare_variable(nullptr); - //std::unique_lock l(init_mutex); + std::unique_lock l; using threads::thread_schedule_hint_mode; @@ -1334,6 +1344,12 @@ namespace hpx::threads::policies { return 0; } #endif + // this scheduler assumes thread bindings for the scheduled threads in + // certain ways preventing direct execution + bool supports_direct_execution() const noexcept override + { + return false; + } protected: typedef queue_holder_numa numa_queues; diff --git a/libs/core/thread_pools/CMakeLists.txt b/libs/core/thread_pools/CMakeLists.txt index c29593938040..5f2937f0a5df 100644 --- a/libs/core/thread_pools/CMakeLists.txt +++ b/libs/core/thread_pools/CMakeLists.txt @@ -43,5 +43,6 @@ add_hpx_module( hpx_itt_notify hpx_logging hpx_schedulers + hpx_threading_base CMAKE_SUBDIRS examples tests ) diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool.hpp index 3245fe512cea..22fa098195b9 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool.hpp @@ -123,7 +123,7 @@ namespace hpx::threads::detail { return sched_->Scheduler::enumerate_threads(f, state); } - void reset_thread_distribution() override + void reset_thread_distribution() noexcept override { return sched_->Scheduler::reset_thread_distribution(); } diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp index eaaa93716c66..2678f684bc04 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp @@ -133,7 +133,7 @@ namespace hpx::threads::detail { void scheduled_thread_pool::print_pool(std::ostream& os) const { os << "[pool \"" << id_.name() << "\", #" << id_.index() //-V128 - << "] with scheduler " << Scheduler::get_scheduler_name() + << "] with scheduler: " << Scheduler::get_scheduler_name() << "\nis running on PUs : \n"; os << hpx::threads::to_string(get_used_processing_units()) #ifdef HPX_HAVE_MAX_CPU_COUNT @@ -184,7 +184,8 @@ namespace hpx::threads::detail { std::size_t num_thread) const { HPX_ASSERT(num_thread != static_cast(-1)); - return sched_->Scheduler::get_state(num_thread).load(); + return sched_->Scheduler::get_state(num_thread) + .load(std::memory_order_acquire); } template @@ -620,6 +621,14 @@ namespace hpx::threads::detail { return; } + if (data.schedulehint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child && + !sched_->Scheduler::supports_direct_execution()) + { + data.schedulehint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + } + detail::create_thread(sched_.get(), data, id, ec); //-V601 // update statistics @@ -641,6 +650,14 @@ namespace hpx::threads::detail { return invalid_thread_id; } + if (data.schedulehint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child && + !sched_->Scheduler::supports_direct_execution()) + { + data.schedulehint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + } + thread_id_ref_type id = detail::create_work(sched_.get(), data, ec); //-V601 diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp index 117a24baffcf..a84afdb668dd 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp @@ -44,10 +44,9 @@ namespace hpx::threads::detail { { } - void collect_exec_time(std::uint64_t timestamp) const noexcept + void collect_exec_time(std::int64_t timestamp) const noexcept { - exec_time_ += static_cast( - util::hardware::timestamp() - timestamp); + exec_time_ += util::hardware::timestamp() - timestamp; } void take_snapshot() noexcept @@ -60,12 +59,11 @@ namespace hpx::threads::detail { } else { - tfunc_time_ = static_cast( - util::hardware::timestamp() - start_timestamp_); + tfunc_time_ = util::hardware::timestamp() - start_timestamp_; } } - std::uint64_t start_timestamp_; + std::int64_t start_timestamp_; std::int64_t& tfunc_time_; std::int64_t& exec_time_; @@ -90,7 +88,7 @@ namespace hpx::threads::detail { idle_rate_.collect_exec_time(timestamp_); } - std::uint64_t timestamp_; + std::int64_t timestamp_; idle_collect_rate& idle_rate_; }; @@ -304,8 +302,10 @@ namespace hpx::threads::detail { thrd_stat = (*thrdptr)(context_storage); - if (thrd_stat.get_previous() == - thread_schedule_state::terminated) + thread_schedule_state s = + thrd_stat.get_previous(); + if (s == thread_schedule_state::terminated || + s == thread_schedule_state::deleted) { profiler.stop(); // just in case, clean up the now dead pointer. @@ -429,7 +429,8 @@ namespace hpx::threads::detail { } } else if (HPX_UNLIKELY( - thread_schedule_state::active == state_val)) + thread_schedule_state::active == state_val && + !get_thread_id_data(thrd)->runs_as_child())) { write_rescheduling_log_warning(scheduler, num_thread, thrd); @@ -445,15 +446,15 @@ namespace hpx::threads::detail { scheduler.SchedulingPolicy::do_some_work(num_thread); } - // Remove the mapping from thread_map_ if HPX thread is depleted - // or terminated, this will delete the HPX thread. REVIEW: what - // has to be done with depleted HPX threads? - if (HPX_LIKELY(state_val == thread_schedule_state::depleted || + // Remove the mapping from thread_map_ if HPX thread is deleted + // or terminated, this will delete the HPX thread. + if (HPX_LIKELY(state_val == thread_schedule_state::deleted || state_val == thread_schedule_state::terminated)) { #ifdef HPX_HAVE_THREAD_CUMULATIVE_COUNTS ++counters.executed_threads_; #endif + HPX_ASSERT(!thrdptr->runs_as_child()); thrd = thread_id_type(); } } diff --git a/libs/core/threading_base/CMakeLists.txt b/libs/core/threading_base/CMakeLists.txt index 35fcff25cad3..3da1b8e31110 100644 --- a/libs/core/threading_base/CMakeLists.txt +++ b/libs/core/threading_base/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2019 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying diff --git a/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp b/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp index 73bf921c8b17..23a1ba2b1485 100644 --- a/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp @@ -239,7 +239,7 @@ namespace hpx::threads::policies { std::size_t num_thread = std::size_t(-1)) const = 0; #endif - virtual void reset_thread_distribution() {} + virtual void reset_thread_distribution() noexcept {} std::ptrdiff_t get_stack_size( threads::thread_stacksize stacksize) const noexcept; @@ -270,6 +270,12 @@ namespace hpx::threads::policies { detail::polling_status custom_polling_function() const; std::size_t get_polling_work_count() const; + // almost all schedulers support direct execution + virtual bool supports_direct_execution() const noexcept + { + return true; + } + protected: // the scheduler mode, protected from false sharing util::cache_line_data> mode_; diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp index 0da75d2b1a8c..97458e302f8d 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp @@ -107,7 +107,7 @@ namespace hpx::threads { /// /// \note This function will be seldom used directly. Most of /// the time the state of a thread will have to be - /// changed using the threadmanager. Moreover, + /// changed using the thread-manager. Moreover, /// changing the thread state using this function does /// not change its scheduling status. It only sets the /// thread's status word. To change the thread's @@ -361,7 +361,7 @@ namespace hpx::threads { { return nullptr; } - static char const* set_backtrace(char const*) noexcept + static constexpr char const* set_backtrace(char const*) noexcept { return nullptr; } @@ -370,7 +370,7 @@ namespace hpx::threads { { return nullptr; } - static util::backtrace const* set_backtrace( + static constexpr util::backtrace const* set_backtrace( util::backtrace const*) noexcept { return nullptr; @@ -485,6 +485,14 @@ namespace hpx::threads { void run_thread_exit_callbacks(); void free_thread_exit_callbacks(); + // no need to protect the variables related to scoped children as those + // are supposed to be accessed by ourselves only + bool runs_as_child( + std::memory_order mo = std::memory_order_acquire) const noexcept + { + return runs_as_child_.load(mo); + } + HPX_FORCEINLINE constexpr bool is_stackless() const noexcept { return is_stackless_; @@ -534,6 +542,14 @@ namespace hpx::threads { hpx::execution_base::this_thread::detail::agent_storage* agent_storage); + /// \brief Directly execute the thread function (inline) + /// + /// \returns This function returns the thread state the thread + /// should be scheduled from this point on. The thread + /// manager will use the returned value to set the + /// thread's scheduling status. + inline coroutine_type::result_type invoke_directly(); + virtual thread_id_type get_thread_id() const { return thread_id_type{const_cast(this)}; @@ -623,6 +639,9 @@ namespace hpx::threads { bool ran_exit_funcs_; bool const is_stackless_; + // support scoped child execution + std::atomic runs_as_child_; + // Singly linked list (heap-allocated) std::forward_list> exit_funcs_; @@ -641,13 +660,13 @@ namespace hpx::threads { #endif }; - HPX_FORCEINLINE thread_data* get_thread_id_data( + HPX_FORCEINLINE constexpr thread_data* get_thread_id_data( thread_id_ref_type const& tid) noexcept { return static_cast(tid.get().get()); } - HPX_FORCEINLINE thread_data* get_thread_id_data( + HPX_FORCEINLINE constexpr thread_data* get_thread_id_data( thread_id_type const& tid) noexcept { return static_cast(tid.get()); @@ -679,6 +698,13 @@ namespace hpx::threads { /// thread (or zero if the current thread is not a HPX thread). HPX_CORE_EXPORT thread_id_type get_self_id() noexcept; + /// The function \a get_outer_self_id returns the HPX thread id of + /// the current outer thread (or zero if the current thread is not a HPX + /// thread). This usually returns the same as \a get_self_id, except for + /// directly executed threads, in which case this returns the thread id + /// of the outermost HPX thread. + HPX_CORE_EXPORT thread_id_type get_outer_self_id() noexcept; + /// The function \a get_parent_id returns the HPX thread id of the /// current thread's parent (or zero if the current thread is not a /// HPX thread). @@ -733,10 +759,24 @@ namespace hpx::threads { HPX_FORCEINLINE coroutine_type::result_type thread_data::operator()( hpx::execution_base::this_thread::detail::agent_storage* agent_storage) { + // once a thread has started it can't be run directly anymore + runs_as_child_.store(false, std::memory_order_release); + if (is_stackless()) { return static_cast(this)->call(); } return static_cast(this)->call(agent_storage); } + + HPX_FORCEINLINE coroutine_type::result_type thread_data::invoke_directly() + { + HPX_ASSERT(runs_as_child()); + + if (is_stackless()) + { + return static_cast(this)->call(); + } + return static_cast(this)->invoke_directly(); + } } // namespace hpx::threads diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp index 3785a3966127..1df4500995f0 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp @@ -68,6 +68,23 @@ namespace hpx::threads { return coroutine_(set_state_ex(thread_restart_state::signaled)); } + HPX_FORCEINLINE coroutine_type::result_type invoke_directly() + { + HPX_ASSERT(get_state().state() == thread_schedule_state::active); + HPX_ASSERT(this == coroutine_.get_thread_id().get()); + + coroutine_type::result_type result = coroutine_.invoke_directly( + set_state_ex(thread_restart_state::signaled)); + + if (result.first == thread_schedule_state::terminated && + runs_as_child()) + { + result.first = thread_schedule_state::deleted; + } + + return result; + } + #if defined(HPX_DEBUG) thread_id_type get_thread_id() const override { diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp index 7d6982147dfd..c27e0babb6da 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp @@ -467,7 +467,7 @@ namespace hpx::threads { return false; } - virtual void reset_thread_distribution() {} + virtual void reset_thread_distribution() noexcept {} virtual void abort_all_suspended_threads() {} virtual bool cleanup_terminated(bool /*delete_all*/) diff --git a/libs/core/threading_base/src/create_work.cpp b/libs/core/threading_base/src/create_work.cpp index f21508bb6938..c0aaca600949 100644 --- a/libs/core/threading_base/src/create_work.cpp +++ b/libs/core/threading_base/src/create_work.cpp @@ -5,6 +5,7 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include +#include #include #include #include @@ -91,8 +92,11 @@ namespace hpx::threads::detail { // create the new thread if (data.priority == thread_priority::default_) + { data.priority = thread_priority::normal; + } + HPX_ASSERT(!data.run_now); data.run_now = (thread_priority::high == data.priority || thread_priority::high_recursive == data.priority || thread_priority::bound == data.priority || diff --git a/libs/core/threading_base/src/execution_agent.cpp b/libs/core/threading_base/src/execution_agent.cpp index 5a0d21171040..64952d6f736a 100644 --- a/libs/core/threading_base/src/execution_agent.cpp +++ b/libs/core/threading_base/src/execution_agent.cpp @@ -66,10 +66,12 @@ namespace hpx::threads { if (k < 4) //-V112 { } +#if defined(HPX_SMT_PAUSE) else if (k < 16) { HPX_SMT_PAUSE; } +#endif else if (k < 32 || k & 1) //-V112 { do_yield(desc, hpx::threads::thread_schedule_state::pending_boost); diff --git a/libs/core/threading_base/src/set_thread_state.cpp b/libs/core/threading_base/src/set_thread_state.cpp index dfba8ac09aaf..603d0d12e578 100644 --- a/libs/core/threading_base/src/set_thread_state.cpp +++ b/libs/core/threading_base/src/set_thread_state.cpp @@ -158,7 +158,8 @@ namespace hpx::threads::detail { return previous_state; // done } - + case thread_schedule_state::deleted: + [[fallthrough]]; case thread_schedule_state::terminated: { LTM_(warning).format( diff --git a/libs/core/threading_base/src/set_thread_state_timed.cpp b/libs/core/threading_base/src/set_thread_state_timed.cpp index ba13d39d4289..9b762a825935 100644 --- a/libs/core/threading_base/src/set_thread_state_timed.cpp +++ b/libs/core/threading_base/src/set_thread_state_timed.cpp @@ -83,7 +83,7 @@ namespace hpx::threads::detail { // create a new thread in suspended state, which will execute the // requested set_state when timer fires and will re-awaken this thread, // allowing the deadline_timer to go out of scope gracefully - thread_id_ref_type const self_id = get_self_id(); // keep alive + thread_id_ref_type const self_id = get_outer_self_id(); // keep alive std::shared_ptr> triggered( std::make_shared>(false)); diff --git a/libs/core/threading_base/src/thread_data.cpp b/libs/core/threading_base/src/thread_data.cpp index 5e305cb193c5..a884fbc33d52 100644 --- a/libs/core/threading_base/src/thread_data.cpp +++ b/libs/core/threading_base/src/thread_data.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // Copyright (c) 2008-2009 Chirag Dekate, Anshul Tandon // Copyright (c) 2011 Bryce Lelbach // @@ -72,6 +72,8 @@ namespace hpx::threads { , enabled_interrupt_(true) , ran_exit_funcs_(false) , is_stackless_(is_stackless) + , runs_as_child_(init_data.schedulehint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child) , scheduler_base_(init_data.scheduler_base) , last_worker_thread_num_(static_cast(-1)) , stacksize_(stacksize) @@ -88,8 +90,7 @@ namespace hpx::threads { // purposes if (parent_thread_id_ == nullptr) { - thread_self* self = get_self_ptr(); - if (self) + if (thread_self const* self = get_self_ptr()) { parent_thread_id_ = threads::get_self_id(); parent_thread_phase_ = self->get_thread_phase(); @@ -142,7 +143,8 @@ namespace hpx::threads { spinlock_pool::spinlock_for(this)); if (ran_exit_funcs_ || - get_state().state() == thread_schedule_state::terminated) + get_state().state() == thread_schedule_state::terminated || + get_state().state() == thread_schedule_state::deleted) { return false; } @@ -218,6 +220,11 @@ namespace hpx::threads { requested_interrupt_ = false; enabled_interrupt_ = true; ran_exit_funcs_ = false; + + runs_as_child_.store(init_data.schedulehint.runs_as_child_mode() == + hpx::threads::thread_execution_hint::run_as_child, + std::memory_order_relaxed); + exit_funcs_.clear(); scheduler_base_ = init_data.scheduler_base; last_worker_thread_num_ = static_cast(-1); @@ -314,6 +321,16 @@ namespace hpx::threads { return threads::invalid_thread_id; } + thread_id_type get_outer_self_id() noexcept + { + if (thread_self const* self = get_self_ptr(); + HPX_LIKELY(nullptr != self)) + { + return self->get_outer_thread_id(); + } + return threads::invalid_thread_id; + } + thread_data* get_self_id_data() noexcept { if (thread_self const* self = get_self_ptr(); @@ -408,8 +425,8 @@ namespace hpx::threads { std::shared_ptr get_self_timer_data() { - thread_data* thrd_data = get_self_id_data(); - if (HPX_LIKELY(nullptr != thrd_data)) + if (thread_data* thrd_data = get_self_id_data(); + HPX_LIKELY(nullptr != thrd_data)) { return thrd_data->get_timer_data(); } @@ -419,8 +436,8 @@ namespace hpx::threads { void set_self_timer_data( std::shared_ptr data) { - thread_data* thrd_data = get_self_id_data(); - if (HPX_LIKELY(nullptr != thrd_data)) + if (thread_data* thrd_data = get_self_id_data(); + HPX_LIKELY(nullptr != thrd_data)) { thrd_data->set_timer_data(data); } diff --git a/libs/core/threading_base/src/thread_helpers.cpp b/libs/core/threading_base/src/thread_helpers.cpp index cac1b91c17e8..ecbd95b0c87d 100644 --- a/libs/core/threading_base/src/thread_helpers.cpp +++ b/libs/core/threading_base/src/thread_helpers.cpp @@ -438,7 +438,7 @@ namespace hpx::this_thread { threads::thread_self& self = threads::get_self(); // keep alive - threads::thread_id_ref_type id = self.get_thread_id(); + threads::thread_id_ref_type id = self.get_outer_thread_id(); // handle interruption, if needed threads::interruption_point(id.noref(), ec); @@ -509,7 +509,7 @@ namespace hpx::this_thread { threads::thread_self& self = threads::get_self(); // keep alive - threads::thread_id_ref_type id = self.get_thread_id(); + threads::thread_id_ref_type id = self.get_outer_thread_id(); // handle interruption, if needed threads::interruption_point(id.noref(), ec); diff --git a/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp b/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp index 87804727a829..b21712446e79 100644 --- a/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp +++ b/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp @@ -297,7 +297,7 @@ namespace hpx { namespace threads { } } - void reset_thread_distribution() + void reset_thread_distribution() noexcept { for (auto& pool_iter : pools_) { diff --git a/libs/full/runtime_distributed/src/runtime_distributed.cpp b/libs/full/runtime_distributed/src/runtime_distributed.cpp index 3555c303931f..2f7731914334 100644 --- a/libs/full/runtime_distributed/src/runtime_distributed.cpp +++ b/libs/full/runtime_distributed/src/runtime_distributed.cpp @@ -1623,6 +1623,7 @@ namespace hpx { std::uint32_t current = (*it).second; (*it).second += cores_needed; + return current; } @@ -1749,6 +1750,14 @@ namespace hpx { components::component_type type, error_code& ec) { std::vector locality_ids; + if (nullptr == hpx::applier::get_applier_ptr()) + { + HPX_THROWS_IF(ec, hpx::error::invalid_status, + "hpx::find_all_localities", + "the runtime system is not available at this time"); + return locality_ids; + } + hpx::applier::get_applier().get_localities(locality_ids, type, ec); return locality_ids; } @@ -1756,6 +1765,14 @@ namespace hpx { std::vector find_all_localities(error_code& ec) { std::vector locality_ids; + if (nullptr == hpx::applier::get_applier_ptr()) + { + HPX_THROWS_IF(ec, hpx::error::invalid_status, + "hpx::find_all_localities", + "the runtime system is not available at this time"); + return locality_ids; + } + hpx::applier::get_applier().get_localities(locality_ids, ec); return locality_ids; } @@ -1764,6 +1781,14 @@ namespace hpx { components::component_type type, error_code& ec) { std::vector locality_ids; + if (nullptr == hpx::applier::get_applier_ptr()) + { + HPX_THROWS_IF(ec, hpx::error::invalid_status, + "hpx::find_remote_localities", + "the runtime system is not available at this time"); + return locality_ids; + } + hpx::applier::get_applier().get_remote_localities( locality_ids, type, ec); return locality_ids; @@ -1772,14 +1797,30 @@ namespace hpx { std::vector find_remote_localities(error_code& ec) { std::vector locality_ids; + if (nullptr == hpx::applier::get_applier_ptr()) + { + HPX_THROWS_IF(ec, hpx::error::invalid_status, + "hpx::find_remote_localities", + "the runtime system is not available at this time"); + return locality_ids; + } + hpx::applier::get_applier().get_remote_localities( locality_ids, components::component_invalid, ec); + return locality_ids; } // find a locality supporting the given component hpx::id_type find_locality(components::component_type type, error_code& ec) { + if (nullptr == hpx::applier::get_applier_ptr()) + { + HPX_THROWS_IF(ec, hpx::error::invalid_status, "hpx::find_locality", + "the runtime system is not available at this time"); + return hpx::invalid_id; + } + std::vector locality_ids; hpx::applier::get_applier().get_localities(locality_ids, type, ec); From aa4d5d54380e677fc1a911a2d9c23953796065d4 Mon Sep 17 00:00:00 2001 From: Panos Date: Tue, 27 Jun 2023 20:06:40 +0300 Subject: [PATCH 073/333] Fix outdated includes --- libs/core/futures/tests/unit/direct_scoped_execution.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/futures/tests/unit/direct_scoped_execution.cpp b/libs/core/futures/tests/unit/direct_scoped_execution.cpp index ba0205d47626..e9f64d02bd1a 100644 --- a/libs/core/futures/tests/unit/direct_scoped_execution.cpp +++ b/libs/core/futures/tests/unit/direct_scoped_execution.cpp @@ -6,9 +6,9 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include -#include -#include -#include +#include +#include +#include #include #include From ab6dc80f85ce84f48d9e79bcb9bac4b74e6064d0 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Sun, 9 Jul 2023 18:04:17 -0500 Subject: [PATCH 074/333] Fix race condition --- .../futures/include/hpx/futures/packaged_continuation.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/core/futures/include/hpx/futures/packaged_continuation.hpp b/libs/core/futures/include/hpx/futures/packaged_continuation.hpp index 0130bb7f55f6..7716a5f4a789 100644 --- a/libs/core/futures/include/hpx/futures/packaged_continuation.hpp +++ b/libs/core/futures/include/hpx/futures/packaged_continuation.hpp @@ -230,11 +230,17 @@ namespace hpx::lcos::detail { hpx::intrusive_ptr this_(this); hpx::threads::thread_description desc(f_, "async"); - this->runs_child_ = spawner( + auto thrd = spawner( [this_ = HPX_MOVE(this_), f = HPX_MOVE(f)]() mutable -> void { this_->template run_impl(HPX_MOVE(f)); }, desc); + + std::lock_guard l(mtx_); + if (!this->base_type::is_ready()) + { + this->runs_child_ = HPX_MOVE(thrd); + } } public: From 2d123472ebf766462f36301d455b564d5c0ea173 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 11 Jul 2023 13:22:10 -0500 Subject: [PATCH 075/333] Working around the need to use a lock during thread creation --- .../hpx/execution/detail/future_exec.hpp | 12 +++++------ .../include/hpx/futures/futures_factory.hpp | 1 + .../hpx/futures/packaged_continuation.hpp | 12 +++-------- .../tests/unit/direct_scoped_execution.cpp | 2 +- .../hpx/threading_base/register_thread.hpp | 21 ++++++++++++++++--- .../hpx/threading_base/thread_data.hpp | 9 +++++--- 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp index 32a4c9953075..c49002325c64 100644 --- a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp +++ b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp @@ -153,8 +153,8 @@ namespace hpx::lcos::detail { struct post_policy_spawner { template - threads::thread_id_ref_type operator()( - F&& f, hpx::threads::thread_description desc) const + void operator()(F&& f, hpx::threads::thread_description desc, + threads::thread_id_ref_type& id) const { threads::thread_init_data data( threads::make_thread_function_nullary(HPX_FORWARD(F, f)), @@ -163,7 +163,7 @@ namespace hpx::lcos::detail { threads::thread_stacksize::default_, threads::thread_schedule_state::pending); - return threads::register_thread(data); + threads::register_thread(data, id); } }; @@ -173,11 +173,11 @@ namespace hpx::lcos::detail { Executor exec; template - threads::thread_id_ref_type operator()( - F&& f, hpx::threads::thread_description) const + void operator()(F&& f, hpx::threads::thread_description, + threads::thread_id_ref_type& id) const { + id = threads::invalid_thread_id; hpx::parallel::execution::post(exec, HPX_FORWARD(F, f)); - return threads::invalid_thread_id; } }; diff --git a/libs/core/futures/include/hpx/futures/futures_factory.hpp b/libs/core/futures/include/hpx/futures/futures_factory.hpp index f61f0000e441..7a1e08409f31 100644 --- a/libs/core/futures/include/hpx/futures/futures_factory.hpp +++ b/libs/core/futures/include/hpx/futures/futures_factory.hpp @@ -8,6 +8,7 @@ #include #include + #include #include #include diff --git a/libs/core/futures/include/hpx/futures/packaged_continuation.hpp b/libs/core/futures/include/hpx/futures/packaged_continuation.hpp index 7716a5f4a789..5f52260f03f4 100644 --- a/libs/core/futures/include/hpx/futures/packaged_continuation.hpp +++ b/libs/core/futures/include/hpx/futures/packaged_continuation.hpp @@ -230,17 +230,11 @@ namespace hpx::lcos::detail { hpx::intrusive_ptr this_(this); hpx::threads::thread_description desc(f_, "async"); - auto thrd = spawner( + spawner( [this_ = HPX_MOVE(this_), f = HPX_MOVE(f)]() mutable -> void { this_->template run_impl(HPX_MOVE(f)); }, - desc); - - std::lock_guard l(mtx_); - if (!this->base_type::is_ready()) - { - this->runs_child_ = HPX_MOVE(thrd); - } + desc, this->runs_child_); } public: @@ -261,8 +255,8 @@ namespace hpx::lcos::detail { if (this->is_ready()) return; // nothing we can do - // 26110: Caller failing to hold lock 'l' #if defined(HPX_MSVC) +// 26110: Caller failing to hold lock 'l' #pragma warning(push) #pragma warning(disable : 26110) #endif diff --git a/libs/core/futures/tests/unit/direct_scoped_execution.cpp b/libs/core/futures/tests/unit/direct_scoped_execution.cpp index e9f64d02bd1a..b65a245d50ea 100644 --- a/libs/core/futures/tests/unit/direct_scoped_execution.cpp +++ b/libs/core/futures/tests/unit/direct_scoped_execution.cpp @@ -5,10 +5,10 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -#include #include #include #include +#include #include #include diff --git a/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp b/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp index 8fb7210141ce..5fb4df759336 100644 --- a/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp @@ -97,6 +97,7 @@ namespace hpx::threads { /// /// \param data [in] The data to use for creating the thread. /// \param pool [in] The thread pool to use for launching the work. + /// \param id [out] The id of the newly created thread (if applicable) /// \param ec [in,out] This represents the error status on exit, /// if this is pre-initialized to \a hpx#throws the /// function will throw on error instead. @@ -111,14 +112,21 @@ namespace hpx::threads { /// throw but returns the result code using the parameter /// \a ec. Otherwise it throws an instance /// of hpx#exception. - inline threads::thread_id_ref_type register_thread( - threads::thread_init_data& data, threads::thread_pool_base* pool, + inline void register_thread(threads::thread_init_data& data, + threads::thread_pool_base* pool, threads::thread_id_ref_type& id, error_code& ec = throws) { HPX_ASSERT(pool); data.run_now = true; - threads::thread_id_ref_type id = threads::invalid_thread_id; pool->create_thread(data, id, ec); + } + + inline threads::thread_id_ref_type register_thread( + threads::thread_init_data& data, threads::thread_pool_base* pool, + error_code& ec = throws) + { + threads::thread_id_ref_type id = threads::invalid_thread_id; + register_thread(data, pool, id, ec); return id; } @@ -128,6 +136,7 @@ namespace hpx::threads { /// on an HPX thread. /// /// \param data [in] The data to use for creating the thread. + /// \param id [out] The id of the newly created thread (if applicable) /// \param ec [in,out] This represents the error status on exit, /// if this is pre-initialized to \a hpx#throws the /// function will throw on error instead. @@ -141,6 +150,12 @@ namespace hpx::threads { /// \a hpx#throws this function doesn't throw but returns /// the result code using the parameter \a ec. Otherwise /// it throws an instance of hpx#exception. + inline void register_thread(threads::thread_init_data& data, + threads::thread_id_ref_type& id, error_code& ec = throws) + { + register_thread(data, detail::get_self_or_default_pool(), id, ec); + } + inline threads::thread_id_ref_type register_thread( threads::thread_init_data& data, error_code& ec = throws) { diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp index 97458e302f8d..45a591885c4f 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp @@ -9,6 +9,8 @@ #pragma once #include +#include + #include #include #include @@ -18,7 +20,6 @@ #include #include #include -#include #include #include #if defined(HPX_HAVE_APEX) @@ -31,10 +32,12 @@ #include #include #include -#include -#include #include +#if defined(HPX_HAVE_THREAD_BACKTRACE_ON_SUSPENSION) +#include +#endif + #include //////////////////////////////////////////////////////////////////////////////// From 19e3355580d7923a867c46e600d84d94999e96a5 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 18 Jul 2023 10:32:20 -0500 Subject: [PATCH 076/333] Enabling this feature on more builders --- .jenkins/lsu/env-clang-11.sh | 2 ++ .jenkins/lsu/env-clang-12.sh | 2 ++ .jenkins/lsu/env-gcc-10.sh | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.jenkins/lsu/env-clang-11.sh b/.jenkins/lsu/env-clang-11.sh index 77b0f84614c5..42c5aea9169e 100644 --- a/.jenkins/lsu/env-clang-11.sh +++ b/.jenkins/lsu/env-clang-11.sh @@ -33,3 +33,5 @@ configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # Make sure HWLOC does not report 'cores'. This is purely an option to enable # testing the topology code under conditions close to those on FreeBSD. configure_extra_options+=" -DHPX_TOPOLOGY_WITH_ADDITIONAL_HWLOC_TESTING=ON" + +configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 712e11d6077a..cea6a999dd14 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -34,3 +34,5 @@ configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # Make sure HWLOC does not report 'cores'. This is purely an option to enable # testing the topology code under conditions close to those on FreeBSD. configure_extra_options+=" -DHPX_TOPOLOGY_WITH_ADDITIONAL_HWLOC_TESTING=ON" + +configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 10eda35ad81f..4835f03edd2c 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -29,3 +29,5 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" + +configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" From 72849e45480bad15152c369d6b51bafc3f1520ef Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Wed, 19 Jul 2023 09:37:04 -0500 Subject: [PATCH 077/333] Fixing more possible race conditions --- libs/core/futures/include/hpx/futures/futures_factory.hpp | 7 +++---- libs/core/futures/src/future_data.cpp | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/libs/core/futures/include/hpx/futures/futures_factory.hpp b/libs/core/futures/include/hpx/futures/futures_factory.hpp index 7a1e08409f31..853969c0979c 100644 --- a/libs/core/futures/include/hpx/futures/futures_factory.hpp +++ b/libs/core/futures/include/hpx/futures/futures_factory.hpp @@ -143,8 +143,8 @@ namespace hpx::lcos::local { { HPX_ASSERT( this->runs_child_ == threads::invalid_thread_id); - this->runs_child_ = - threads::register_thread(data, pool, ec); + threads::register_thread( + data, pool, this->runs_child_, ec); return this->runs_child_; } @@ -164,8 +164,7 @@ namespace hpx::lcos::local { threads::thread_schedule_state::suspended, true); HPX_ASSERT(this->runs_child_ == threads::invalid_thread_id); - this->runs_child_ = - threads::register_thread(data, pool, ec); + threads::register_thread(data, pool, this->runs_child_, ec); // now run the thread threads::set_thread_state(this->runs_child_.noref(), diff --git a/libs/core/futures/src/future_data.cpp b/libs/core/futures/src/future_data.cpp index 4d82818ee419..4c608c77399a 100644 --- a/libs/core/futures/src/future_data.cpp +++ b/libs/core/futures/src/future_data.cpp @@ -122,7 +122,7 @@ namespace hpx::lcos::detail { return false; } - auto state = this->state_.load(std::memory_order_acquire); + auto const state = this->state_.load(std::memory_order_acquire); if (state != this->empty) { return false; From 5f7541dbe5811379a2e31f3a249394c231546676 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Thu, 20 Jul 2023 08:51:33 -0500 Subject: [PATCH 078/333] Fix race condition --- libs/core/futures/src/future_data.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/futures/src/future_data.cpp b/libs/core/futures/src/future_data.cpp index 4c608c77399a..38f0b92a2712 100644 --- a/libs/core/futures/src/future_data.cpp +++ b/libs/core/futures/src/future_data.cpp @@ -117,7 +117,8 @@ namespace hpx::lcos::detail { bool future_data_base::execute_thread() { // we try to directly execute the thread exactly once - if (!runs_child_) + threads::thread_id_ref_type runs_child = runs_child_; + if (!runs_child) { return false; } @@ -129,7 +130,6 @@ namespace hpx::lcos::detail { } // this thread would block on the future - threads::thread_id_ref_type runs_child = runs_child_; auto* thrd = get_thread_id_data(runs_child); HPX_UNUSED(thrd); // might be unused From d60382c3e31381cdb7bc771176be39d8ba2940c7 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 21 Jul 2023 12:29:09 -0500 Subject: [PATCH 079/333] Fixing assert in schedule_pool test - flyby: additional unrelated cleanup --- .../cmake/templates/config_version.hpp.in | 7 ++--- .../futures/src/detail/execute_thread.cpp | 6 ++++- .../tests/unit/CMakeLists.txt | 27 +++++++++++++------ .../tests/unit/suspend_pool.cpp | 18 ++++++------- .../include/hpx/schedulers/thread_queue.hpp | 18 +++++++++---- .../hpx/threading_base/thread_data.hpp | 2 +- .../threading_base/thread_data_stackful.hpp | 2 +- 7 files changed, 52 insertions(+), 28 deletions(-) diff --git a/libs/core/config/cmake/templates/config_version.hpp.in b/libs/core/config/cmake/templates/config_version.hpp.in index 217b7fa1d510..a8f8e7bb48b7 100644 --- a/libs/core/config/cmake/templates/config_version.hpp.in +++ b/libs/core/config/cmake/templates/config_version.hpp.in @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2020 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // Copyright (c) 2011 Bryce Lelbach // Copyright (c) 2013 Adrian Serio // @@ -30,8 +30,9 @@ /// ``HPX_VERSION_FULL & 0x00FF00 == HPX_VERSION_MINOR`` /// ``HPX_VERSION_FULL & 0x0000FF == HPX_VERSION_SUBMINOR`` #define HPX_VERSION_FULL \ - ((HPX_VERSION_MAJOR << 16) | (HPX_VERSION_MINOR << 8) | \ - HPX_VERSION_SUBMINOR) + ((HPX_PP_CAT(0x, HPX_VERSION_MAJOR) << 16) | \ + (HPX_PP_CAT(0x, HPX_VERSION_MINOR) << 8) | \ + HPX_PP_CAT(0x, HPX_VERSION_SUBMINOR)) /// Evaluates to the release date of this HPX version in the format YYYYMMDD. #define HPX_VERSION_DATE @HPX_VERSION_DATE@ diff --git a/libs/core/futures/src/detail/execute_thread.cpp b/libs/core/futures/src/detail/execute_thread.cpp index 7bcd763eca72..196cb37a1099 100644 --- a/libs/core/futures/src/detail/execute_thread.cpp +++ b/libs/core/futures/src/detail/execute_thread.cpp @@ -201,7 +201,11 @@ namespace hpx::threads::detail { set_thread_state(thrd.noref(), thread_schedule_state::pending, thread_restart_state::signaled); auto* scheduler = thrdptr->get_scheduler_base(); - scheduler->schedule_thread_last(thrd, thread_schedule_hint()); + + auto const hint = thread_schedule_hint(static_cast( + thrdptr->get_last_worker_thread_num())); + scheduler->schedule_thread_last(HPX_MOVE(thrd), hint); + scheduler->do_some_work(hint.hint); } HPX_ASSERT(state_val != thread_schedule_state::terminated); diff --git a/libs/core/resource_partitioner/tests/unit/CMakeLists.txt b/libs/core/resource_partitioner/tests/unit/CMakeLists.txt index cbf5a0cb1a0d..0710e6e98415 100644 --- a/libs/core/resource_partitioner/tests/unit/CMakeLists.txt +++ b/libs/core/resource_partitioner/tests/unit/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2017 Hartmut Kaiser +# Copyright (c) 2017-2023 Hartmut Kaiser # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -33,13 +33,24 @@ set(used_pus_PARAMETERS THREADS_PER_LOCALITY 4 RUN_SERIAL) set(scheduler_priority_check_PARAMETERS THREADS_PER_LOCALITY -1) set(shutdown_suspended_pus_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_disabled_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_pool_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_pool_external_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_runtime_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_thread_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_thread_external_PARAMETERS THREADS_PER_LOCALITY 4) -set(suspend_thread_timed_PARAMETERS THREADS_PER_LOCALITY 4) + +if(CMAKE_SYSTEM_NAME MATCHES "Linux") + set(additional_parameters "--hpx:ini=hpx.stacks.use_guard_pages=0") +endif() + +set(suspend_disabled_PARAMETERS THREADS_PER_LOCALITY 4 ${additional_parameters}) +set(suspend_pool_PARAMETERS THREADS_PER_LOCALITY 4 ${additional_parameters}) +set(suspend_pool_external_PARAMETERS THREADS_PER_LOCALITY 4 + ${additional_parameters} +) +set(suspend_runtime_PARAMETERS THREADS_PER_LOCALITY 4 ${additional_parameters}) +set(suspend_thread_PARAMETERS THREADS_PER_LOCALITY 4 ${additional_parameters}) +set(suspend_thread_external_PARAMETERS THREADS_PER_LOCALITY 4 + ${additional_parameters} +) +set(suspend_thread_timed_PARAMETERS THREADS_PER_LOCALITY 4 + ${additional_parameters} +) foreach(test ${tests}) set(sources ${test}.cpp) diff --git a/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp b/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp index d8dec3d78922..b316b0fdfb3f 100644 --- a/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp +++ b/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp @@ -19,15 +19,13 @@ #include #include -#include #include -#include #include #include #include -std::size_t const max_threads = (std::min)( - std::size_t(4), std::size_t(hpx::threads::hardware_concurrency())); +std::size_t const max_threads = (std::min)(static_cast(4), + static_cast(hpx::threads::hardware_concurrency())); int hpx_main() { @@ -55,7 +53,7 @@ int hpx_main() { // Suspend and resume pool with future - hpx::chrono::high_resolution_timer t; + hpx::chrono::high_resolution_timer const t; while (t.elapsed() < 1) { @@ -78,7 +76,7 @@ int hpx_main() { // Suspend and resume pool with callback hpx::counting_semaphore_var<> sem; - hpx::chrono::high_resolution_timer t; + hpx::chrono::high_resolution_timer const t; while (t.elapsed() < 1) { @@ -106,7 +104,7 @@ int hpx_main() { // Suspend pool with some threads already suspended - hpx::chrono::high_resolution_timer t; + hpx::chrono::high_resolution_timer const t; while (t.elapsed() < 1) { @@ -144,7 +142,9 @@ void test_scheduler( init_args.cfg = {"hpx.os_threads=" + std::to_string(max_threads)}; init_args.rp_callback = [scheduler](auto& rp, hpx::program_options::variables_map const&) { - rp.create_thread_pool("worker", scheduler); + rp.create_thread_pool("worker", scheduler, + hpx::threads::policies::scheduler_mode::default_ | + hpx::threads::policies::scheduler_mode::enable_elasticity); std::size_t const worker_pool_threads = max_threads - 1; HPX_ASSERT(worker_pool_threads >= 1); @@ -173,7 +173,7 @@ int main(int argc, char* argv[]) { HPX_ASSERT(max_threads >= 2); - std::vector schedulers = { + std::vector const schedulers = { hpx::resource::scheduling_policy::local, hpx::resource::scheduling_policy::local_priority_fifo, #if defined(HPX_HAVE_CXX11_STD_ATOMIC_128BIT) diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 0894ccaf8c6c..1bc5a5e89191 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -398,8 +398,12 @@ namespace hpx::threads::policies { thread_id_type tid(todelete); --terminated_items_count_; - // this thread has to be in this map - HPX_ASSERT(thread_map_.find(tid) != thread_map_.end()); + // this thread has to be managed by this queue, it may have + // ended up on the terminate threads list more than once, + // however + HPX_ASSERT( + &get_thread_id_data(tid)->get_queue() == + this); if (thread_map_.erase(tid) != 0) { @@ -426,9 +430,12 @@ namespace hpx::threads::policies { thread_id_type tid(todelete); --terminated_items_count_; - // this thread has to be in this map, except if it has changed - // its priority, then it could be elsewhere - HPX_ASSERT(thread_map_.find(tid) != thread_map_.end()); + // this thread has to be managed by this queue, it may have + // ended up on the terminate threads list more than once, + // however + HPX_ASSERT( + &get_thread_id_data(tid)->get_queue() == + this); if (thread_map_.erase(tid) != 0) { @@ -888,6 +895,7 @@ namespace hpx::threads::policies { threads::thread_id_ref_type thrd, bool other_end = false) { ++work_items_count_.data_; + #ifdef HPX_HAVE_THREAD_QUEUE_WAITTIME work_items_.push(new thread_description{HPX_MOVE(thrd), hpx::chrono::high_resolution_clock::now()}, diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp index 45a591885c4f..cb47ff33fabc 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp @@ -774,7 +774,7 @@ namespace hpx::threads { HPX_FORCEINLINE coroutine_type::result_type thread_data::invoke_directly() { - HPX_ASSERT(runs_as_child()); + HPX_ASSERT(runs_as_child(std::memory_order_relaxed)); if (is_stackless()) { diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp index 1df4500995f0..c9eee85b8752 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data_stackful.hpp @@ -77,7 +77,7 @@ namespace hpx::threads { set_state_ex(thread_restart_state::signaled)); if (result.first == thread_schedule_state::terminated && - runs_as_child()) + runs_as_child(std::memory_order_relaxed)) { result.first = thread_schedule_state::deleted; } From 81e4a797d8aaf9447f19fcd727c3411e586f7c26 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 24 Jul 2023 12:21:18 -0500 Subject: [PATCH 080/333] Improve error handling --- .../src/init_runtime_local.cpp | 6 ++-- .../tests/unit/suspend_pool_external.cpp | 20 +++++++---- libs/core/runtime_local/src/runtime_local.cpp | 21 ++++++------ .../scheduled_thread_pool_impl.hpp | 33 +++++++++++++------ 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/libs/core/init_runtime_local/src/init_runtime_local.cpp b/libs/core/init_runtime_local/src/init_runtime_local.cpp index 3ddcd74bf3b1..14d816e7e5b0 100644 --- a/libs/core/init_runtime_local/src/init_runtime_local.cpp +++ b/libs/core/init_runtime_local/src/init_runtime_local.cpp @@ -357,14 +357,14 @@ namespace hpx { } // non-blocking version - start(*rt, cfg.hpx_main_f_, cfg.vm_, HPX_MOVE(startup), - HPX_MOVE(shutdown)); + int const result = start(*rt, cfg.hpx_main_f_, cfg.vm_, + HPX_MOVE(startup), HPX_MOVE(shutdown)); // pointer to runtime is stored in TLS hpx::runtime* p = rt.release(); (void) p; - return 0; + return result; } //////////////////////////////////////////////////////////////////////// diff --git a/libs/core/resource_partitioner/tests/unit/suspend_pool_external.cpp b/libs/core/resource_partitioner/tests/unit/suspend_pool_external.cpp index 4eea2224c9e9..39151f23ec72 100644 --- a/libs/core/resource_partitioner/tests/unit/suspend_pool_external.cpp +++ b/libs/core/resource_partitioner/tests/unit/suspend_pool_external.cpp @@ -30,27 +30,29 @@ void test_scheduler( hpx::local::init_params init_args; init_args.cfg = {"hpx.os_threads=" + - std::to_string(((std::min)(std::size_t(4), - std::size_t(hpx::threads::hardware_concurrency()))))}; + std::to_string(((std::min)(static_cast(4), + static_cast(hpx::threads::hardware_concurrency()))))}; init_args.rp_callback = [scheduler](auto& rp, hpx::program_options::variables_map const&) { rp.create_thread_pool("default", scheduler); }; - hpx::local::start(nullptr, argc, argv, init_args); + HPX_TEST(hpx::local::start(nullptr, argc, argv, init_args)); hpx::threads::thread_pool_base& default_pool = hpx::resource::get_thread_pool("default"); std::size_t const default_pool_threads = hpx::resource::get_num_threads("default"); - hpx::chrono::high_resolution_timer t; + hpx::chrono::high_resolution_timer const t; while (t.elapsed() < 2) { + std::atomic count_tasks = default_pool_threads * 10000; + for (std::size_t i = 0; i < default_pool_threads * 10000; ++i) { - hpx::post([]() {}); + hpx::post([&]() { --count_tasks; }); } bool suspended = false; @@ -72,6 +74,12 @@ void test_scheduler( { std::this_thread::yield(); } + + // wait for tasks finish running + while (count_tasks.load() != 0) + { + std::this_thread::yield(); + } } hpx::post([]() { hpx::local::finalize(); }); @@ -81,7 +89,7 @@ void test_scheduler( int main(int argc, char* argv[]) { - std::vector schedulers = { + std::vector const schedulers = { hpx::resource::scheduling_policy::local, hpx::resource::scheduling_policy::local_priority_fifo, #if defined(HPX_HAVE_CXX11_STD_ATOMIC_128BIT) diff --git a/libs/core/runtime_local/src/runtime_local.cpp b/libs/core/runtime_local/src/runtime_local.cpp index 94626ecfb40c..2aa33b83e009 100644 --- a/libs/core/runtime_local/src/runtime_local.cpp +++ b/libs/core/runtime_local/src/runtime_local.cpp @@ -1424,7 +1424,6 @@ namespace hpx { // see: http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=100319 _isatty(0); #endif - // {{{ early startup code - local // initialize instrumentation system #ifdef HPX_HAVE_APEX @@ -1448,9 +1447,13 @@ namespace hpx { "I/O service pool"; #endif // start the thread manager - thread_manager_->run(); + if (!thread_manager_->run()) + { + std::cerr << "runtime::start: failed to start threadmanager\n"; + return -1; + } + lbt_ << "(1st stage) runtime::start: started threadmanager"; - // }}} // {{{ launch main // register the given main function with the thread manager @@ -1473,13 +1476,11 @@ namespace hpx { { return wait(); // wait for the shutdown_action to be executed } - else - { - // wait for at least hpx::state::running - util::yield_while( - [this]() { return get_state() < hpx::state::running; }, - "runtime::start"); - } + + // wait for at least hpx::state::running + util::yield_while( + [this]() { return get_state() < hpx::state::running; }, + "runtime::start"); return 0; // return zero as we don't know the outcome of hpx_main yet } diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp index 2678f684bc04..72c30824fe08 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduled_thread_pool_impl.hpp @@ -229,10 +229,24 @@ namespace hpx::threads::detail { if (!threads_.empty()) { // wait for all work to be done before requesting threads to shut - // down + // down, but only if all threads were successfully initialized if (blocking) { - wait(); + bool must_wait = true; + for (const auto& thread : threads_) + { + // skip this if already stopped + if (!thread.joinable()) + { + must_wait = false; + break; + } + } + + if (must_wait) + { + wait(); + } } // wake up if suspended @@ -329,9 +343,9 @@ namespace hpx::threads::detail { // get_pu_mask expects index according to ordering of masks // in affinity_data::affinity_masks_ // which is in order of occupied PU - LTM_(info).format( - "run: {} create OS thread {}: will run on processing units " - "within this mask: {}", + LTM_(info).format("run: {} create OS thread {}: will run " + "on processing units " + "within this mask: {}", id_.name(), global_thread_num, hpx::threads::to_string(mask)); @@ -352,8 +366,7 @@ namespace hpx::threads::detail { "run: {} failed with: {}", id_.name(), e.what()); // trigger the barrier - pool_threads -= (thread_num + 1); - while (pool_threads-- != 0) + while (thread_num-- != 0) startup->wait(); stop_locked(l); @@ -1908,10 +1921,10 @@ namespace hpx::threads::detail { std::atomic& state = sched_->Scheduler::get_state(virt_core); - hpx::state oldstate = state.exchange(hpx::state::initialized); + [[maybe_unused]] hpx::state const oldstate = + state.exchange(hpx::state::initialized); HPX_ASSERT(oldstate == hpx::state::stopped || oldstate == hpx::state::initialized); - HPX_UNUSED(oldstate); threads_[virt_core] = std::thread(&scheduled_thread_pool::thread_func, this, virt_core, thread_num, HPX_MOVE(startup)); @@ -1941,7 +1954,7 @@ namespace hpx::threads::detail { sched_->Scheduler::get_state(virt_core); // inform the scheduler to stop the virtual core - hpx::state oldstate = state.exchange(hpx::state::stopping); + hpx::state const oldstate = state.exchange(hpx::state::stopping); if (oldstate > hpx::state::stopping) { From fa7a01fcf9a9421a1f25bc556b7b24909e2fa487 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 27 Jul 2023 20:01:26 -0500 Subject: [PATCH 081/333] Disable run_as_child mode for some of the tests --- .../detail/async_launch_policy_dispatch.hpp | 12 ++-- .../execution/detail/post_policy_dispatch.hpp | 62 +++++-------------- .../tests/unit/suspend_pool.cpp | 7 ++- .../tests/unit/suspend_thread.cpp | 34 ++++++---- .../include/hpx/schedulers/thread_queue.hpp | 3 - 5 files changed, 46 insertions(+), 72 deletions(-) diff --git a/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp index 3af112f5c98b..a314139d54a0 100644 --- a/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp @@ -148,7 +148,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy&& policy, hpx::threads::thread_description const& desc, + call(Policy policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { HPX_ASSERT(pool); @@ -172,10 +172,8 @@ namespace hpx::detail { lcos::local::futures_factory p( util::deferred_call(HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...)); - threads::thread_id_ref_type tid = - p.post(pool, desc.get_description(), HPX_MOVE(policy)); - - if (tid) + if (threads::thread_id_ref_type tid = + p.post(pool, desc.get_description(), HPX_MOVE(policy))) { auto runs_as_child = hint.runs_as_child_mode(); if (runs_as_child == @@ -230,7 +228,7 @@ namespace hpx::detail { HPX_FORCEINLINE static std::enable_if_t< traits::detail::is_deferred_invocable_v, hpx::future>> - call(Policy&& policy, hpx::threads::thread_description const& desc, + call(Policy policy, hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { HPX_ASSERT(pool != nullptr); @@ -258,7 +256,7 @@ namespace hpx::detail { p.post(pool, desc.get_description(), policy); // make sure this thread is executed last - threads::thread_id_type tid_self = threads::get_self_id(); + threads::thread_id_type const tid_self = threads::get_self_id(); if (tid && tid_self && get_thread_id_data(tid)->get_scheduler_base() == get_thread_id_data(tid_self)->get_scheduler_base()) diff --git a/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp index bfd014d447b4..2d3365ccebdb 100644 --- a/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp @@ -37,16 +37,14 @@ namespace hpx::detail { hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { + // run_as_child doesn't make sense if we _post_ a tasks auto hint = policy.hint(); if (hint.runs_as_child_mode() == hpx::threads::thread_execution_hint::run_as_child) { - if (!pool->get_scheduler()->supports_direct_execution()) - { - hint.runs_as_child_mode( - hpx::threads::thread_execution_hint::none); - policy.set_hint(hint); - } + hint.runs_as_child_mode( + hpx::threads::thread_execution_hint::none); + policy.set_hint(hint); } threads::thread_init_data data( @@ -76,18 +74,8 @@ namespace hpx::detail { hpx::threads::thread_description const& desc, threads::thread_pool_base* pool, F&& f, Ts&&... ts) { + // run_as_child doesn't make sense if we _post_ a tasks auto hint = policy.hint(); - if (hint.runs_as_child_mode() == - hpx::threads::thread_execution_hint::run_as_child) - { - if (!pool->get_scheduler()->supports_direct_execution()) - { - hint.runs_as_child_mode( - hpx::threads::thread_execution_hint::none); - policy.set_hint(hint); - } - } - threads::thread_init_data data( threads::make_thread_function_nullary(hpx::util::deferred_call( HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...)), @@ -95,13 +83,14 @@ namespace hpx::detail { threads::thread_schedule_hint( threads::thread_schedule_hint_mode::thread, static_cast(get_worker_thread_num()), - hint.placement_mode(), hint.runs_as_child_mode()), + hint.placement_mode(), + hpx::threads::thread_execution_hint::none), policy.stacksize(), threads::thread_schedule_state::pending_do_not_schedule, true); - threads::thread_id_ref_type tid = + threads::thread_id_ref_type const tid = threads::register_thread(data, pool); - threads::thread_id_type tid_self = threads::get_self_id(); + threads::thread_id_type const tid_self = threads::get_self_id(); // make sure this thread is executed last if (tid && tid_self && @@ -181,52 +170,29 @@ namespace hpx::detail { { HPX_ASSERT(pool != nullptr); - auto hint = policy.hint(); - if (hint.runs_as_child_mode() == - hpx::threads::thread_execution_hint::run_as_child) - { - if (!pool->get_scheduler()->supports_direct_execution()) - { - hint.runs_as_child_mode( - hpx::threads::thread_execution_hint::none); - policy.set_hint(hint); - } - } - + // run_as_child doesn't make sense if we _post_ a tasks if (policy == launch::sync) { - auto mod_policy = launch::sync_policy( - policy.priority(), policy.stacksize(), hint); - post_policy_dispatch::call( - HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_MOVE(policy), desc, pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } else if (policy == launch::deferred) { - auto mod_policy = launch::deferred_policy( - policy.priority(), policy.stacksize(), hint); - post_policy_dispatch::call( - HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_MOVE(policy), desc, pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } else if (policy == launch::fork) { - auto mod_policy = launch::fork_policy( - policy.priority(), policy.stacksize(), hint); - post_policy_dispatch::call( - HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_MOVE(policy), desc, pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } else { - auto mod_policy = launch::async_policy( - policy.priority(), policy.stacksize(), hint); - post_policy_dispatch::call( - HPX_MOVE(mod_policy), desc, pool, HPX_FORWARD(F, f), + HPX_MOVE(policy), desc, pool, HPX_FORWARD(F, f), HPX_FORWARD(Ts, ts)...); } } diff --git a/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp b/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp index b316b0fdfb3f..184d70c292d8 100644 --- a/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp +++ b/libs/core/resource_partitioner/tests/unit/suspend_pool.cpp @@ -46,11 +46,16 @@ int hpx_main() hpx::threads::thread_pool_base& worker_pool = hpx::resource::get_thread_pool("worker"); - hpx::execution::parallel_executor worker_exec( + hpx::execution::parallel_executor exec( &hpx::resource::get_thread_pool("worker")); std::size_t const worker_pool_threads = hpx::resource::get_num_threads("worker"); + hpx::threads::thread_schedule_hint hint; + hint.runs_as_child_mode(hpx::threads::thread_execution_hint::none); + + auto worker_exec = hpx::execution::experimental::with_hint(exec, hint); + { // Suspend and resume pool with future hpx::chrono::high_resolution_timer const t; diff --git a/libs/core/resource_partitioner/tests/unit/suspend_thread.cpp b/libs/core/resource_partitioner/tests/unit/suspend_thread.cpp index 694cc31be2ea..71f1d647715c 100644 --- a/libs/core/resource_partitioner/tests/unit/suspend_thread.cpp +++ b/libs/core/resource_partitioner/tests/unit/suspend_thread.cpp @@ -23,19 +23,20 @@ #include #include -std::size_t const max_threads = (std::min)( - std::size_t(4), std::size_t(hpx::threads::hardware_concurrency())); +std::size_t const max_threads = (std::min)(static_cast(4), + static_cast(hpx::threads::hardware_concurrency())); int hpx_main() { std::size_t const num_threads = hpx::resource::get_num_threads("default"); - HPX_TEST_EQ(std::size_t(max_threads), num_threads); + HPX_TEST_EQ(static_cast(max_threads), num_threads); hpx::threads::thread_pool_base& tp = hpx::resource::get_thread_pool("default"); - HPX_TEST_EQ(tp.get_active_os_thread_count(), std::size_t(max_threads)); + HPX_TEST_EQ( + tp.get_active_os_thread_count(), static_cast(max_threads)); { // Check number of used resources @@ -43,7 +44,7 @@ int hpx_main() ++thread_num) { hpx::threads::suspend_processing_unit(tp, thread_num).get(); - HPX_TEST_EQ(std::size_t(num_threads - thread_num - 1), + HPX_TEST_EQ(static_cast(num_threads - thread_num - 1), tp.get_active_os_thread_count()); } @@ -51,8 +52,8 @@ int hpx_main() ++thread_num) { hpx::threads::resume_processing_unit(tp, thread_num).get(); - HPX_TEST_EQ( - std::size_t(thread_num + 2), tp.get_active_os_thread_count()); + HPX_TEST_EQ(static_cast(thread_num + 2), + tp.get_active_os_thread_count()); } } @@ -61,7 +62,7 @@ int hpx_main() // NOTE: This only works as long as there is another OS thread which has // no work and is able to steal. - std::size_t worker_thread_num = hpx::get_worker_thread_num(); + std::size_t const worker_thread_num = hpx::get_worker_thread_num(); hpx::threads::suspend_processing_unit(tp, worker_thread_num).get(); hpx::threads::resume_processing_unit(tp, worker_thread_num).get(); } @@ -150,13 +151,20 @@ int hpx_main() std::size_t thread_num = 0; bool up = true; std::vector> fs; - hpx::chrono::high_resolution_timer t; + + hpx::threads::thread_schedule_hint hint; + hint.runs_as_child_mode(hpx::threads::thread_execution_hint::none); + + hpx::launch::async_policy policy; + policy.set_hint(hint); + + hpx::chrono::high_resolution_timer const t; while (t.elapsed() < 2) { for (std::size_t i = 0; i < hpx::resource::get_num_threads("default") * 10; ++i) { - fs.push_back(hpx::async([]() {})); + fs.push_back(hpx::async(policy, []() {})); } if (up) @@ -189,7 +197,7 @@ int hpx_main() } } - hpx::when_all(std::move(fs)).get(); + hpx::wait_all(std::move(fs)); // Don't exit with suspended pus for (std::size_t thread_num_resume = 0; thread_num_resume < thread_num; @@ -226,7 +234,7 @@ int main(int argc, char* argv[]) { // These schedulers should succeed - std::vector schedulers = { + std::vector const schedulers = { hpx::resource::scheduling_policy::local, hpx::resource::scheduling_policy::local_priority_fifo, #if defined(HPX_HAVE_CXX11_STD_ATOMIC_128BIT) @@ -247,7 +255,7 @@ int main(int argc, char* argv[]) { // These schedulers should fail - std::vector schedulers = { + std::vector const schedulers = { hpx::resource::scheduling_policy::static_, hpx::resource::scheduling_policy::static_priority, }; diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 1bc5a5e89191..7758770b9306 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -1176,9 +1176,6 @@ namespace hpx::threads::policies { parameters_.small_stacksize_, thread_id_addref::no); HPX_ASSERT(p); - // We initialize the stack eagerly - p->init(); - // Finally, store the thread for later use thread_heap_small_.emplace_back(p); } From 63b011fb12155f63962b9518f5ff7e4a38f0ae48 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Fri, 28 Jul 2023 18:13:20 -0500 Subject: [PATCH 082/333] Fix memory leak when not using guard pages --- .../hpx/coroutines/detail/posix_utility.hpp | 19 ++++++++++++------- .../synchronization/counting_semaphore.hpp | 2 +- .../hpx/synchronization/sliding_semaphore.hpp | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libs/core/coroutines/include/hpx/coroutines/detail/posix_utility.hpp b/libs/core/coroutines/include/hpx/coroutines/detail/posix_utility.hpp index 4bc4b248eda3..8de7ad66a0b0 100644 --- a/libs/core/coroutines/include/hpx/coroutines/detail/posix_utility.hpp +++ b/libs/core/coroutines/include/hpx/coroutines/detail/posix_utility.hpp @@ -83,16 +83,21 @@ namespace hpx::threads::coroutines::detail::posix { inline void* alloc_stack(std::size_t size) { - void* real_stack = - ::mmap(nullptr, size + EXEC_PAGESIZE, PROT_READ | PROT_WRITE, +#if defined(HPX_HAVE_THREAD_GUARD_PAGE) + if (use_guard_pages) + { + size += EXEC_PAGESIZE; + } +#endif + void* real_stack = ::mmap(nullptr, size, PROT_READ | PROT_WRITE, #if defined(__APPLE__) - MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, #elif defined(__FreeBSD__) - MAP_PRIVATE | MAP_ANON, + MAP_PRIVATE | MAP_ANON, #else - MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, #endif - -1, 0); + -1, 0); if (real_stack == MAP_FAILED) { @@ -112,7 +117,7 @@ namespace hpx::threads::coroutines::detail::posix { #if defined(HPX_HAVE_THREAD_GUARD_PAGE) if (use_guard_pages) { - // Add a guard page. + // Set the guard page. ::mprotect(real_stack, EXEC_PAGESIZE, PROT_NONE); void** stack = static_cast(real_stack) + diff --git a/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp index 8a09113a611f..07b33e7e89f1 100644 --- a/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/counting_semaphore.hpp @@ -10,7 +10,7 @@ #pragma once #include -#include +#include #include #include #include diff --git a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp index 6bcfb3e24b44..1402d8d44151 100644 --- a/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp +++ b/libs/core/synchronization/include/hpx/synchronization/sliding_semaphore.hpp @@ -7,7 +7,7 @@ #pragma once #include -#include +#include #include #include From bf60bf4a914f4375756ed97a65f5656d8d7b6cd5 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 3 Jun 2023 06:21:25 -0500 Subject: [PATCH 083/333] move as buffers when data type is trivially copyable Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 856ec14f4538..c86b213542fc 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -7,6 +7,7 @@ /// \file parallel/algorithms/uninitialized_copy.hpp #pragma once +#include #if defined(DOXYGEN) namespace hpx { @@ -252,15 +253,23 @@ namespace hpx::parallel { InIter1 first, std::size_t count, InIter2 dest, util::cancellation_token& tok) { - return {std::next(first, count), - util::loop_with_cleanup_n_with_token( - first, count, dest, tok, - [](InIter1 it, InIter2 dest) -> void { - hpx::construct_at(std::addressof(*dest), *it); - }, - [](InIter2 dest) -> void { - std::destroy_at(std::addressof(*dest)); - })}; + using T = ::hpx::traits::iter_value_t; + if constexpr (std::is_trivially_copyable_v) + { + InIter1 dest = std::advance(first, count); + std::memcpy(std::addressof(*first), std::addressof(*dest), + count * sizeof(T)); + } + else + return {std::next(first, count), + util::loop_with_cleanup_n_with_token( + first, count, dest, tok, + [](InIter1 it, InIter2 dest) -> void { + hpx::construct_at(std::addressof(*dest), *it); + }, + [](InIter2 dest) -> void { + std::destroy_at(std::addressof(*dest)); + })}; } /////////////////////////////////////////////////////////////////////// @@ -334,10 +343,8 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_copy( - first, dest, [last](InIter1 first, FwdIter2) -> bool { - return first != last; - }); + return sequential_uninitialized_copy_n( + first, std::distance(first, last), dest); } template ; + if constexpr (std::is_trivially_copyable_v) + { + std::memcpy(std::addressof(*first), + std::addressof(*dest), count * sizeof(T)); + } for (/* */; count > 0; ++first, (void) ++current, --count) { hpx::construct_at(std::addressof(*current), *first); From 2d1ad02aa04557b8cad6b02d5e4922ea483433a2 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 11:49:29 -0500 Subject: [PATCH 084/333] copy as buffers for triviably copyable types Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index c86b213542fc..f09da5046155 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -235,7 +235,7 @@ namespace hpx::parallel { { hpx::construct_at(std::addressof(*current), *first); } - return util::in_out_result{first, current}; + return {first, current}; } catch (...) { @@ -256,11 +256,15 @@ namespace hpx::parallel { using T = ::hpx::traits::iter_value_t; if constexpr (std::is_trivially_copyable_v) { - InIter1 dest = std::advance(first, count); - std::memcpy(std::addressof(*first), std::addressof(*dest), + std::memcpy(std::addressof(*dest), std::addressof(*first), count * sizeof(T)); + std::advance(first, count); + std::advance(dest, count); + + return util::in_out_result{first, dest}; } else + { return {std::next(first, count), util::loop_with_cleanup_n_with_token( first, count, dest, tok, @@ -270,6 +274,7 @@ namespace hpx::parallel { [](InIter2 dest) -> void { std::destroy_at(std::addressof(*dest)); })}; + } } /////////////////////////////////////////////////////////////////////// @@ -424,29 +429,39 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { - FwdIter2 current = dest; - try + using T = ::hpx::traits::iter_value_t; + + if constexpr (std::is_trivially_copyable_v) { - using T = ::hpx::traits::iter_value_t; - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*first), - std::addressof(*dest), count * sizeof(T)); - } - for (/* */; count > 0; ++first, (void) ++current, --count) - { - hpx::construct_at(std::addressof(*current), *first); - } - return util::in_out_result{ - first, current}; + std::memcpy(std::addressof(*dest), std::addressof(*first), + count * sizeof(T)); + std::advance(first, count); + std::advance(dest, count); + + return util::in_out_result{first, dest}; } - catch (...) + else { - for (/* */; dest != current; ++dest) + FwdIter2 current = dest; + + try + { + for (/* */; count > 0; + ++first, (void) ++current, --count) + { + hpx::construct_at(std::addressof(*current), *first); + } + return util::in_out_result{ + first, current}; + } + catch (...) { - std::destroy_at(std::addressof(*dest)); + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; } - throw; } } From c23155828b5b8a2d46712e3e50e84544990c30d3 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 13:53:22 -0500 Subject: [PATCH 085/333] added cancellation token Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index f09da5046155..add66abe7f66 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -7,7 +7,6 @@ /// \file parallel/algorithms/uninitialized_copy.hpp #pragma once -#include #if defined(DOXYGEN) namespace hpx { @@ -348,8 +347,10 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy, InIter1 first, Sent last, FwdIter2 dest) { + util::cancellation_token tok; + return sequential_uninitialized_copy_n( - first, std::distance(first, last), dest); + first, std::distance(first, last), dest, tok); } template Date: Sun, 4 Jun 2023 14:50:03 -0500 Subject: [PATCH 086/333] added unseq support Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 118 ++++++++++++++++-- .../include/hpx/parallel/unseq/loop.hpp | 113 +++++++++++++++++ 2 files changed, 224 insertions(+), 7 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index add66abe7f66..414f5a657b34 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -198,6 +198,7 @@ namespace hpx { #include #include #include +#include #include #include #include @@ -210,6 +211,7 @@ namespace hpx { #include #include +#include #include #include #include @@ -246,10 +248,63 @@ namespace hpx::parallel { } } + template + util::in_out_result sequential_uninitialized_copy_n( + InIter1 first, FwdIter2 dest, std::size_t count, + std::true_type /*is_unseq*/) + { + FwdIter2 current = dest; + try + { + // clang-format on + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/* */; count != 0; + (void) ++first, + ++current, count--) + { + hpx::construct_at(std::addressof(*current), *first); + } + // clang-format off + + return {first, current}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + + template + util::in_out_result sequential_uninitialized_copy_n( + InIter1 first, FwdIter2 dest, std::size_t count, + std::false_type /*is_unseq*/) + { + FwdIter2 current = dest; + try + { + for (/* */; count != 0; (void) ++first, ++current, count--) + { + hpx::construct_at(std::addressof(*current), *first); + } + return {first, current}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + /////////////////////////////////////////////////////////////////////// template util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, std::size_t count, InIter2 dest, + InIter1 first, InIter2 dest, std::size_t count, util::cancellation_token& tok) { using T = ::hpx::traits::iter_value_t; @@ -308,7 +363,7 @@ namespace hpx::parallel { return std::make_pair(dest, util::get_second_element( sequential_uninitialized_copy_n( - get<0>(iters), part_size, dest, tok))); + get<0>(iters), dest, part_size, tok))); }, // finalize, called once if no error occurred [dest, first, count](auto&& data) mutable @@ -345,12 +400,10 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { - util::cancellation_token tok; - return sequential_uninitialized_copy_n( - first, std::distance(first, last), dest, tok); + ExPolicy, first, dest, std::distance(first, last)); } template + + // vectorized overload + template >> + static util::in_out_result sequential( + ExPolicy, InIter first, std::size_t count, FwdIter2 dest) + { + using T = ::hpx::traits::iter_value_t; + + if constexpr (std::is_trivially_copyable_v) + { + std::memcpy(std::addressof(*dest), std::addressof(*first), + count * sizeof(T)); + std::advance(first, count); + std::advance(dest, count); + + return util::in_out_result{first, dest}; + } + else + { + FwdIter2 current = dest; + + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/* */; count > 0; + ++first, (void) ++current, --count) + { + hpx::construct_at(std::addressof(*current), *first); + } + // clang-format on + + return util::in_out_result{ + first, current}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + } + + // non vectorized overload + template >> static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { diff --git a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp index 32fa4ec41275..131f62492014 100644 --- a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp @@ -200,6 +200,119 @@ namespace hpx::parallel::util { return call(base_idx, it, count, HPX_FORWARD(F, f)); } }; + + template + struct unseq_loop_with_cleanup_n + { + /////////////////////////////////////////////////////////////////// + template + static FwdIter call( + FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + { + FwdIter base = it; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < count; + (void) ++it, i += 4) // -V112 + { + HPX_INVOKE(f, it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it) + { + HPX_INVOKE(f, it); + } + // clang-format on + + return it; + } + catch (...) + { + for (/**/; base != it; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + + template + static FwdIter call(Iter it, std::size_t num, FwdIter dest, F&& f, + Cleanup&& cleanup) + { + FwdIter base = dest; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + + //clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; + i < count; + (void) ++it, ++dest, + i += 4) // -V112 + { + HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); + } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; count < num; + (void) ++count, + ++it, ++dest) + { + HPX_INVOKE(f, it, dest); + } + //clang-format on + + return dest; + } + catch (...) + { + for (/**/; base != dest; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token( + FwdIter it, std::size_t num, CancelToken& tok, F&& f, + Cleanup&& cleanup) + { + // check at the start of a partition only + if (tok.was_cancelled()) + return it; + + return call( + it, num, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + static FwdIter call_with_token(Iter it, std::size_t num, + FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) + { + // check at the start of a partition only + if (tok.was_cancelled()) + return dest; + + return call(it, num, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + }; } // namespace detail /////////////////////////////////////////////////////////////////////////// From dd488ca3fb80292ace032ae2dbf3b101cc1a95f3 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 15:24:54 -0500 Subject: [PATCH 087/333] fixed unused error Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 414f5a657b34..c96fdacb5b32 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -403,7 +403,7 @@ namespace hpx::parallel { ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { return sequential_uninitialized_copy_n( - ExPolicy, first, dest, std::distance(first, last)); + ExPolicy policy, first, dest, std::distance(first, last)); } template >> + template static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { From 1811f81778fa56cc8bfd2f6d8799fbaf30d62b6c Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 15:32:28 -0500 Subject: [PATCH 088/333] unedd parameter Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index c96fdacb5b32..fbe33787c913 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -403,7 +403,7 @@ namespace hpx::parallel { ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { return sequential_uninitialized_copy_n( - ExPolicy policy, first, dest, std::distance(first, last)); + policy, first, dest, std::distance(first, last)); } template Date: Sun, 4 Jun 2023 19:04:57 -0500 Subject: [PATCH 089/333] enable if else error to be fixed Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_copy.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index fbe33787c913..2e8e649cf5bf 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -256,14 +256,14 @@ namespace hpx::parallel { FwdIter2 current = dest; try { - // clang-format on + // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/* */; count != 0; (void) ++first, ++current, count--) { hpx::construct_at(std::addressof(*current), *first); } - // clang-format off + // clang-format on return {first, current}; } @@ -402,8 +402,9 @@ namespace hpx::parallel { static util::in_out_result sequential( ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_copy_n( - policy, first, dest, std::distance(first, last)); + return sequential_uninitialized_copy_n(first, dest, + std::distance(first, last), + hpx::is_unsequenced_execution_policy_v()); } template Date: Sun, 4 Jun 2023 19:08:37 -0500 Subject: [PATCH 090/333] cf Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 2e8e649cf5bf..5d3100a1ac85 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -480,7 +480,6 @@ namespace hpx::parallel { { } - // vectorized overload template Date: Sun, 4 Jun 2023 20:09:18 -0500 Subject: [PATCH 091/333] unused param Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 5d3100a1ac85..d0108f4dc383 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -400,7 +400,7 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy policy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy, InIter1 first, Sent last, FwdIter2 dest) { return sequential_uninitialized_copy_n(first, dest, std::distance(first, last), From b354e87737c158cd5c12859c2a7fdcafb7db5e62 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 04:59:55 -0500 Subject: [PATCH 092/333] customisation point for loop_with_cleanup Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 5 +- .../include/hpx/parallel/util/loop.hpp | 185 ++++++++++++------ 2 files changed, 130 insertions(+), 60 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index d0108f4dc383..ec05c700edaf 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -257,9 +257,8 @@ namespace hpx::parallel { try { // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/* */; count != 0; - (void) ++first, - ++current, count--) + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/* */; count != 0; (void) ++first, ++current, count--) { hpx::construct_at(std::addressof(*current), *first); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 8071cd73ff50..10eaf4ba6d43 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -38,23 +38,65 @@ namespace hpx::parallel::util { HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( Begin it, End end, F&& f) { + return call(hpx::execution::seq, it, end, HPX_FORWARD(F, f)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( + Begin it, End end, CancelToken& tok, F&& f) + { + return call( + hpx::execution::seq, it, end, tok, HPX_FORWARD(F, f)); + } + + template && + !hpx::is_parallel_execution_policy_v)> + HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( + ExPolicy, Begin it, End end, F&& f) + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != end; ++it) { HPX_INVOKE(f, it); } + // clang-format on + return it; } - template + template && + !hpx::is_parallel_execution_policy_v)> + HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( + ExPolicy, Begin it, End end, F&& f) + { + for (/**/; it != end; ++it) + { + HPX_INVOKE(f, it); + } + + return it; + } + + template HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( - Begin it, End end, CancelToken& tok, F&& f) + typename ExPolicy policy, Begin it, End end, CancelToken& tok, + F&& f) { // check at the start of a partition only if (tok.was_cancelled()) return it; - return call(it, end, HPX_FORWARD(F, f)); + return call( + HPX_FORWARD(ExPolicy, policy), it, end, HPX_FORWARD(F, f)); } }; } // namespace detail @@ -64,10 +106,11 @@ namespace hpx::parallel::util { private: template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&&, + tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&& policy, Begin begin, End end, F&& f) { - return detail::loop::call(begin, end, HPX_FORWARD(F, f)); + return detail::loop::call( + HPX_FORWARD(ExPolicy, policy), begin, end, HPX_FORWARD(F, f)); } template ::call( + return detail::loop::call(HPX_FORWARD(ExPolicy, policy), begin, end, tok, HPX_FORWARD(F, f)); } }; @@ -575,31 +618,50 @@ namespace hpx::parallel::util { struct loop_with_cleanup { /////////////////////////////////////////////////////////////////// - template - static FwdIter call( - FwdIter it, FwdIter last, F&& f, Cleanup&& cleanup) + template + static FwdIter call(ExPolicy&& policy, FwdIter first, FwdIter last, + F&& f, Cleanup&& cleanup) { - FwdIter base = it; + return call(HPX_FORWARD(ExPolicy, policy), first, last, first, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template && + !hpx::is_parallel_execution_policy_v)> + static FwdIter call(ExPolicy&& policy, Iter it, Iter last, + FwdIter dest, F&& f, Cleanup&& cleanup) + { + FwdIter base = dest; try { - for (/**/; it != last; ++it) - { - HPX_INVOKE(f, it); - } - return it; + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != last; (void) ++it, ++dest) + f(it, dest); + // clang-format on + + return dest; } catch (...) { - for (/**/; base != it; ++base) - cleanup(base); + for (/**/; base != dest; ++base) + { + HPX_INVOKE(cleanup, base); + } throw; } } - template - static FwdIter call( - Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) + template )> + static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, + F&& f, Cleanup&& cleanup) { FwdIter base = dest; try @@ -620,24 +682,52 @@ namespace hpx::parallel::util { }; } // namespace detail - /////////////////////////////////////////////////////////////////////////// - template - HPX_FORCEINLINE constexpr Iter loop_with_cleanup( - Iter it, Iter last, F&& f, Cleanup&& cleanup) + inline constexpr struct loop_with_cleanup_t final + : hpx::functional::detail::tag_fallback { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup::call( - it, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } + private: + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin + tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, + ExPolicy&& policy, Iter it, Iter last, FwdIter dest, F&& f, + Cleanup&& cleanup) + { + return detail::loop_with_cleanup::call( + HPX_FORWARD(ExPolicy, policy), it, last, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } - template - HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup( - Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) - { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup::call( - it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin + tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, ExPolicy&&, + ExPolicy&& policy, FwdIter first, FwdIter last, F&& f, + Cleanup&& cleanup) + { + return detail::loop_with_cleanup::call( + HPX_FORWARD(ExPolicy, policy), it, last, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + } loop_with_cleanup{}; + + // /////////////////////////////////////////////////////////////////////////// + // template + // HPX_FORCEINLINE constexpr Iter loop_with_cleanup( + // Iter it, Iter last, F&& f, Cleanup&& cleanup) + // { + // using cat = typename std::iterator_traits::iterator_category; + // return detail::loop_with_cleanup::call( + // it, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + // } + + // template + // HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup( + // Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) + // { + // using cat = typename std::iterator_traits::iterator_category; + // return detail::loop_with_cleanup::call( + // it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + // } /////////////////////////////////////////////////////////////////////////// namespace detail { @@ -742,25 +832,6 @@ namespace hpx::parallel::util { }; } // namespace detail - /////////////////////////////////////////////////////////////////////////// - template - HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n( - Iter it, std::size_t count, F&& f, Cleanup&& cleanup) - { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call( - it, count, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } - - template - HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup_n( - Iter it, std::size_t count, FwdIter dest, F&& f, Cleanup&& cleanup) - { - using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call( - it, count, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } - template HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) From 7d9e1eb75b88486b699269807c65f0403c2566c1 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:04:15 -0500 Subject: [PATCH 093/333] fixed ambiguos funtion error Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_copy.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index ec05c700edaf..99f5c9fd2e43 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -481,8 +481,9 @@ namespace hpx::parallel { // vectorized overload template >> + HPX_CONCEPT_REQUIRES_( // forces hpx::execution::unseq + hpx::is_unsequenced_execution_policy_v && + !hpx::is_parallel_execution_policy_v)> static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { @@ -527,7 +528,9 @@ namespace hpx::parallel { } // non vectorized overload - template + template )> static util::in_out_result sequential( ExPolicy, InIter first, std::size_t count, FwdIter2 dest) { From d32c15dbacfe1b09e4451fd2a39b0864ff25670d Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:17:19 -0500 Subject: [PATCH 094/333] fixed template and typename mismatch Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 10eaf4ba6d43..ad82a5f4564b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -56,7 +56,7 @@ namespace hpx::parallel::util { hpx::is_unsequenced_execution_policy_v && !hpx::is_parallel_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy, Begin it, End end, F&& f) + ExPolicy&&, Begin it, End end, F&& f) { // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE @@ -70,12 +70,12 @@ namespace hpx::parallel::util { } template && !hpx::is_parallel_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy, Begin it, End end, F&& f) + ExPolicy&&, Begin it, End end, F&& f) { for (/**/; it != end; ++it) { @@ -88,8 +88,7 @@ namespace hpx::parallel::util { template HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( - typename ExPolicy policy, Begin it, End end, CancelToken& tok, - F&& f) + ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) { // check at the start of a partition only if (tok.was_cancelled()) @@ -116,7 +115,7 @@ namespace hpx::parallel::util { template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&&, + tag_fallback_invoke(hpx::parallel::util::loop_t, ExPolicy&& policy, Begin begin, End end, CancelToken& tok, F&& f) { return detail::loop::call(HPX_FORWARD(ExPolicy, policy), @@ -632,8 +631,8 @@ namespace hpx::parallel::util { HPX_CONCEPT_REQUIRES_( // forces hpx::execution::unseq hpx::is_unsequenced_execution_policy_v && !hpx::is_parallel_execution_policy_v)> - static FwdIter call(ExPolicy&& policy, Iter it, Iter last, - FwdIter dest, F&& f, Cleanup&& cleanup) + static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, + F&& f, Cleanup&& cleanup) { FwdIter base = dest; try @@ -686,23 +685,22 @@ namespace hpx::parallel::util { : hpx::functional::detail::tag_fallback { private: - template + template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, - ExPolicy&& policy, Iter it, Iter last, FwdIter dest, F&& f, - Cleanup&& cleanup) + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + Begin it, End last, FwdIter dest, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( HPX_FORWARD(ExPolicy, policy), it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - template + template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(hpx::parallel::util::loop_with_cleanup, ExPolicy&&, - ExPolicy&& policy, FwdIter first, FwdIter last, F&& f, - Cleanup&& cleanup) + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + Begin first, End last, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( HPX_FORWARD(ExPolicy, policy), it, last, HPX_FORWARD(F, f), From aaa8f0e02a07506cb9d3367b2fcd6441721152ed Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:50:18 -0500 Subject: [PATCH 095/333] template arguments were not provided in certain cases Signed-off-by: Hari Hara Naveen S --- .../core/algorithms/include/hpx/parallel/util/loop.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index ad82a5f4564b..2729088b06c7 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -688,22 +688,22 @@ namespace hpx::parallel::util { template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, - Begin it, End last, FwdIter dest, F&& f, Cleanup&& cleanup) + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), it, last, dest, + HPX_FORWARD(ExPolicy, policy), first, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } template friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, + tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, Begin first, End last, F&& f, Cleanup&& cleanup) { return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), it, last, HPX_FORWARD(F, f), + HPX_FORWARD(ExPolicy, policy), first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } } loop_with_cleanup{}; From 002ca1977acbdf81a143df15670bd661210fd471 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 06:00:47 -0500 Subject: [PATCH 096/333] added concept to resolve ambiguity Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 2729088b06c7..da5dade885d0 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -86,7 +86,8 @@ namespace hpx::parallel::util { } template + typename CancelToken, typename F, + HPX_CONCEPT_REQUIRES_(hpx::is_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) { From 442e1d876e730c42cf21a6c0a09c4c8cc5b9bd08 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 09:22:02 -0500 Subject: [PATCH 097/333] fixed ambiguity using concepts Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index da5dade885d0..90ed618a28c1 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -42,7 +42,9 @@ namespace hpx::parallel::util { } template + typename F, + HPX_CONCEPT_REQUIRES_(hpx::traits::is_iterator_v&& + hpx::traits::is_forward_iterator_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( Begin it, End end, CancelToken& tok, F&& f) { @@ -72,8 +74,7 @@ namespace hpx::parallel::util { template && - !hpx::is_parallel_execution_policy_v)> + hpx::is_sequenced_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) { From 1af6fd739efc63a3139eaaaa81486934c5b6763a Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 10:54:22 -0500 Subject: [PATCH 098/333] fixed inspect and include errors Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 99f5c9fd2e43..a59b8118dee6 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -403,7 +403,7 @@ namespace hpx::parallel { { return sequential_uninitialized_copy_n(first, dest, std::distance(first, last), - hpx::is_unsequenced_execution_policy_v()); + hpx::is_unsequenced_execution_policy_v); } template #include #include +#include #include #include #include @@ -61,7 +62,7 @@ namespace hpx::parallel::util { ExPolicy&&, Begin it, End end, F&& f) { // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != end; ++it) { HPX_INVOKE(f, it); From 4d6107f589eaccc7a1268e04b0cd31340cc65123 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 01:22:39 -0500 Subject: [PATCH 099/333] used hpx::parallel::util::transfer for uninit_copy Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 229 ++---------------- .../include/hpx/parallel/util/transfer.hpp | 114 +++++++++ 2 files changed, 139 insertions(+), 204 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index a59b8118dee6..2b5c0fc4fd3a 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -206,6 +206,7 @@ namespace hpx { #include #include #include +#include #include #include @@ -225,116 +226,11 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// - template - util::in_out_result sequential_uninitialized_copy( - InIter1 first, FwdIter2 dest, Cond cond) - { - FwdIter2 current = dest; - try - { - for (/* */; cond(first, current); (void) ++first, ++current) - { - hpx::construct_at(std::addressof(*current), *first); - } - return {first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template - util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, FwdIter2 dest, std::size_t count, - std::true_type /*is_unseq*/) - { - FwdIter2 current = dest; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/* */; count != 0; (void) ++first, ++current, count--) - { - hpx::construct_at(std::addressof(*current), *first); - } - // clang-format on - - return {first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template - util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, FwdIter2 dest, std::size_t count, - std::false_type /*is_unseq*/) - { - FwdIter2 current = dest; - try - { - for (/* */; count != 0; (void) ++first, ++current, count--) - { - hpx::construct_at(std::addressof(*current), *first); - } - return {first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - /////////////////////////////////////////////////////////////////////// - template - util::in_out_result sequential_uninitialized_copy_n( - InIter1 first, InIter2 dest, std::size_t count, - util::cancellation_token& tok) - { - using T = ::hpx::traits::iter_value_t; - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*dest), std::addressof(*first), - count * sizeof(T)); - std::advance(first, count); - std::advance(dest, count); - - return util::in_out_result{first, dest}; - } - else - { - return {std::next(first, count), - util::loop_with_cleanup_n_with_token( - first, count, dest, tok, - [](InIter1 it, InIter2 dest) -> void { - hpx::construct_at(std::addressof(*dest), *it); - }, - [](InIter2 dest) -> void { - std::destroy_at(std::addressof(*dest)); - })}; - } - } - /////////////////////////////////////////////////////////////////////// template typename util::detail::algorithm_result>::type - parallel_sequential_uninitialized_copy_n( + parallel_uninitialized_copy_n( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { if (count == 0) @@ -354,14 +250,15 @@ namespace hpx::parallel { call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [tok](zip_iterator t, std::size_t part_size) mutable + [tok, policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); FwdIter2 dest = get<1>(iters); return std::make_pair(dest, util::get_second_element( - sequential_uninitialized_copy_n( + hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), get<0>(iters), dest, part_size, tok))); }, // finalize, called once if no error occurred @@ -399,11 +296,11 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy&& policy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_copy_n(first, dest, - std::distance(first, last), - hpx::is_unsequenced_execution_policy_v); + std::size_t n = std::distance(first, last); + return hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), first, n, dest); } template >::type parallel(ExPolicy&& policy, Iter first, Sent last, FwdIter2 dest) { - return parallel_sequential_uninitialized_copy_n( + return parallel_uninitialized_copy_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last), dest); } @@ -437,13 +334,16 @@ namespace hpx::parallel { template - static util::in_out_result sequential(ExPolicy, - InIter1 first, Sent1 last, FwdIter2 dest, Sent2 last_d) + static util::in_out_result sequential( + ExPolicy&& policy, InIter1 first, Sent1 last, FwdIter2 dest, + Sent2 last_d) { - return sequential_uninitialized_copy(first, dest, - [last, last_d](InIter1 first, FwdIter2 current) -> bool { - return !(first == last || current == last_d); - }); + std::size_t const dist1 = detail::distance(first, last); + std::size_t const dist2 = detail::distance(dest, last_d); + std::size_t dist = dist1 <= dist2 ? dist1 : dist2; + + return hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), first, dist, dest); } template && - !hpx::is_parallel_execution_policy_v)> - static util::in_out_result sequential( - ExPolicy, InIter first, std::size_t count, FwdIter2 dest) - { - using T = ::hpx::traits::iter_value_t; - - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*dest), std::addressof(*first), - count * sizeof(T)); - std::advance(first, count); - std::advance(dest, count); - - return util::in_out_result{first, dest}; - } - else - { - FwdIter2 current = dest; - - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/* */; count > 0; - ++first, (void) ++current, --count) - { - hpx::construct_at(std::addressof(*current), *first); - } - // clang-format on - - return util::in_out_result{ - first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - } - // non vectorized overload template )> static util::in_out_result sequential( - ExPolicy, InIter first, std::size_t count, FwdIter2 dest) + ExPolicy&& policy, InIter first, std::size_t count, + FwdIter2 dest) { - using T = ::hpx::traits::iter_value_t; - - if constexpr (std::is_trivially_copyable_v) - { - std::memcpy(std::addressof(*dest), std::addressof(*first), - count * sizeof(T)); - std::advance(first, count); - std::advance(dest, count); - - return util::in_out_result{first, dest}; - } - else - { - FwdIter2 current = dest; - - try - { - for (/* */; count > 0; - ++first, (void) ++current, --count) - { - hpx::construct_at(std::addressof(*current), *first); - } - return util::in_out_result{ - first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } + return hpx::parallel::util::uninit_copy_n( + HPX_FORWARD(ExPolicy, policy), first, count, dest); } template @@ -576,7 +397,7 @@ namespace hpx::parallel { parallel( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { - return parallel_sequential_uninitialized_copy_n( + return parallel_uninitialized_copy_n( HPX_FORWARD(ExPolicy, policy), first, count, dest); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index dd93874fff7e..c67139787914 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -38,6 +38,9 @@ namespace hpx::parallel::util { template struct move_n_helper; + template + struct uninit_copy_n_helper; + /////////////////////////////////////////////////////////////////////// template HPX_FORCEINLINE std::enable_if_t, char*> to_ptr( @@ -348,4 +351,115 @@ namespace hpx::parallel::util { std::decay_t>; return detail::move_n_helper::call(first, count, dest); } + + // helpers for uninit_copy_n + namespace detail { + // Customization point for optimizing copy_n operations + template + struct uninit_copy_n_helper + { + template )> + HPX_FORCEINLINE static constexpr in_out_result + call(ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + + try + { + std::size_t count( + num & static_cast(-4)); // -V112 + for (std::size_t i = 0; i < count; i += 4) //-V112 + { + hpx::construct_at(std::addressof(*current++), *first++); + // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) + hpx::construct_at(std::addressof(*current++), *first++); + // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) + hpx::construct_at(std::addressof(*current++), *first++); + // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) + hpx::construct_at(std::addressof(*current++), *first++); + } + for (/**/; count < num; count++) + { + hpx::construct_at(std::addressof(*current++), *first++); + } + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + + template )> + HPX_FORCEINLINE static constexpr in_out_result + call(ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < num; i++) //-V112 + { + hpx::construct_at(std::addressof(*current++), *first++); + } + // clang-format on + + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + }; + + template + struct uninit_copy_n_helper + { + template + HPX_FORCEINLINE static in_out_result call(ExPolicy, + InIter first, std::size_t count, OutIter dest) noexcept + { + return copy_memmove(first, count, dest); + } + }; + } // namespace detail + + template + HPX_FORCEINLINE constexpr in_out_result uninit_copy_n( + InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_copy_n_helper::call( + hpx::execution::seq, first, count, dest); + } + + template + HPX_FORCEINLINE constexpr in_out_result uninit_copy_n( + ExPolicy&& policy, InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_copy_n_helper::call( + HPX_FORWARD(ExPolicy, policy), first, count, dest); + } } // namespace hpx::parallel::util From 7a8dc58008d6aef93813d85bed377909920da9ab Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 02:04:49 -0500 Subject: [PATCH 100/333] try in constexpr function fixed Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index c67139787914..8226d757aa7b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -17,6 +17,7 @@ #include #include // for std::memmove #include +#include #include #include @@ -361,7 +362,7 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static constexpr in_out_result + HPX_FORCEINLINE static in_out_result call(ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; @@ -400,7 +401,7 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static constexpr in_out_result + HPX_FORCEINLINE static in_out_result call(ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; From 3fe0b6b64de3d09df5128c723e20b24cb7b8f11f Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 04:22:09 -0500 Subject: [PATCH 101/333] fixed missing includes Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_copy.hpp | 6 ++---- .../algorithms/include/hpx/parallel/util/transfer.hpp | 10 ++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 2b5c0fc4fd3a..cacc9c3b09a3 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -243,8 +243,6 @@ namespace hpx::parallel { using zip_iterator = hpx::util::zip_iterator; using partition_result_type = std::pair; - util::cancellation_token tok; - return util::partitioner_with_cleanup, partition_result_type>:: call( @@ -259,7 +257,7 @@ namespace hpx::parallel { util::get_second_element( hpx::parallel::util::uninit_copy_n( HPX_FORWARD(ExPolicy, policy), - get<0>(iters), dest, part_size, tok))); + get<0>(iters), part_size, dest))); }, // finalize, called once if no error occurred [dest, first, count](auto&& data) mutable @@ -381,7 +379,7 @@ namespace hpx::parallel { // non vectorized overload template )> static util::in_out_result sequential( ExPolicy&& policy, InIter first, std::size_t count, diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 8226d757aa7b..bee76809bc8b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include #include @@ -362,8 +364,8 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static in_out_result - call(ExPolicy, InIter first, std::size_t num, OutIter dest) + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; @@ -401,8 +403,8 @@ namespace hpx::parallel::util { template )> - HPX_FORCEINLINE static in_out_result - call(ExPolicy, InIter first, std::size_t num, OutIter dest) + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; try From 89b739c3e8d93ecfbc6f5314171053a9d2ecb075 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 04:28:13 -0500 Subject: [PATCH 102/333] variable name Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index cacc9c3b09a3..7fd8495aa91f 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -248,7 +248,7 @@ namespace hpx::parallel { call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [tok, policy](zip_iterator t, std::size_t part_size) mutable + [policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); From ad9bda01bf8763c0223625e5999db43eba688d5e Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 05:36:35 -0500 Subject: [PATCH 103/333] changed concept Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index bee76809bc8b..82739b98f695 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -402,7 +402,7 @@ namespace hpx::parallel::util { template )> + !hpx::is_unsequenced_execution_policy_v)> HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { From 9f5b1cd0c939ee72fd0044793b3c73bb4819abcf Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 9 Jun 2023 06:40:48 -0500 Subject: [PATCH 104/333] fixed concept ambiguity Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 82739b98f695..668453c21c13 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -363,7 +363,7 @@ namespace hpx::parallel::util { { template )> + !hpx::is_unsequenced_execution_policy_v)> HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { @@ -402,7 +402,7 @@ namespace hpx::parallel::util { template )> + hpx::is_unsequenced_execution_policy_v)> HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { From ba6878c788415bc2b011f292cb0da0493c62dd6b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 10 Jun 2023 04:01:55 -0500 Subject: [PATCH 105/333] fixes extra free which happens during tests Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/transfer.hpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 668453c21c13..2e63c86dcb13 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -372,20 +372,21 @@ namespace hpx::parallel::util { try { std::size_t count( - num & static_cast(-4)); // -V112 - for (std::size_t i = 0; i < count; i += 4) //-V112 + num & static_cast(-4)); // -V112 + for (std::size_t i = 0; i < count; + i += 4, ++current, ++first) //-V112 { - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*current), *first); // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); } for (/**/; count < num; count++) { - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); } return in_out_result{ HPX_MOVE(first), HPX_MOVE(current)}; @@ -413,7 +414,7 @@ namespace hpx::parallel::util { HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < num; i++) //-V112 { - hpx::construct_at(std::addressof(*current++), *first++); + hpx::construct_at(std::addressof(*++current), *++first); } // clang-format on From 33d2598018dba7d35aa55999f26ddb8b3f0aa445 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 10 Jun 2023 08:06:31 -0500 Subject: [PATCH 106/333] fixed bug Signed-off-by: Hari Hara Naveen S --- .../algorithms/include/hpx/parallel/util/transfer.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 2e63c86dcb13..9994aab89c8e 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -384,9 +384,9 @@ namespace hpx::parallel::util { // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) hpx::construct_at(std::addressof(*++current), *++first); } - for (/**/; count < num; count++) + for (/**/; count < num; ++count, ++current, ++first) { - hpx::construct_at(std::addressof(*++current), *++first); + hpx::construct_at(std::addressof(*current), *first); } return in_out_result{ HPX_MOVE(first), HPX_MOVE(current)}; @@ -412,9 +412,9 @@ namespace hpx::parallel::util { { // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (std::size_t i = 0; i < num; i++) //-V112 + for (std::size_t i = 0; i < num; ++i, ++current, ++first) //-V112 { - hpx::construct_at(std::addressof(*++current), *++first); + hpx::construct_at(std::addressof(*current), *first); } // clang-format on From c7a1bd1a91c63e71fe531cd3ba2eb23cd83c341c Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 10 Jun 2023 10:45:54 -0500 Subject: [PATCH 107/333] manual unrolling causing errors Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/transfer.hpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 9994aab89c8e..c3ea0ea173bb 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -371,20 +371,8 @@ namespace hpx::parallel::util { try { - std::size_t count( - num & static_cast(-4)); // -V112 - for (std::size_t i = 0; i < count; - i += 4, ++current, ++first) //-V112 - { - hpx::construct_at(std::addressof(*current), *first); - // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*++current), *++first); - // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*++current), *++first); - // NOLINTNEXTLINE(bugprone-macro-repeated-side-effects) - hpx::construct_at(std::addressof(*++current), *++first); - } - for (/**/; count < num; ++count, ++current, ++first) + for (std::size_t i = 0; i < num; + ++i, ++current, ++first) //-V112 { hpx::construct_at(std::addressof(*current), *first); } From a50008e8de92053eaa0294a36ac1c31f6c53c597 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 12:23:10 -0500 Subject: [PATCH 108/333] uninit default construct Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 83 +++--------- .../include/hpx/parallel/util/loop.hpp | 118 ++++++++++++------ 2 files changed, 96 insertions(+), 105 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 5a485c16718a..c3989037a1cd 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -193,44 +193,18 @@ namespace hpx::parallel { namespace detail { /// \cond NOINTERNAL - // provide our own implementation of - // std::uninitialized_default_construct as some versions of MSVC - // horribly fail at compiling it for some types T - template - Iter std_uninitialized_default_construct(Iter first, Sent last) - { - using value_type = typename std::iterator_traits::value_type; - - Iter s_first = first; - try - { - for (/* */; first != last; ++first) - { - ::new (std::addressof(*first)) value_type; - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - /////////////////////////////////////////////////////////////////////// - template - InIter sequential_uninitialized_default_construct_n(InIter first, - std::size_t count, - util::cancellation_token& tok) + template + InIter sequential_uninitialized_default_construct_n( + ExPolicy&& policy, InIter first, std::size_t count) { using value_type = typename std::iterator_traits::value_type; + util::cancellation_token tok; + return util::loop_with_cleanup_n_with_token( - first, count, tok, + HPX_FORWARD(ExPolicy, policy), first, count, tok, [](InIter it) -> void { ::new (std::addressof(*it)) value_type; }, @@ -242,7 +216,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::detail::algorithm_result_t - parallel_sequential_uninitialized_default_construct_n( + parallel_uninitialized_default_construct_n( ExPolicy&& policy, FwdIter first, std::size_t count) { if (count == 0) @@ -252,17 +226,17 @@ namespace hpx::parallel { } using partition_result_type = std::pair; - util::cancellation_token tok; + return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [tok](FwdIter it, std::size_t part_size) mutable + [tok, policy](FwdIter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_default_construct_n( - it, part_size, tok)); + HPX_FORWARD(ExPolicy, policy), it, part_size)); }, // finalize, called once if no error occurred [first, count](auto&& data) mutable -> FwdIter { @@ -295,16 +269,17 @@ namespace hpx::parallel { } template - static FwdIter sequential(ExPolicy, FwdIter first, Sent last) + static FwdIter sequential(ExPolicy policy, FwdIter first, Sent last) { - return std_uninitialized_default_construct(first, last); + return sequential_uninitialized_default_construct_n( + HPX_FORWARD(ExPolicy, policy), first, std::distance(first, last)); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_sequential_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } @@ -317,34 +292,6 @@ namespace hpx::parallel { namespace detail { /// \cond NOINTERNAL - // provide our own implementation of std::uninitialized_default_construct as some - // versions of MSVC horribly fail at compiling it for some types T - template - InIter std_uninitialized_default_construct_n( - InIter first, std::size_t count) - { - using value_type = - typename std::iterator_traits::value_type; - - InIter s_first = first; - try - { - for (/* */; count != 0; (void) ++first, --count) - { - ::new (std::addressof(*first)) value_type; - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - template struct uninitialized_default_construct_n : public algorithm, @@ -367,7 +314,7 @@ namespace hpx::parallel { static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, std::size_t count) { - return parallel_sequential_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, count); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 0c62a0a72338..322e84997b3b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -739,31 +739,34 @@ namespace hpx::parallel::util { struct loop_with_cleanup_n { /////////////////////////////////////////////////////////////////// - template - static FwdIter call( - FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + template )> + static FwdIter call(ExPolicy, Iter it, std::size_t num, + FwdIter dest, F&& f, Cleanup&& cleanup) { - FwdIter base = it; + FwdIter base = dest; try { std::size_t count(num & std::size_t(-4)); // -V112 for (std::size_t i = 0; i < count; - (void) ++it, i += 4) // -V112 + (void) ++it, ++dest, i += 4) // -V112 { - HPX_INVOKE(f, it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); + HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); + HPX_INVOKE(f, ++it, ++dest); } - for (/**/; count < num; (void) ++count, ++it) + for (/**/; count < num; (void) ++count, ++it, ++dest) { - HPX_INVOKE(f, it); + HPX_INVOKE(f, it, dest); } - return it; + return dest; } catch (...) { - for (/**/; base != it; ++base) + for (/**/; base != dest; ++base) { HPX_INVOKE(cleanup, base); } @@ -771,27 +774,33 @@ namespace hpx::parallel::util { } } - template - static FwdIter call(Iter it, std::size_t num, FwdIter dest, F&& f, - Cleanup&& cleanup) + template )> + static FwdIter call(ExPolicy, Iter it, std::size_t num, + FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; try { std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; (void) ++it, ++dest, i += 4) // -V112 { HPX_INVOKE(f, it, dest); - HPX_INVOKE(f, ++it, ++dest); - HPX_INVOKE(f, ++it, ++dest); - HPX_INVOKE(f, ++it, ++dest); } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; count < num; (void) ++count, ++it, ++dest) { HPX_INVOKE(f, it, dest); } + // clang-format on + return dest; } catch (...) @@ -804,53 +813,88 @@ namespace hpx::parallel::util { } } - template + template HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token( - FwdIter it, std::size_t num, CancelToken& tok, F&& f, - Cleanup&& cleanup) + ExPolicy&& policy, FwdIter it, std::size_t num, + CancelToken& tok, F&& f, Cleanup&& cleanup) { // check at the start of a partition only if (tok.was_cancelled()) return it; - return call( - it, num, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + return call(HPX_FORWARD(ExPolicy, policy), it, num, it, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - template - static FwdIter call_with_token(Iter it, std::size_t num, - FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) + template + static FwdIter call_with_token(ExPolicy&& policy, Iter it, + std::size_t num, FwdIter dest, CancelToken& tok, F&& f, + Cleanup&& cleanup) { // check at the start of a partition only if (tok.was_cancelled()) return dest; - return call(it, num, dest, HPX_FORWARD(F, f), - HPX_FORWARD(Cleanup, cleanup)); + return call(HPX_FORWARD(ExPolicy, policy), it, num, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } }; } // namespace detail - template + template )> HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( - it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); } template + typename Cleanup, + HPX_CONCEPT_REQUIRES_(hpx::traits::is_iterator_v&& + hpx::traits::is_iterator_v)> HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup_n_with_token(Iter it, std::size_t count, FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token(it, count, - dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + return detail::loop_with_cleanup_n::call_with_token( + ::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + + template && + hpx::traits::is_iterator_v)> + HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( + ExPolicy&& policy, Iter it, std::size_t count, CancelToken& tok, F&& f, + Cleanup&& cleanup) + { + using cat = typename std::iterator_traits::iterator_category; + return detail::loop_with_cleanup_n::call_with_token( + HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + + template && hpx::traits::is_iterator_v< + Iter>&& hpx::traits::is_iterator_v)> + HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup_n_with_token( + ExPolicy&& policy, Iter it, std::size_t count, FwdIter dest, + CancelToken& tok, F&& f, Cleanup&& cleanup) + { + using cat = typename std::iterator_traits::iterator_category; + return detail::loop_with_cleanup_n::call_with_token( + HPX_FORWARD(ExPolicy, policy), it, count, dest, tok, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } /////////////////////////////////////////////////////////////////////////// From 0d5c1d9f96c17b2f57f947a3a6ed7778ec15803b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 15:57:17 -0500 Subject: [PATCH 109/333] added another call overload to loop_with_cleanup_n Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 322e84997b3b..01753276ef26 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -789,7 +789,7 @@ namespace hpx::parallel::util { // clang-format off HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; - (void) ++it, ++dest, i += 4) // -V112 + (void) ++it, ++dest, ++i) // -V112 { HPX_INVOKE(f, it, dest); } @@ -813,6 +813,79 @@ namespace hpx::parallel::util { } } + template )> + static FwdIter call( + ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + { + FwdIter base = it; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + for (std::size_t i = 0; i < count; + (void) ++it, i += 4) // -V112 + { + HPX_INVOKE(f, it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + } + for (/**/; count < num; (void) ++count, ++it) + { + HPX_INVOKE(f, it); + } + return it; + } + catch (...) + { + for (/**/; base != it; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + template )> + static FwdIter call( + ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + { + FwdIter base = it; + try + { + std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < count; + (void) ++it, ++i) // -V112 + { + HPX_INVOKE(f, it); + } + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it) + { + HPX_INVOKE(f, it); + } + // clang-format on + + return it; + } + catch (...) + { + for (/**/; base != it; ++base) + { + HPX_INVOKE(cleanup, base); + } + throw; + } + } + template HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token( @@ -823,7 +896,7 @@ namespace hpx::parallel::util { if (tok.was_cancelled()) return it; - return call(HPX_FORWARD(ExPolicy, policy), it, num, it, + return call(HPX_FORWARD(ExPolicy, policy), it, num, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } From 648d6134662e8d790611825c6c71ec67b0ff9c31 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 16:11:51 -0500 Subject: [PATCH 110/333] clang-format Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_default_construct.hpp | 3 ++- .../algorithms/include/hpx/parallel/util/loop.hpp | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index c3989037a1cd..15565220891b 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -272,7 +272,8 @@ namespace hpx::parallel { static FwdIter sequential(ExPolicy policy, FwdIter first, Sent last) { return sequential_uninitialized_default_construct_n( - HPX_FORWARD(ExPolicy, policy), first, std::distance(first, last)); + HPX_FORWARD(ExPolicy, policy), first, + std::distance(first, last)); } template diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 01753276ef26..26a500c183b7 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -813,12 +813,12 @@ namespace hpx::parallel::util { } } - template )> static FwdIter call( - ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { FwdIter base = it; try @@ -847,12 +847,12 @@ namespace hpx::parallel::util { throw; } } - template )> static FwdIter call( - ExPolicy, Iter it, std::size_t num, F&& f, Cleanup&& cleanup) + ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { FwdIter base = it; try @@ -873,7 +873,7 @@ namespace hpx::parallel::util { HPX_INVOKE(f, it); } // clang-format on - + return it; } catch (...) From 89dc7079213818e00c84ca931a65686b12e5c7d2 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 17:24:55 -0500 Subject: [PATCH 111/333] fixed concept Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 26 +++++++++++-------- .../include/hpx/parallel/util/loop.hpp | 12 ++++----- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 15565220891b..fe2f44b7f1ee 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -269,9 +269,10 @@ namespace hpx::parallel { } template - static FwdIter sequential(ExPolicy policy, FwdIter first, Sent last) + static FwdIter sequential( + ExPolicy&& policy, FwdIter first, Sent last) { - return sequential_uninitialized_default_construct_n( + sequential_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, std::distance(first, last)); } @@ -280,7 +281,7 @@ namespace hpx::parallel { static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_uninitialized_default_construct_n( + parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } @@ -306,9 +307,10 @@ namespace hpx::parallel { template static FwdIter sequential( - ExPolicy, FwdIter first, std::size_t count) + ExPolicy&& policy, FwdIter first, std::size_t count) { - return std_uninitialized_default_construct_n(first, count); + return sequential_uninitialized_default_construct_n( + HPX_FORWARD(ExPolicy, policy), first, count); } template @@ -331,13 +333,14 @@ namespace hpx { : hpx::detail::tag_parallel_algorithm { // clang-format off - template + hpx::traits::is_forward_iterator_v && + hpx::traits::is_iterator_v )> // clang-format on friend void tag_fallback_invoke( - hpx::uninitialized_default_construct_t, FwdIter first, FwdIter last) + hpx::uninitialized_default_construct_t, FwdIter first, Sent last) { static_assert(hpx::traits::is_forward_iterator_v, "Requires at least forward iterator."); @@ -347,15 +350,16 @@ namespace hpx { } // clang-format off - template && - hpx::traits::is_forward_iterator_v + hpx::traits::is_forward_iterator_v && + hpx::traits::is_iterator_v )> // clang-format on friend hpx::parallel::util::detail::algorithm_result_t tag_fallback_invoke(hpx::uninitialized_default_construct_t, - ExPolicy&& policy, FwdIter first, FwdIter last) + ExPolicy&& policy, FwdIter first, Sent last) { static_assert(hpx::traits::is_forward_iterator_v, "Requires at least forward iterator."); diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 26a500c183b7..9d97c70b5c27 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -74,8 +74,8 @@ namespace hpx::parallel::util { template )> + HPX_CONCEPT_REQUIRES_( + !hpx::is_unsequenced_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) { @@ -660,8 +660,8 @@ namespace hpx::parallel::util { template )> + HPX_CONCEPT_REQUIRES_( + !hpx::is_unsequenced_execution_policy_v)> static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -742,7 +742,7 @@ namespace hpx::parallel::util { template )> + !hpx::is_unsequenced_execution_policy_v)> static FwdIter call(ExPolicy, Iter it, std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -816,7 +816,7 @@ namespace hpx::parallel::util { template )> + !hpx::is_unsequenced_execution_policy_v)> static FwdIter call( ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { From 110f1d68a9f83e084240128742234fa0f94b95af Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Tue, 13 Jun 2023 16:21:26 -0500 Subject: [PATCH 112/333] fixed error from test which hass seninal not convertible to iterable type Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 66 +++++++++++++++++-- .../include/hpx/parallel/util/loop.hpp | 4 +- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index fe2f44b7f1ee..6529ba3efc08 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -166,6 +166,7 @@ namespace hpx { #include #include +#include #include #include #include @@ -194,6 +195,64 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// + template )> + InIter sequential_uninitialized_default_construct( + ExPolicy&& policy, InIter first, Sent last) + { + using value_type = typename std::iterator_traits::value_type; + + Iter s_first = first; + try + { + for (/* */; first != last; ++first) + { + ::new (std::addressof(*first)) value_type; + } + return first; + } + catch (...) + { + for (/* */; s_first != first; ++s_first) + { + std::destroy_at(std::addressof(*s_first)); + } + throw; + } + } + + template )> + InIter sequential_uninitialized_default_construct( + ExPolicy&& policy, InIter first, Sent last) + { + using value_type = typename std::iterator_traits::value_type; + + Iter s_first = first; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/* */; first != last; ++first) + { + ::new (std::addressof(*first)) value_type; + } + // clang-format on + + return first; + } + catch (...) + { + for (/* */; s_first != first; ++s_first) + { + std::destroy_at(std::addressof(*s_first)); + } + throw; + } + } + template InIter sequential_uninitialized_default_construct_n( ExPolicy&& policy, InIter first, std::size_t count) @@ -272,16 +331,15 @@ namespace hpx::parallel { static FwdIter sequential( ExPolicy&& policy, FwdIter first, Sent last) { - sequential_uninitialized_default_construct_n( - HPX_FORWARD(ExPolicy, policy), first, - std::distance(first, last)); + return sequential_uninitialized_default_construct( + HPX_FORWARD(ExPolicy, policy), first, last); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - parallel_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 9d97c70b5c27..979c334833c2 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -74,7 +74,7 @@ namespace hpx::parallel::util { template )> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) @@ -660,7 +660,7 @@ namespace hpx::parallel::util { template )> static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) From f0d580af3a28771854c690f5f05e1e627087edf8 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Tue, 13 Jun 2023 16:27:26 -0500 Subject: [PATCH 113/333] unused variable fixed Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_default_construct.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 6529ba3efc08..69d07effc387 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -195,11 +195,11 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// - template )> - InIter sequential_uninitialized_default_construct( - ExPolicy&& policy, InIter first, Sent last) + Iter sequential_uninitialized_default_construct( + ExPolicy&&, Iter first, Sent last) { using value_type = typename std::iterator_traits::value_type; @@ -222,11 +222,11 @@ namespace hpx::parallel { } } - template )> - InIter sequential_uninitialized_default_construct( - ExPolicy&& policy, InIter first, Sent last) + Iter sequential_uninitialized_default_construct( + ExPolicy&&, Iter first, Sent last) { using value_type = typename std::iterator_traits::value_type; From 82cc7c124ffb4734b18b5c316416c513988a0055 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 14 Jun 2023 03:56:22 -0500 Subject: [PATCH 114/333] uninit move Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_move.hpp | 102 +++++------------ .../include/hpx/parallel/util/transfer.hpp | 106 ++++++++++++++++++ 2 files changed, 132 insertions(+), 76 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp index 5fc3c6407123..19d49746e327 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp @@ -208,6 +208,7 @@ namespace hpx { #include #include #include +#include #include #include @@ -224,33 +225,6 @@ namespace hpx::parallel { // uninitialized_move namespace detail { /// \cond NOINTERNAL - - /////////////////////////////////////////////////////////////////////// - template - util::in_out_result sequential_uninitialized_move( - InIter1 first, FwdIter2 dest, Cond cond) - { - FwdIter2 current = dest; - try - { - for (/* */; HPX_INVOKE(cond, first, current); - (void) ++first, ++current) - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - return util::in_out_result{first, current}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - /////////////////////////////////////////////////////////////////////// template util::in_out_result sequential_uninitialized_move_n( @@ -273,7 +247,7 @@ namespace hpx::parallel { template typename util::detail::algorithm_result>::type - parallel_sequential_uninitialized_move_n( + parallel_uninitialized_move_n( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { if (count == 0) @@ -286,21 +260,21 @@ namespace hpx::parallel { using zip_iterator = hpx::util::zip_iterator; using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup, partition_result_type>:: call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [tok](zip_iterator t, std::size_t part_size) mutable + [&policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); FwdIter2 dst = get<1>(iters); return std::make_pair(dst, util::get_second_element( - sequential_uninitialized_move_n( - get<0>(iters), part_size, dst, tok))); + hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), + get<0>(iters), part_size, dst))); }, // finalize, called once if no error occurred [first, dest, count](auto&& data) mutable @@ -337,12 +311,11 @@ namespace hpx::parallel { template static util::in_out_result sequential( - ExPolicy, InIter1 first, Sent last, FwdIter2 dest) + ExPolicy&& policy, InIter1 first, Sent last, FwdIter2 dest) { - return sequential_uninitialized_move( - first, dest, [last](InIter1 first, FwdIter2) -> bool { - return first != last; - }); + return hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), first, + std::distance(first, last), dest); } template >::type parallel(ExPolicy&& policy, Iter first, Sent last, FwdIter2 dest) { - return parallel_sequential_uninitialized_move_n( + return parallel_uninitialized_move_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last), dest); } @@ -376,13 +349,16 @@ namespace hpx::parallel { template - static util::in_out_result sequential(ExPolicy, - InIter1 first, Sent1 last, FwdIter2 dest, Sent2 last_d) + static util::in_out_result sequential( + ExPolicy&& policy, InIter1 first, Sent1 last, FwdIter2 dest, + Sent2 last_d) { - return sequential_uninitialized_move(first, dest, - [last, last_d](InIter1 first, FwdIter2 current) -> bool { - return !(first == last || current == last_d); - }); + std::size_t const dist1 = detail::distance(first, last); + std::size_t const dist2 = detail::distance(dest, last_d); + std::size_t dist = dist1 <= dist2 ? dist1 : dist2; + + return hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), first, dist, dest); } template - util::in_out_result std_uninitialized_move_n( - InIter1 first, std::size_t count, InIter2 d_first) - { - InIter2 current = d_first; - try - { - for (/* */; count != 0; ++first, (void) ++current, --count) - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - return util::in_out_result{first, current}; - } - catch (...) - { - for (/* */; d_first != current; ++d_first) - { - std::destroy_at(std::addressof(*d_first)); - } - throw; - } - } - template struct uninitialized_move_n : public algorithm, IterPair> @@ -445,10 +394,11 @@ namespace hpx::parallel { } template - static IterPair sequential( - ExPolicy, InIter1 first, std::size_t count, InIter2 dest) + static IterPair sequential(ExPolicy&& policy, InIter1 first, + std::size_t count, InIter2 dest) { - return std_uninitialized_move_n(first, count, dest); + return hpx::parallel::util::uninit_move_n( + HPX_FORWARD(ExPolicy, policy), first, count, dest); } template @@ -457,7 +407,7 @@ namespace hpx::parallel { parallel( ExPolicy&& policy, Iter first, std::size_t count, FwdIter2 dest) { - return parallel_sequential_uninitialized_move_n( + return parallel_uninitialized_move_n( HPX_FORWARD(ExPolicy, policy), first, count, dest); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index c3ea0ea173bb..a6b0b978da83 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -44,6 +44,9 @@ namespace hpx::parallel::util { template struct uninit_copy_n_helper; + template + struct uninit_move_n_helper; + /////////////////////////////////////////////////////////////////////// template HPX_FORCEINLINE std::enable_if_t, char*> to_ptr( @@ -454,4 +457,107 @@ namespace hpx::parallel::util { return detail::uninit_copy_n_helper::call( HPX_FORWARD(ExPolicy, policy), first, count, dest); } + + // helpers for uninit_move_n + namespace detail { + // Customization point for optimizing move_n operations + template + struct uninit_move_n_helper + { + template )> + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + + try + { + for (std::size_t i = 0; i < num; + ++i, ++current, ++first) //-V112 + { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*first)); + } + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + + template )> + HPX_FORCEINLINE static in_out_result call( + ExPolicy, InIter first, std::size_t num, OutIter dest) + { + OutIter current = dest; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < num; + ++i, ++current, ++first) //-V112 + { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*first)); + } + // clang-format on + + return in_out_result{ + HPX_MOVE(first), HPX_MOVE(current)}; + } + catch (...) + { + for (/* */; dest != current; ++dest) + { + std::destroy_at(std::addressof(*dest)); + } + throw; + } + } + }; + + template + struct uninit_move_n_helper + { + template + HPX_FORCEINLINE static in_out_result call(ExPolicy, + InIter first, std::size_t count, OutIter dest) noexcept + { + return copy_memmove(first, count, dest); + } + }; + } // namespace detail + + template + HPX_FORCEINLINE constexpr in_out_result uninit_move_n( + InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_move_n_helper::call( + hpx::execution::seq, first, count, dest); + } + + template + HPX_FORCEINLINE constexpr in_out_result uninit_move_n( + ExPolicy&& policy, InIter first, std::size_t count, OutIter dest) + { + using category = + hpx::traits::pointer_copy_category_t, + std::decay_t>; + return detail::uninit_move_n_helper::call( + HPX_FORWARD(ExPolicy, policy), first, count, dest); + } } // namespace hpx::parallel::util From 7e6ff8848ea62212e9dcb3ec6b093a3c095935ce Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Thu, 15 Jun 2023 12:03:19 -0500 Subject: [PATCH 115/333] uninit value construct Signed-off-by: Hari Hara Naveen S --- .../uninitialized_value_construct.hpp | 90 +++++----------- .../include/hpx/parallel/util/loop.hpp | 101 ++++++++++++------ 2 files changed, 97 insertions(+), 94 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp index 59ad6801cdb8..d49f4579fcf1 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp @@ -193,36 +193,29 @@ namespace hpx::parallel { // provide our own implementation of std::uninitialized_value_construct // as some versions of MSVC horribly fail at compiling it for some types // T - template - InIter std_uninitialized_value_construct(InIter first, Sent last) + template + InIter sequential_uninitialized_value_construct( + ExPolicy&& policy, InIter first, Sent last) { - InIter s_first = first; - try - { - for (/* */; first != last; ++first) - { - hpx::construct_at(std::addressof(*first)); - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } + return util::loop_with_cleanup( + HPX_FORWARD(ExPolicy, policy), first, last, + [](InIter it) -> void { + hpx::construct_at(std::addressof(*it)); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + }); } /////////////////////////////////////////////////////////////////////// - template - InIter sequential_uninitialized_value_construct_n(InIter first, - std::size_t count, - util::cancellation_token& tok) + template + InIter sequential_uninitialized_value_construct_n( + ExPolicy&& policy, InIter first, std::size_t count) { + util::cancellation_token tok; + return util::loop_with_cleanup_n_with_token( - first, count, tok, + HPX_FORWARD(ExPolicy, policy), first, count, tok, [](InIter it) -> void { hpx::construct_at(std::addressof(*it)); }, @@ -234,7 +227,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::detail::algorithm_result_t - parallel_sequential_uninitialized_value_construct_n( + parallel_uninitialized_value_construct_n( ExPolicy&& policy, FwdIter first, std::size_t count) { if (count == 0) @@ -245,16 +238,15 @@ namespace hpx::parallel { using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [tok](FwdIter it, std::size_t part_size) mutable + [policy](FwdIter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_value_construct_n( - it, part_size, tok)); + HPX_FORWARD(ExPolicy, policy), it, part_size)); }, // finalize, called once if no error occurred [first, count](auto&& data) mutable -> FwdIter { @@ -288,16 +280,18 @@ namespace hpx::parallel { } template - static FwdIter sequential(ExPolicy, FwdIter first, Sent last) + static FwdIter sequential( + ExPolicy&& policy, FwdIter first, Sent last) { - return std_uninitialized_value_construct(first, last); + return sequential_uninitialized_value_construct( + HPX_FORWARD(ExPolicy, policy), first, last); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_sequential_uninitialized_value_construct_n( + return parallel_uninitialized_value_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } @@ -308,33 +302,6 @@ namespace hpx::parallel { // uninitialized_value_construct_n namespace detail { /// \cond NOINTERNAL - - // provide our own implementation of std::uninitialized_value_construct - // as some versions of MSVC horribly fail at compiling it for some types - // T - template - InIter std_uninitialized_value_construct_n( - InIter first, std::size_t count) - { - InIter s_first = first; - try - { - for (/* */; count != 0; (void) ++first, --count) - { - hpx::construct_at(std::addressof(*first)); - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - template struct uninitialized_value_construct_n : public algorithm, FwdIter> @@ -347,16 +314,17 @@ namespace hpx::parallel { template static FwdIter sequential( - ExPolicy, FwdIter first, std::size_t count) + ExPolicy&& policy, FwdIter first, std::size_t count) { - return std_uninitialized_value_construct_n(first, count); + return sequential_uninitialized_value_construct_n( + HPX_FORWARD(ExPolicy, policy), first, count); } template static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, std::size_t count) { - return parallel_sequential_uninitialized_value_construct_n( + return parallel_uninitialized_value_construct_n( HPX_FORWARD(ExPolicy, policy), first, count); } }; diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 979c334833c2..22497219e990 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -616,24 +616,65 @@ namespace hpx::parallel::util { // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. If an exception is thrown, // the given cleanup function will be called. - template struct loop_with_cleanup { /////////////////////////////////////////////////////////////////// template - static FwdIter call(ExPolicy&& policy, FwdIter first, FwdIter last, - F&& f, Cleanup&& cleanup) + typename Sent, typename Cleanup, + HPX_CONCEPT_REQUIRES_( + hpx::is_unsequenced_execution_policy_v)> + static FwdIter call( + ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { - return call(HPX_FORWARD(ExPolicy, policy), first, last, first, - HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + FwdIter it = first; + try + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != last; (void) ++it) + HPX_INVOKE(f, it); + // clang-format on + + return it; + } + catch (...) + { + for (/**/; first != it; ++first) + { + HPX_INVOKE(cleanup, first); + } + throw; + } + } + + template )> + static FwdIter call( + ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + { + FwdIter it = first; + try + { + for (/**/; it != last; (void) ++it) + HPX_INVOKE(f, it); + return it; + } + catch (...) + { + for (/**/; first != it; ++first) + { + HPX_INVOKE(cleanup, first); + } + throw; + } } template && - !hpx::is_parallel_execution_policy_v)> + HPX_CONCEPT_REQUIRES_( + hpx::is_unsequenced_execution_policy_v)> static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -684,32 +725,26 @@ namespace hpx::parallel::util { }; } // namespace detail - inline constexpr struct loop_with_cleanup_t final - : hpx::functional::detail::tag_fallback + template )> + HPX_FORCEINLINE constexpr Begin loop_with_cleanup(ExPolicy&& policy, + Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) { - private: - template - friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, - Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) - { - return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), first, last, dest, - HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - } + return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), + first, last, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } - template - friend HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin - tag_fallback_invoke(detail::loop_with_cleanup, ExPolicy&& policy, - Begin first, End last, F&& f, Cleanup&& cleanup) - { - return detail::loop_with_cleanup::call( - HPX_FORWARD(ExPolicy, policy), first, last, HPX_FORWARD(F, f), - HPX_FORWARD(Cleanup, cleanup)); - } - } loop_with_cleanup{}; + template )> + HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + ExPolicy&& policy, Begin first, End last, F&& f, Cleanup&& cleanup) + { + return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), + first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } // /////////////////////////////////////////////////////////////////////////// // template From 1be8420a6f297c9a0f0e486e5c2cffc6c70a6e39 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Thu, 15 Jun 2023 12:20:53 -0500 Subject: [PATCH 116/333] uninit fill Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_fill.hpp | 90 +++++++------------ 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp index 4f1b0b8dd033..2ffbb717cfaf 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp @@ -196,35 +196,28 @@ namespace hpx::parallel { // provide our own implementation of std::uninitialized_fill as some // versions of MSVC horribly fail at compiling it for some types T - template - InIter std_uninitialized_fill(InIter first, Sent last, T const& value) + template + InIter sequential_uninitialized_fill( + ExPolicy&& policy, InIter first, Sent last, T const& value) { - InIter current = first; - try - { - for (/* */; current != last; ++current) - { - hpx::construct_at(std::addressof(*current), value); - } - return current; - } - catch (...) - { - for (/* */; first != current; ++first) - { - std::destroy_at(std::addressof(*first)); - } - throw; - } + return util::loop_with_cleanup( + HPX_FORWARD(ExPolicy, policy), first, last, + [&value](InIter it) -> void { + hpx::construct_at(std::addressof(*it), value); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + }); } - template - InIter sequential_uninitialized_fill_n(InIter first, std::size_t count, - T const& value, - util::cancellation_token& tok) + template + InIter sequential_uninitialized_fill_n( + ExPolicy&& policy, InIter first, std::size_t count, T const& value) { + util::cancellation_token tok; + return util::loop_with_cleanup_n_with_token( - first, count, tok, + HPX_FORWARD(ExPolicy, policy), first, count, tok, [&value](InIter it) -> void { hpx::construct_at(std::addressof(*it), value); }, @@ -236,7 +229,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::detail::algorithm_result_t - parallel_sequential_uninitialized_fill_n( + parallel_uninitialized_fill_n( ExPolicy&& policy, Iter first, std::size_t count, T const& value) { if (count == 0) @@ -252,11 +245,12 @@ namespace hpx::parallel { partition_result_type>:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [value, tok](Iter it, std::size_t part_size) mutable + [value, tok, policy](Iter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_fill_n( - it, part_size, value, tok)); + HPX_FORWARD(ExPolicy, policy), it, part_size, + value)); }, // finalize, called once if no error occurred [first, count](auto&& data) mutable -> Iter { @@ -290,9 +284,10 @@ namespace hpx::parallel { template static Iter sequential( - ExPolicy, Iter first, Sent last, T const& value) + ExPolicy&& policy, Iter first, Sent last, T const& value) { - return std_uninitialized_fill(first, last, value); + return sequential_uninitialized_fill( + HPX_FORWARD(ExPolicy, policy), first, last, value); } template @@ -303,7 +298,7 @@ namespace hpx::parallel { return util::detail::algorithm_result::get( HPX_MOVE(first)); - return parallel_sequential_uninitialized_fill_n( + return parallel_uninitialized_fill_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last), value); } @@ -315,32 +310,6 @@ namespace hpx::parallel { // uninitialized_fill_n namespace detail { /// \cond NOINTERNAL - - // provide our own implementation of std::uninitialized_fill_n as some - // versions of MSVC horribly fail at compiling it for some types T - template - InIter std_uninitialized_fill_n( - InIter first, Size count, T const& value) - { - InIter current = first; - try - { - for (/* */; count > 0; ++current, (void) --count) - { - hpx::construct_at(std::addressof(*current), value); - } - return current; - } - catch (...) - { - for (/* */; first != current; ++first) - { - std::destroy_at(std::addressof(*first)); - } - throw; - } - } - template struct uninitialized_fill_n : public algorithm, Iter> @@ -351,10 +320,11 @@ namespace hpx::parallel { } template - static Iter sequential( - ExPolicy, Iter first, std::size_t count, T const& value) + static Iter sequential(ExPolicy&& policy, Iter first, + std::size_t count, T const& value) { - return std_uninitialized_fill_n(first, count, value); + return sequential_uninitialized_fill_n( + HPX_FORWARD(ExPolicy, policy), first, count, value); } template @@ -362,7 +332,7 @@ namespace hpx::parallel { ExPolicy&& policy, Iter first, std::size_t count, T const& value) { - return parallel_sequential_uninitialized_fill_n( + return parallel_uninitialized_fill_n( HPX_FORWARD(ExPolicy, policy), first, count, value); } }; From d425df9cd6f0fdae078c6dbd6eeead900748af73 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Thu, 15 Jun 2023 17:43:08 -0500 Subject: [PATCH 117/333] removed commented out code Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 22497219e990..2a2a7ee63b6d 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -746,26 +746,6 @@ namespace hpx::parallel::util { first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - // /////////////////////////////////////////////////////////////////////////// - // template - // HPX_FORCEINLINE constexpr Iter loop_with_cleanup( - // Iter it, Iter last, F&& f, Cleanup&& cleanup) - // { - // using cat = typename std::iterator_traits::iterator_category; - // return detail::loop_with_cleanup::call( - // it, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - // } - - // template - // HPX_FORCEINLINE constexpr FwdIter loop_with_cleanup( - // Iter it, Iter last, FwdIter dest, F&& f, Cleanup&& cleanup) - // { - // using cat = typename std::iterator_traits::iterator_category; - // return detail::loop_with_cleanup::call( - // it, last, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); - // } - - /////////////////////////////////////////////////////////////////////////// namespace detail { // Helper class to repeatedly call a function a given number of times From 269aff62fc5a3f4fb6db31726fbb0424142397f5 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 10:59:38 -0500 Subject: [PATCH 118/333] reeplaced try catch loops with loop_with_cleanup Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 63 ++------ .../include/hpx/parallel/unseq/loop.hpp | 14 +- .../include/hpx/parallel/util/transfer.hpp | 136 ++++-------------- 3 files changed, 45 insertions(+), 168 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 69d07effc387..757b5ccda4ec 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -195,62 +195,21 @@ namespace hpx::parallel { /// \cond NOINTERNAL /////////////////////////////////////////////////////////////////////// - template )> - Iter sequential_uninitialized_default_construct( - ExPolicy&&, Iter first, Sent last) - { - using value_type = typename std::iterator_traits::value_type; - - Iter s_first = first; - try - { - for (/* */; first != last; ++first) - { - ::new (std::addressof(*first)) value_type; - } - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } - } - - template )> + template Iter sequential_uninitialized_default_construct( ExPolicy&&, Iter first, Sent last) { - using value_type = typename std::iterator_traits::value_type; - - Iter s_first = first; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/* */; first != last; ++first) - { - ::new (std::addressof(*first)) value_type; - } - // clang-format on + using value_type = + typename std::iterator_traits::value_type; - return first; - } - catch (...) - { - for (/* */; s_first != first; ++s_first) - { - std::destroy_at(std::addressof(*s_first)); - } - throw; - } + return util::loop_with_cleanup( + HPX_FORWARD(ExPolicy, policy), first, last, + [](InIter it) -> void { + ::new (std::addressof(*it)) value_type; + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + }); } template diff --git a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp index 131f62492014..3b7e5d3c7526 100644 --- a/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/unseq/loop.hpp @@ -254,11 +254,10 @@ namespace hpx::parallel::util { { std::size_t count(num & std::size_t(-4)); // -V112 - //clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; - i < count; - (void) ++it, ++dest, - i += 4) // -V112 + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (std::size_t i = 0; i < count; (void) ++it, ++dest, + i += 4) // -V112 { HPX_INVOKE(f, it, dest); HPX_INVOKE(f, ++it, ++dest); @@ -266,9 +265,8 @@ namespace hpx::parallel::util { HPX_INVOKE(f, ++it, ++dest); } - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; count < num; - (void) ++count, - ++it, ++dest) + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it, ++dest) { HPX_INVOKE(f, it, dest); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index a6b0b978da83..97c217295d08 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -364,62 +364,23 @@ namespace hpx::parallel::util { template struct uninit_copy_n_helper { - template )> - HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) - { - OutIter current = dest; - - try - { - for (std::size_t i = 0; i < num; - ++i, ++current, ++first) //-V112 - { - hpx::construct_at(std::addressof(*current), *first); - } - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template )> + template HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) + ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (std::size_t i = 0; i < num; ++i, ++current, ++first) //-V112 - { - hpx::construct_at(std::addressof(*current), *first); - } - // clang-format on - - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } + using value_type = + typename std::iterator_traits::value_type; + + return in_out_result{std::next(first, num), + util::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it) -> void { + hpx::construct_at(std::addressof(*current), *first); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; @@ -464,65 +425,24 @@ namespace hpx::parallel::util { template struct uninit_move_n_helper { - template )> - HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) - { - OutIter current = dest; - - try - { - for (std::size_t i = 0; i < num; - ++i, ++current, ++first) //-V112 - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } - } - - template )> + template HPX_FORCEINLINE static in_out_result call( ExPolicy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; - try - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (std::size_t i = 0; i < num; - ++i, ++current, ++first) //-V112 - { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); - } - // clang-format on - - return in_out_result{ - HPX_MOVE(first), HPX_MOVE(current)}; - } - catch (...) - { - for (/* */; dest != current; ++dest) - { - std::destroy_at(std::addressof(*dest)); - } - throw; - } + using value_type = + typename std::iterator_traits::value_type; + + return in_out_result{std::next(first, num), + util::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it) -> void { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*first)); + }, + [](InIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; From 866bd46fc9c67f3a40f92b10abc834119a99f294 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 11:16:22 -0500 Subject: [PATCH 119/333] changed variable name Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 97c217295d08..af82bf1b30e6 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -373,7 +373,7 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - util::loop_with_cleanup_n::call( + loop_with_cleanup_n::call( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at(std::addressof(*current), *first); @@ -427,7 +427,7 @@ namespace hpx::parallel::util { { template HPX_FORCEINLINE static in_out_result call( - ExPolicy, InIter first, std::size_t num, OutIter dest) + ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { OutIter current = dest; using value_type = @@ -438,7 +438,7 @@ namespace hpx::parallel::util { HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at( - std::addressof(*current), HPX_MOVE(*first)); + std::addressof(*current), HPX_MOVE(*it)); }, [](InIter it) -> void { std::destroy_at(std::addressof(*it)); From 894b93ac2055182e225879c2eb9652d7a60a2193 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 12:07:58 -0500 Subject: [PATCH 120/333] namespace errors Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_default_construct.hpp | 11 +++-------- .../algorithms/include/hpx/parallel/util/transfer.hpp | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 757b5ccda4ec..f31df1a42e81 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -199,17 +199,12 @@ namespace hpx::parallel { Iter sequential_uninitialized_default_construct( ExPolicy&&, Iter first, Sent last) { - using value_type = - typename std::iterator_traits::value_type; + using value_type = typename std::iterator_traits::value_type; return util::loop_with_cleanup( HPX_FORWARD(ExPolicy, policy), first, last, - [](InIter it) -> void { - ::new (std::addressof(*it)) value_type; - }, - [](InIter it) -> void { - std::destroy_at(std::addressof(*it)); - }); + [](Iter it) -> void { ::new (std::addressof(*it)) value_type; }, + [](Iter it) -> void { std::destroy_at(std::addressof(*it)); }); } template diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index af82bf1b30e6..eef8bf198b6d 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -373,7 +373,7 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - loop_with_cleanup_n::call( + hpx::parallel::util::loop_with_cleanup_n::call( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at(std::addressof(*current), *first); @@ -434,7 +434,7 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - util::loop_with_cleanup_n::call( + hpx::parallel::util::loop_with_cleanup_n::call( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it) -> void { hpx::construct_at( From 14ece10eab8626a2d2da524133af21f556e13ec1 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 02:01:10 -0500 Subject: [PATCH 121/333] fixed template errors Signed-off-by: Hari Hara Naveen S --- .../uninitialized_default_construct.hpp | 2 +- .../include/hpx/parallel/util/loop.hpp | 1 - .../include/hpx/parallel/util/transfer.hpp | 38 ++++++++++--------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index f31df1a42e81..1a852004bd03 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -197,7 +197,7 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template Iter sequential_uninitialized_default_construct( - ExPolicy&&, Iter first, Sent last) + ExPolicy&& policy, Iter first, Sent last) { using value_type = typename std::iterator_traits::value_type; diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 2a2a7ee63b6d..04430e07f587 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -747,7 +747,6 @@ namespace hpx::parallel::util { } namespace detail { - // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. template diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index eef8bf198b6d..9c02be34ae36 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -373,14 +374,16 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - hpx::parallel::util::loop_with_cleanup_n::call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it) -> void { - hpx::construct_at(std::addressof(*current), *first); - }, - [](InIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n:: + call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at( + std::addressof(*current), *it); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; @@ -434,15 +437,16 @@ namespace hpx::parallel::util { typename std::iterator_traits::value_type; return in_out_result{std::next(first, num), - hpx::parallel::util::loop_with_cleanup_n::call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it) -> void { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*it)); - }, - [](InIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n:: + call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*it)); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; From 25f6e3dc43fde60fd61522cf9926ac50837383f3 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 02:13:57 -0500 Subject: [PATCH 122/333] cf Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/transfer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 9c02be34ae36..200ef5b2e3f2 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -362,7 +362,7 @@ namespace hpx::parallel::util { // helpers for uninit_copy_n namespace detail { // Customization point for optimizing copy_n operations - template + template struct uninit_copy_n_helper { template From 9999e5a40b42282b57e16dfc825916d15f4e1b68 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 07:04:16 -0500 Subject: [PATCH 123/333] removed unused vars Signed-off-by: Hari Hara Naveen S --- .../algorithms/include/hpx/parallel/util/transfer.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index 200ef5b2e3f2..de65cef2247a 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -369,10 +369,6 @@ namespace hpx::parallel::util { HPX_FORCEINLINE static in_out_result call( ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { - OutIter current = dest; - using value_type = - typename std::iterator_traits::value_type; - return in_out_result{std::next(first, num), ::hpx::parallel::util::detail::loop_with_cleanup_n:: call( @@ -432,10 +428,6 @@ namespace hpx::parallel::util { HPX_FORCEINLINE static in_out_result call( ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { - OutIter current = dest; - using value_type = - typename std::iterator_traits::value_type; - return in_out_result{std::next(first, num), ::hpx::parallel::util::detail::loop_with_cleanup_n:: call( From cc10353f07ea2f08426dd17096cd2c8cb7c2a7a6 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 24 Jul 2023 04:56:13 -0500 Subject: [PATCH 124/333] requested changes Signed-off-by: Hari Hara Naveen S --- .../parallel/algorithms/uninitialized_default_construct.hpp | 3 +-- .../include/hpx/parallel/algorithms/uninitialized_move.hpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 1a852004bd03..ceab0f25b5b2 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -239,13 +239,12 @@ namespace hpx::parallel { } using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [tok, policy](FwdIter it, std::size_t part_size) mutable + [policy](FwdIter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_default_construct_n( diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp index 19d49746e327..1422be66b4b8 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp @@ -265,7 +265,7 @@ namespace hpx::parallel { call( HPX_FORWARD(ExPolicy, policy), zip_iterator(first, dest), count, - [&policy](zip_iterator t, std::size_t part_size) mutable + [policy](zip_iterator t, std::size_t part_size) mutable -> partition_result_type { using hpx::get; auto iters = t.get_iterator_tuple(); From 315887b881d3b34164f0f5ca5769c5116f53978f Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 28 Jul 2023 15:28:03 -0500 Subject: [PATCH 125/333] unseq dispatch psuhed to tag_invoke stage Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 157 ++++++++++++------ .../include/hpx/parallel/util/transfer.hpp | 37 ++--- 2 files changed, 125 insertions(+), 69 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 04430e07f587..8467855f761f 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -747,16 +747,15 @@ namespace hpx::parallel::util { } namespace detail { + // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. - template - struct loop_with_cleanup_n + struct loop_with_cleanup_n final + : hpx::functional::detail::tag_fallback { /////////////////////////////////////////////////////////////////// template )> + typename F, typename Cleanup> static FwdIter call(ExPolicy, Iter it, std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) { @@ -788,38 +787,32 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call(ExPolicy, Iter it, std::size_t num, - FwdIter dest, F&& f, Cleanup&& cleanup) + template + static FwdIter call( + ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { - FwdIter base = dest; + FwdIter base = it; try { std::size_t count(num & std::size_t(-4)); // -V112 - - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; - (void) ++it, ++dest, ++i) // -V112 + (void) ++it, i += 4) // -V112 { - HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); + HPX_INVOKE(f, ++it); } - - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/**/; count < num; (void) ++count, ++it, ++dest) + for (/**/; count < num; (void) ++count, ++it) { - HPX_INVOKE(f, it, dest); + HPX_INVOKE(f, it); } - // clang-format on - - return dest; + return it; } catch (...) { - for (/**/; base != dest; ++base) + for (/**/; base != it; ++base) { HPX_INVOKE(cleanup, base); } @@ -827,45 +820,46 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call( - ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + template + FwdIter unseq_call(Iter it, std::size_t num, FwdIter dest, F&& f, + Cleanup&& cleanup) { - FwdIter base = it; + FwdIter base = dest; try { std::size_t count(num & std::size_t(-4)); // -V112 + + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (std::size_t i = 0; i < count; - (void) ++it, i += 4) // -V112 + (void) ++it, ++dest, ++i) // -V112 { - HPX_INVOKE(f, it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); - HPX_INVOKE(f, ++it); + HPX_INVOKE(f, it, dest); } - for (/**/; count < num; (void) ++count, ++it) + + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; count < num; (void) ++count, ++it, ++dest) { - HPX_INVOKE(f, it); + HPX_INVOKE(f, it, dest); } - return it; + // clang-format on + + return dest; } catch (...) { - for (/**/; base != it; ++base) + for (/**/; base != dest; ++base) { HPX_INVOKE(cleanup, base); } throw; } } + template )> - static FwdIter call( + typename Cleanup> + static FwdIter unseq_call( ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) { FwdIter base = it; @@ -930,13 +924,78 @@ namespace hpx::parallel::util { }; } // namespace detail +#if !defined(HPX_COMPUTE_DEVICE_CODE) + inline constexpr detail::loop_with_cleanup_n loop_with_cleanup_n = + detail::loop_with_cleanup_n{}; +#else + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n( + ExPolicy&& policy, Begin begin, std::size_t n, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n{}( + HPX_FORWARD(ExPolicy, policy), begin, n, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n( + ExPolicy&& policy, Begin begin, std::size_t n, Begin2 dest, F&& f, + Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n{}( + HPX_FORWARD(ExPolicy, policy), begin, n, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } +#endif + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, + std::size_t n, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + std::size_t n, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, + std::size_t n, Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup_n, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + std::size_t n, Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup_n::unseq_call( + begin, n, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + template )> HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } @@ -950,7 +1009,7 @@ namespace hpx::parallel::util { Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } @@ -964,7 +1023,7 @@ namespace hpx::parallel::util { Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } @@ -979,7 +1038,7 @@ namespace hpx::parallel::util { CancelToken& tok, F&& f, Cleanup&& cleanup) { using cat = typename std::iterator_traits::iterator_category; - return detail::loop_with_cleanup_n::call_with_token( + return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index de65cef2247a..a8e74c536527 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -370,16 +370,14 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n:: - call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it, OutIter current) -> void { - hpx::construct_at( - std::addressof(*current), *it); - }, - [](OutIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at(std::addressof(*current), *it); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; @@ -429,16 +427,15 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n:: - call( - HPX_FORWARD(ExPolicy, policy), first, num, dest, - [](InIter it, OutIter current) -> void { - hpx::construct_at( - std::addressof(*current), HPX_MOVE(*it)); - }, - [](OutIter it) -> void { - std::destroy_at(std::addressof(*it)); - })}; + ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + HPX_FORWARD(ExPolicy, policy), first, num, dest, + [](InIter it, OutIter current) -> void { + hpx::construct_at( + std::addressof(*current), HPX_MOVE(*it)); + }, + [](OutIter it) -> void { + std::destroy_at(std::addressof(*it)); + })}; } }; From 109c95b42ca1e9c35eae20089cd4f4bde6dc8933 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 2 Aug 2023 16:13:56 -0500 Subject: [PATCH 126/333] loop with cleanup moved to tag fallback invoke Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 126 ++++++++++++------ 1 file changed, 82 insertions(+), 44 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 8467855f761f..23fbbbef4f3c 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -616,14 +616,13 @@ namespace hpx::parallel::util { // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. If an exception is thrown, // the given cleanup function will be called. - struct loop_with_cleanup + struct loop_with_cleanup final + : hpx::functional::detail::tag_fallback { /////////////////////////////////////////////////////////////////// template )> - static FwdIter call( + typename Sent, typename Cleanup> + static FwdIter unseq_call( ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { FwdIter it = first; @@ -648,11 +647,10 @@ namespace hpx::parallel::util { } template )> - static FwdIter call( - ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + typename Sent, typename Cleanup> + friend FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, + FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { FwdIter it = first; try @@ -671,22 +669,17 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, - F&& f, Cleanup&& cleanup) + template + friend FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, + Iter it, Sent last, FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; try { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != last; (void) ++it, ++dest) f(it, dest); - // clang-format on - return dest; } catch (...) @@ -699,18 +692,20 @@ namespace hpx::parallel::util { } } - template )> - static FwdIter call(ExPolicy&&, Iter it, Iter last, FwdIter dest, - F&& f, Cleanup&& cleanup) + template + static FwdIter unseq_call(ExPolicy&&, Iter it, Sent last, + FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; try { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != last; (void) ++it, ++dest) f(it, dest); + // clang-format on + return dest; } catch (...) @@ -725,25 +720,72 @@ namespace hpx::parallel::util { }; } // namespace detail - template )> - HPX_FORCEINLINE constexpr Begin loop_with_cleanup(ExPolicy&& policy, - Begin first, End last, Begin dest, F&& f, Cleanup&& cleanup) +#if !defined(HPX_COMPUTE_DEVICE_CODE) + inline constexpr detail::loop_with_cleanup loop_with_cleanup = + detail::loop_with_cleanup{}; +#else + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + ExPolicy&& policy, Begin begin, Sent end, F&& f, Cleanup&& cleanup) { - return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), - first, last, dest, HPX_FORWARD(F, f), + return hpx::parallel::util::detail::loop_with_cleanup{}( + HPX_FORWARD(ExPolicy, policy), begin, end, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } - template )> - HPX_FORCEINLINE constexpr Begin loop_with_cleanup( - ExPolicy&& policy, Begin first, End last, F&& f, Cleanup&& cleanup) + template + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + ExPolicy&& policy, Begin begin, Sent end, Begin2 dest, F&& f, + Cleanup&& cleanup) { - return detail::loop_with_cleanup::call(HPX_FORWARD(ExPolicy, policy), - first, last, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + return hpx::parallel::util::detail::loop_with_cleanup{}( + HPX_FORWARD(ExPolicy, policy), begin, end, dest, HPX_FORWARD(F, f), + HPX_FORWARD(Cleanup, cleanup)); + } +#endif + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, Sent end, + F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + Sent end, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin, Sent end, + Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin, + Sent end, Begin2 dest, F&& f, Cleanup&& cleanup) + { + return hpx::parallel::util::detail::loop_with_cleanup::unseq_call( + begin, end, dest, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); } namespace detail { @@ -994,7 +1036,6 @@ namespace hpx::parallel::util { HPX_FORCEINLINE constexpr Iter loop_with_cleanup_n_with_token( Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); @@ -1008,7 +1049,6 @@ namespace hpx::parallel::util { std::size_t count, FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( ::hpx::execution::seq, it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); @@ -1022,7 +1062,6 @@ namespace hpx::parallel::util { ExPolicy&& policy, Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); @@ -1037,7 +1076,6 @@ namespace hpx::parallel::util { ExPolicy&& policy, Iter it, std::size_t count, FwdIter dest, CancelToken& tok, F&& f, Cleanup&& cleanup) { - using cat = typename std::iterator_traits::iterator_category; return detail::loop_with_cleanup_n::call_with_token( HPX_FORWARD(ExPolicy, policy), it, count, dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); From 6d3dbfae839b2f88139e1f921ac4ada99f4b6267 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 5 Aug 2023 05:35:21 -0500 Subject: [PATCH 127/333] removed tokens Signed-off-by: Hari Hara Naveen S --- .../algorithms/uninitialized_copy.hpp | 1 - .../uninitialized_default_construct.hpp | 7 +- .../algorithms/uninitialized_fill.hpp | 10 +- .../algorithms/uninitialized_move.hpp | 8 +- .../uninitialized_value_construct.hpp | 7 +- .../include/hpx/parallel/util/loop.hpp | 101 ++++++++++++------ .../include/hpx/parallel/util/transfer.hpp | 4 +- 7 files changed, 82 insertions(+), 56 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 7fd8495aa91f..8fb58d086586 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -199,7 +199,6 @@ namespace hpx { #include #include #include -#include #include #include #include diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index ceab0f25b5b2..47d3967b9dde 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -171,7 +171,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -214,10 +213,8 @@ namespace hpx::parallel { using value_type = typename std::iterator_traits::value_type; - util::cancellation_token tok; - - return util::loop_with_cleanup_n_with_token( - HPX_FORWARD(ExPolicy, policy), first, count, tok, + return util::loop_with_cleanup_n( + HPX_FORWARD(ExPolicy, policy), first, count, [](InIter it) -> void { ::new (std::addressof(*it)) value_type; }, diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp index 2ffbb717cfaf..0587f4962566 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_fill.hpp @@ -170,7 +170,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -214,10 +213,8 @@ namespace hpx::parallel { InIter sequential_uninitialized_fill_n( ExPolicy&& policy, InIter first, std::size_t count, T const& value) { - util::cancellation_token tok; - - return util::loop_with_cleanup_n_with_token( - HPX_FORWARD(ExPolicy, policy), first, count, tok, + return util::loop_with_cleanup_n( + HPX_FORWARD(ExPolicy, policy), first, count, [&value](InIter it) -> void { hpx::construct_at(std::addressof(*it), value); }, @@ -240,12 +237,11 @@ namespace hpx::parallel { using partition_result_type = std::pair; - util::cancellation_token tok; return util::partitioner_with_cleanup:: call( HPX_FORWARD(ExPolicy, policy), first, count, - [value, tok, policy](Iter it, std::size_t part_size) mutable + [value, policy](Iter it, std::size_t part_size) mutable -> partition_result_type { return std::make_pair(it, sequential_uninitialized_fill_n( diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp index 1422be66b4b8..2749c5773852 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_move.hpp @@ -201,7 +201,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -228,13 +227,12 @@ namespace hpx::parallel { /////////////////////////////////////////////////////////////////////// template util::in_out_result sequential_uninitialized_move_n( - InIter1 first, std::size_t count, InIter2 dest, - util::cancellation_token& tok) + InIter1 first, std::size_t count, InIter2 dest) { return util::in_out_result{ std::next(first, count), - util::loop_with_cleanup_n_with_token( - first, count, dest, tok, + util::loop_with_cleanup_n( + first, count, dest, [](InIter1 it, InIter2 dest) -> void { hpx::construct_at(std::addressof(*dest), HPX_MOVE(*it)); }, diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp index d49f4579fcf1..790709d40baa 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_value_construct.hpp @@ -167,7 +167,6 @@ namespace hpx { #include #include #include -#include #include #include #include @@ -212,10 +211,8 @@ namespace hpx::parallel { InIter sequential_uninitialized_value_construct_n( ExPolicy&& policy, InIter first, std::size_t count) { - util::cancellation_token tok; - - return util::loop_with_cleanup_n_with_token( - HPX_FORWARD(ExPolicy, policy), first, count, tok, + return util::loop_with_cleanup_n( + HPX_FORWARD(ExPolicy, policy), first, count, [](InIter it) -> void { hpx::construct_at(std::addressof(*it)); }, diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 23fbbbef4f3c..dfa67230e5d6 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -622,18 +622,14 @@ namespace hpx::parallel::util { /////////////////////////////////////////////////////////////////// template - static FwdIter unseq_call( + static FwdIter call( ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) { FwdIter it = first; try { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE for (/**/; it != last; (void) ++it) HPX_INVOKE(f, it); - // clang-format on - return it; } catch (...) @@ -645,48 +641,71 @@ namespace hpx::parallel::util { throw; } } - - template - friend FwdIter tag_fallback_invoke( - hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, - FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + template + static FwdIter call(ExPolicy&&, Iter it, Sent last, FwdIter dest, + F&& f, Cleanup&& cleanup) { - FwdIter it = first; + FwdIter base = dest; try { - for (/**/; it != last; (void) ++it) - HPX_INVOKE(f, it); - return it; + for (/**/; it != last; (void) ++it, ++dest) + f(it, dest); + return dest; } catch (...) { - for (/**/; first != it; ++first) + for (/**/; base != dest; ++base) { - HPX_INVOKE(cleanup, first); + HPX_INVOKE(cleanup, base); } throw; } } + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + ExPolicy&& policy, FwdIter first, Sent last, F&& f, + Cleanup&& cleanup) + { + return call(HPX_FORWARD(ExPolicy, policy), first, last, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + template - friend FwdIter tag_fallback_invoke( - hpx::parallel::util::detail::loop_with_cleanup, ExPolicy&&, - Iter it, Sent last, FwdIter dest, F&& f, Cleanup&& cleanup) + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + hpx::parallel::util::detail::loop_with_cleanup, + ExPolicy&& policy, Iter it, Sent last, FwdIter dest, F&& f, + Cleanup&& cleanup) { - FwdIter base = dest; + return call(HPX_FORWARD(ExPolicy, policy), it, last, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + static FwdIter unseq_call( + ExPolicy&&, FwdIter first, Sent last, F&& f, Cleanup&& cleanup) + { + FwdIter it = first; try { - for (/**/; it != last; (void) ++it, ++dest) - f(it, dest); - return dest; + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != last; (void) ++it) + HPX_INVOKE(f, it); + // clang-format on + + return it; } catch (...) { - for (/**/; base != dest; ++base) + for (/**/; first != it; ++first) { - HPX_INVOKE(cleanup, base); + HPX_INVOKE(cleanup, first); } throw; } @@ -798,7 +817,7 @@ namespace hpx::parallel::util { /////////////////////////////////////////////////////////////////// template - static FwdIter call(ExPolicy, Iter it, std::size_t num, + static FwdIter call(ExPolicy&&, Iter it, std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) { FwdIter base = dest; @@ -831,8 +850,8 @@ namespace hpx::parallel::util { template - static FwdIter call( - ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup) + static FwdIter call(ExPolicy&&, FwdIter it, std::size_t num, F&& f, + Cleanup&& cleanup) { FwdIter base = it; try @@ -862,6 +881,26 @@ namespace hpx::parallel::util { } } + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + loop_with_cleanup_n, ExPolicy&& policy, Iter it, + std::size_t num, FwdIter dest, F&& f, Cleanup&& cleanup) + { + return call(HPX_FORWARD(ExPolicy, policy), it, num, dest, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + + template + friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke( + loop_with_cleanup_n, ExPolicy&& policy, FwdIter it, + std::size_t num, F&& f, Cleanup&& cleanup) + { + return call(HPX_FORWARD(ExPolicy, policy), it, num, + HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup)); + } + template FwdIter unseq_call(Iter it, std::size_t num, FwdIter dest, F&& f, @@ -1092,7 +1131,7 @@ namespace hpx::parallel::util { /////////////////////////////////////////////////////////////////// // handle sequences of non-futures template - HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Iter call( + HPX_HOST_DEVICE HPX_FORCEINLINE static Iter call( std::size_t base_idx, Iter it, std::size_t num, F&& f) { std::size_t count(num & std::size_t(-4)); // -V112 @@ -1113,7 +1152,7 @@ namespace hpx::parallel::util { } template - HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Iter call( + HPX_HOST_DEVICE HPX_FORCEINLINE static Iter call( std::size_t base_idx, Iter it, std::size_t count, CancelToken& tok, F&& f) { diff --git a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp index a8e74c536527..69f2556ebb26 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/transfer.hpp @@ -370,7 +370,7 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + ::hpx::parallel::util::loop_with_cleanup_n( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it, OutIter current) -> void { hpx::construct_at(std::addressof(*current), *it); @@ -427,7 +427,7 @@ namespace hpx::parallel::util { ExPolicy&& policy, InIter first, std::size_t num, OutIter dest) { return in_out_result{std::next(first, num), - ::hpx::parallel::util::detail::loop_with_cleanup_n::call( + ::hpx::parallel::util::loop_with_cleanup_n( HPX_FORWARD(ExPolicy, policy), first, num, dest, [](InIter it, OutIter current) -> void { hpx::construct_at( From 6964b8502fa66d10ca7eefd7cca98969cb7b2d1a Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 7 May 2023 11:32:35 -0500 Subject: [PATCH 128/333] Adding release notes page for next release --- docs/sphinx/releases.rst | 3 ++- docs/sphinx/releases/whats_new_1_10_0.rst | 25 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 docs/sphinx/releases/whats_new_1_10_0.rst diff --git a/docs/sphinx/releases.rst b/docs/sphinx/releases.rst index 1de9f8930e91..d1d6a3558fe6 100644 --- a/docs/sphinx/releases.rst +++ b/docs/sphinx/releases.rst @@ -1,5 +1,5 @@ .. - Copyright (C) 2018-2022 STE||AR Group + Copyright (C) 2018-2023 STE||AR Group SPDX-License-Identifier: BSL-1.0 Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -17,6 +17,7 @@ List of releases .. toctree:: :maxdepth: 1 + releases/whats_new_1_10_0 releases/whats_new_1_9_0 releases/whats_new_1_8_1 releases/whats_new_1_8_0 diff --git a/docs/sphinx/releases/whats_new_1_10_0.rst b/docs/sphinx/releases/whats_new_1_10_0.rst new file mode 100644 index 000000000000..747a33e79230 --- /dev/null +++ b/docs/sphinx/releases/whats_new_1_10_0.rst @@ -0,0 +1,25 @@ +.. + Copyright (C) 2007-2023 Hartmut Kaiser + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _hpx_1_10_0: + +=========================== +|hpx| V1.10.0 (TBD) +=========================== + +General changes +=============== + +Breaking changes +================ + +Closed issues +============= + +Closed pull requests +==================== + From 3d6914192da2bd194cb2d0f2825d18672f3e87c7 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 30 Apr 2023 16:21:31 -0500 Subject: [PATCH 129/333] Adding zero-copy support on the receiving end of the TCP parcel port - flyby: cleaning up and modernizing TCP parcel port --- libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp | 1 + libs/full/parcelset/src/parcel.cpp | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp b/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp index 502bc8513025..7078f6d93662 100644 --- a/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp +++ b/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp @@ -388,6 +388,7 @@ namespace hpx::parcelset::policies::tcp { --operation_in_flight_; buffer_ = parcel_buffer_type(); parcels_.clear(); + chunk_buffers_.clear(); } else diff --git a/libs/full/parcelset/src/parcel.cpp b/libs/full/parcelset/src/parcel.cpp index 5dff726b9cca..8d385d23e0f5 100644 --- a/libs/full/parcelset/src/parcel.cpp +++ b/libs/full/parcelset/src/parcel.cpp @@ -480,7 +480,6 @@ namespace hpx::parcelset::detail { reinterpret_cast( action_->get_parent_thread_id().get())); #endif - return false; } From 656f8a4905346295276aa6f0f60f4ad37f5a0996 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 2 May 2023 15:29:56 -0500 Subject: [PATCH 130/333] Adding hpx.parcels.zero_copy_receive_optimization configuration key - this can be used to disable the zero-copy serialization on the receiving end, this option is enabled by default - flyby: modernize parcelset code --- libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp b/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp index 7078f6d93662..502bc8513025 100644 --- a/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp +++ b/libs/full/parcelport_tcp/include/hpx/parcelport_tcp/receiver.hpp @@ -388,7 +388,6 @@ namespace hpx::parcelset::policies::tcp { --operation_in_flight_; buffer_ = parcel_buffer_type(); parcels_.clear(); - chunk_buffers_.clear(); } else From 38529746d8205ffcb4aeacde0c48a8b677b51176 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 5 May 2023 07:43:03 -0500 Subject: [PATCH 131/333] Merge changes from #5910 --- libs/full/parcelset/src/parcel.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/full/parcelset/src/parcel.cpp b/libs/full/parcelset/src/parcel.cpp index 8d385d23e0f5..5dff726b9cca 100644 --- a/libs/full/parcelset/src/parcel.cpp +++ b/libs/full/parcelset/src/parcel.cpp @@ -480,6 +480,7 @@ namespace hpx::parcelset::detail { reinterpret_cast( action_->get_parent_thread_id().get())); #endif + return false; } From e1a296c54d57ad47024f4ae7041538ad3b25e90d Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 8 May 2023 09:06:55 -0500 Subject: [PATCH 132/333] Modernize more modules (levels 12-16) --- libs/core/threading_base/src/set_thread_state.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/threading_base/src/set_thread_state.cpp b/libs/core/threading_base/src/set_thread_state.cpp index 603d0d12e578..4e02b0684643 100644 --- a/libs/core/threading_base/src/set_thread_state.cpp +++ b/libs/core/threading_base/src/set_thread_state.cpp @@ -160,6 +160,7 @@ namespace hpx::threads::detail { } case thread_schedule_state::deleted: [[fallthrough]]; + case thread_schedule_state::terminated: { LTM_(warning).format( From 96ad1272b8de4154b5697648f083386f1679267b Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 13 Apr 2023 16:12:18 +0200 Subject: [PATCH 133/333] Remove unnecessary README.rst files --- docs/sphinx/releases/whats_new_1_10_0.rst | 25 ---------------------- libs/core/gasnet_base/docs/index.rst | 18 ---------------- libs/full/parcelport_gasnet/docs/index.rst | 18 ---------------- 3 files changed, 61 deletions(-) delete mode 100644 docs/sphinx/releases/whats_new_1_10_0.rst delete mode 100644 libs/core/gasnet_base/docs/index.rst delete mode 100644 libs/full/parcelport_gasnet/docs/index.rst diff --git a/docs/sphinx/releases/whats_new_1_10_0.rst b/docs/sphinx/releases/whats_new_1_10_0.rst deleted file mode 100644 index 747a33e79230..000000000000 --- a/docs/sphinx/releases/whats_new_1_10_0.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. - Copyright (C) 2007-2023 Hartmut Kaiser - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -.. _hpx_1_10_0: - -=========================== -|hpx| V1.10.0 (TBD) -=========================== - -General changes -=============== - -Breaking changes -================ - -Closed issues -============= - -Closed pull requests -==================== - diff --git a/libs/core/gasnet_base/docs/index.rst b/libs/core/gasnet_base/docs/index.rst deleted file mode 100644 index cab19901c949..000000000000 --- a/libs/core/gasnet_base/docs/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. - Copyright (c) 2020 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -.. _modules_gasnet_base: - -=========== -gasnet_base -=========== - -This module provides helper functionality for detecting GASNET environments. - -See the :ref:`API reference ` of this module for more -details. - diff --git a/libs/full/parcelport_gasnet/docs/index.rst b/libs/full/parcelport_gasnet/docs/index.rst deleted file mode 100644 index e26471c5e5e6..000000000000 --- a/libs/full/parcelport_gasnet/docs/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. - Copyright (c) 2020-2021 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -.. _modules_parcelport_gasnet: - -================= -parcelport_gasnet -================= - -TODO: High-level description of the module. - -See the :ref:`API reference ` of this module for more -details. - From d6645a3c6192778ea1a30f574f2f508be96ec8c7 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 11 May 2023 18:50:59 -0500 Subject: [PATCH 134/333] Making sure mutex in locking_hook base stays alive - flyby: fixing problems on older MSVC versions --- .../include/hpx/components_base/server/locking_hook.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp b/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp index b3805ddbe822..42adbcf7342b 100644 --- a/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp +++ b/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include From 9dd1fbadbb808446693430b11e60d55c3bd2d29e Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 12 May 2023 08:05:33 -0500 Subject: [PATCH 135/333] Fixing startup issues in global barrier synchronization --- .../include/hpx/components_base/server/locking_hook.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp b/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp index 42adbcf7342b..b3805ddbe822 100644 --- a/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp +++ b/libs/full/components_base/include/hpx/components_base/server/locking_hook.hpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include From 50de2b1a39e99b8eb1d96e3819321d0bda2630cc Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 22 May 2023 09:26:06 -0500 Subject: [PATCH 136/333] Preparing V1.9.1 release --- docs/sphinx/releases/whats_new_1_9_1.rst | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/sphinx/releases/whats_new_1_9_1.rst diff --git a/docs/sphinx/releases/whats_new_1_9_1.rst b/docs/sphinx/releases/whats_new_1_9_1.rst new file mode 100644 index 000000000000..a0379f4d25c2 --- /dev/null +++ b/docs/sphinx/releases/whats_new_1_9_1.rst @@ -0,0 +1,35 @@ +.. + Copyright (C) 2007-2023 Hartmut Kaiser + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _hpx_1_9_1: + +=========================== +|hpx| V1.9.1 (TBD) +=========================== + +General changes +=============== + +This point release fixes a couple of problems reported for the V1.9.0 release. +Most importantly, we fixed various occasional hanging during startup and shutdown +in distributed scenarios. We also added support for zero-copy serialization on +the receiving side to the TCP, MPI, and LCI parcelports. Last but not least, we +have added support for Visual Studio 2019 and gcc using MINGW on Windows. + + +Breaking changes +================ + +None + +Closed issues +============= + + +Closed pull requests +==================== + From ba44587a71a8d82682d29caf0d4adf308ddee9ad Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 11 May 2023 18:37:18 +0200 Subject: [PATCH 137/333] Fix docs build warning: title overline too short --- docs/sphinx/manual/using_the_lci_parcelport.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/manual/using_the_lci_parcelport.rst b/docs/sphinx/manual/using_the_lci_parcelport.rst index 95e8193282cd..be5b909e8e35 100644 --- a/docs/sphinx/manual/using_the_lci_parcelport.rst +++ b/docs/sphinx/manual/using_the_lci_parcelport.rst @@ -122,4 +122,4 @@ when using the LCI parcelport to get better performance. $ export LCI_PACKET_SIZE=12288 This setting needs roughly 800MB memory per process. The memory consumption mainly -comes from the packets, which can be calculated using `LCI_SERVER_NUM_PKTS x LCI_PACKET_SIZE`. \ No newline at end of file +comes from the packets, which can be calculated using `LCI_SERVER_NUM_PKTS x LCI_PACKET_SIZE`. From 6f9ca1fbd77633feea9da37411ced02fd17295cc Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 18 May 2023 21:03:40 -0500 Subject: [PATCH 138/333] Resolve gcc problems on Windows - flyby: resolve winsock2.h warnings on windows - flyby: add CI builder for gcc/mingw on Windows --- .github/workflows/windows_debug_gcc_mingw.yml | 77 +++++++++++++++++++ libs/core/runtime_local/CMakeLists.txt | 4 + .../thread_support/src/set_thread_name.cpp | 3 + .../hpx/parcelport_lci/parcelport_lci.hpp | 2 +- 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/windows_debug_gcc_mingw.yml diff --git a/.github/workflows/windows_debug_gcc_mingw.yml b/.github/workflows/windows_debug_gcc_mingw.yml new file mode 100644 index 000000000000..4f61f32fc9b3 --- /dev/null +++ b/.github/workflows/windows_debug_gcc_mingw.yml @@ -0,0 +1,77 @@ +# Copyright (c) 2023 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +name: Windows CI (Debug, gcc/mingw toolset) + +on: [pull_request] + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + - uses: jwlawson/actions-setup-cmake@v1.9 + with: + cmake-version: '3.22.x' + - name: Install dependencies + run: | + choco install ninja -y + md C:\projects + $client = new-object System.Net.WebClient + $client.DownloadFile("https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z","C:\projects\boost_1_78_0.7z") + 7z x C:\projects\boost_1_78_0.7z -y -oC:\projects\boost + cd C:\projects\boost\boost_1_78_0 + .\bootstrap.bat gcc + .\b2.exe ` + link=shared ` + variant=debug ` + architecture=x86 ` + address-model=64 ` + threading=multi ` + --with-iostreams ` + --build-type=complete ` + install + $client.DownloadFile("https://download.open-mpi.org/release/hwloc/v2.8/hwloc-win64-build-2.8.0.zip","C:\projects\hwloc-win64-build-2.8.0.zip") + 7z x C:\projects\hwloc-win64-build-2.8.0.zip -y -oC:\projects + - name: Configure + shell: bash + run: | + cmake . -Bbuild -G'Ninja' \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=g++ \ + -DBOOST_ROOT="C:\Boost" \ + -DHWLOC_ROOT="C:\projects\hwloc-win64-build-2.8.0" \ + -DHPX_WITH_CXX_STANDARD=20 \ + -DHPX_WITH_STATIC_LINKING=ON \ + -DHPX_WITH_MALLOC=system \ + -DHPX_WITH_FETCH_ASIO=ON \ + -DHPX_WITH_PKGCONFIG=OFF \ + -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ + -DHPX_WITH_EXAMPLES=ON \ + -DHPX_WITH_TESTS=ON \ + -DHPX_WITH_TESTS_UNIT=ON \ + -DHPX_WITH_DEPRECATION_WARNINGS=OFF \ + -DHPX_WITH_MODULES_AS_STATIC_LIBRARIES=OFF + - name: Build + shell: bash + run: | + cmake --build build --config Debug -- -j 4 + - name: Install + shell: bash + run: | + cmake --install build --config Debug + - name: Test + run: | + Set-Alias -Name grep -Value 'C:\Program Files\Git\usr\bin\grep.exe' + Set-Alias -Name sed -Value 'C:\Program Files\Git\usr\bin\sed.exe' + cd build + ctest ` + --output-on-failure ` + --build-config Debug ` + --tests-regex tests.examples ` + --exclude-regex ` + $(grep -v -e ^# -e ^$ D:/a/hpx/hpx/.github/workflows/tests.examples.targets | sed ':b;N;$!bb;s/\n/|/g') diff --git a/libs/core/runtime_local/CMakeLists.txt b/libs/core/runtime_local/CMakeLists.txt index b0215a7d8942..864a8bc32fc4 100644 --- a/libs/core/runtime_local/CMakeLists.txt +++ b/libs/core/runtime_local/CMakeLists.txt @@ -38,6 +38,10 @@ set(runtime_local_headers hpx/runtime_local/thread_stacktrace.hpp ) +if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) + set(runtime_local_headers ${runtime_local_headers} hpx/init.hpp) +endif() + # cmake-format: off set(runtime_local_compat_headers hpx/custom_exception_info.hpp => hpx/modules/runtime_local.hpp diff --git a/libs/core/thread_support/src/set_thread_name.cpp b/libs/core/thread_support/src/set_thread_name.cpp index 08810b0a3f4f..7bf69ca7676f 100644 --- a/libs/core/thread_support/src/set_thread_name.cpp +++ b/libs/core/thread_support/src/set_thread_name.cpp @@ -16,6 +16,9 @@ !defined(HPX_MINGW) #include +#include +#include + namespace hpx::util { constexpr DWORD MS_VC_EXCEPTION = 0x406D1388; diff --git a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp index 42d9b17b4cb3..95cb7ae38908 100644 --- a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp +++ b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2023 Thomas Heller +../data/kokkos/taskspawn/test1.cpp// Copyright (c) 2014-2023 Thomas Heller // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying From 8eb2c6ca016cb0b45b2faebfea906c2af4d6e62d Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 21 May 2023 14:45:17 -0500 Subject: [PATCH 139/333] Remove duplicate headers, instead generate those on demand --- .github/workflows/windows_debug_gcc_mingw.yml | 77 ------------------- docs/sphinx/api/public_api.rst | 62 +++++++-------- libs/core/runtime_local/CMakeLists.txt | 4 - 3 files changed, 31 insertions(+), 112 deletions(-) delete mode 100644 .github/workflows/windows_debug_gcc_mingw.yml diff --git a/.github/workflows/windows_debug_gcc_mingw.yml b/.github/workflows/windows_debug_gcc_mingw.yml deleted file mode 100644 index 4f61f32fc9b3..000000000000 --- a/.github/workflows/windows_debug_gcc_mingw.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2023 The STE||AR-Group -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -name: Windows CI (Debug, gcc/mingw toolset) - -on: [pull_request] - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - uses: jwlawson/actions-setup-cmake@v1.9 - with: - cmake-version: '3.22.x' - - name: Install dependencies - run: | - choco install ninja -y - md C:\projects - $client = new-object System.Net.WebClient - $client.DownloadFile("https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z","C:\projects\boost_1_78_0.7z") - 7z x C:\projects\boost_1_78_0.7z -y -oC:\projects\boost - cd C:\projects\boost\boost_1_78_0 - .\bootstrap.bat gcc - .\b2.exe ` - link=shared ` - variant=debug ` - architecture=x86 ` - address-model=64 ` - threading=multi ` - --with-iostreams ` - --build-type=complete ` - install - $client.DownloadFile("https://download.open-mpi.org/release/hwloc/v2.8/hwloc-win64-build-2.8.0.zip","C:\projects\hwloc-win64-build-2.8.0.zip") - 7z x C:\projects\hwloc-win64-build-2.8.0.zip -y -oC:\projects - - name: Configure - shell: bash - run: | - cmake . -Bbuild -G'Ninja' \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_COMPILER=g++ \ - -DBOOST_ROOT="C:\Boost" \ - -DHWLOC_ROOT="C:\projects\hwloc-win64-build-2.8.0" \ - -DHPX_WITH_CXX_STANDARD=20 \ - -DHPX_WITH_STATIC_LINKING=ON \ - -DHPX_WITH_MALLOC=system \ - -DHPX_WITH_FETCH_ASIO=ON \ - -DHPX_WITH_PKGCONFIG=OFF \ - -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ - -DHPX_WITH_EXAMPLES=ON \ - -DHPX_WITH_TESTS=ON \ - -DHPX_WITH_TESTS_UNIT=ON \ - -DHPX_WITH_DEPRECATION_WARNINGS=OFF \ - -DHPX_WITH_MODULES_AS_STATIC_LIBRARIES=OFF - - name: Build - shell: bash - run: | - cmake --build build --config Debug -- -j 4 - - name: Install - shell: bash - run: | - cmake --install build --config Debug - - name: Test - run: | - Set-Alias -Name grep -Value 'C:\Program Files\Git\usr\bin\grep.exe' - Set-Alias -Name sed -Value 'C:\Program Files\Git\usr\bin\sed.exe' - cd build - ctest ` - --output-on-failure ` - --build-config Debug ` - --tests-regex tests.examples ` - --exclude-regex ` - $(grep -v -e ^# -e ^$ D:/a/hpx/hpx/.github/workflows/tests.examples.targets | sed ':b;N;$!bb;s/\n/|/g') diff --git a/docs/sphinx/api/public_api.rst b/docs/sphinx/api/public_api.rst index 098e7238cd41..e7a0a55784dd 100644 --- a/docs/sphinx/api/public_api.rst +++ b/docs/sphinx/api/public_api.rst @@ -23,7 +23,7 @@ sub-namespaces will eventually be removed. ``hpx/algorithm.hpp`` ===================== -The header :hpx-header:`libs/full/include/include,hpx/algorithm.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/algorithm.hpp` corresponds to the C++ standard library header :cppreference-header:`algorithm`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -191,7 +191,7 @@ Functions ``hpx/any.hpp`` =============== -The header :hpx-header:`libs/full/include/include,hpx/any.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include,hpx/any.hpp` corresponds to the C++ standard library header :cppreference-header:`any`. :cpp:type:`hpx::any` is compatible with ``std::any``. @@ -253,7 +253,7 @@ Macros ``hpx/barrier.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/barrier.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/barrier.hpp` corresponds to the C++ standard library header :cppreference-header:`barrier` and contains a distributed barrier implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -282,7 +282,7 @@ Classes ``hpx/channel.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/channel.hpp` contains a local and a +The header :hpx-header:`libs/full/include,hpx/channel.hpp` contains a local and a distributed channel implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -310,7 +310,7 @@ Classes ``hpx/chrono.hpp`` ================== -The header :hpx-header:`libs/full/include/include,hpx/chrono.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/chrono.hpp` corresponds to the C++ standard library header :cppreference-header:`chrono`. The following replacements and extensions are provided compared to :cppreference-header:`chrono`. @@ -332,7 +332,7 @@ Classes ``hpx/condition_variable.hpp`` ============================== -The header :hpx-header:`libs/full/include/include,hpx/condition_variable.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include,hpx/condition_variable.hpp` corresponds to the C++ standard library header :cppreference-header:`condition_variable`. Classes @@ -353,7 +353,7 @@ Classes ``hpx/exception.hpp`` ===================== -The header :hpx-header:`libs/full/include/include,hpx/exception.hpp` corresponds to +The header :hpx-header:`libs/full/include,hpx/exception.hpp` corresponds to the C++ standard library header :cppreference-header:`exception`. :cpp:class:`hpx::exception` extends ``std::exception`` and is the base class for all exceptions thrown in |hpx|. :c:macro:`HPX_THROW_EXCEPTION` can be used to throw |hpx| exceptions with file and line information @@ -380,7 +380,7 @@ Classes ``hpx/execution.hpp`` ===================== -The header :hpx-header:`libs/full/include/include,hpx/execution.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/execution.hpp` corresponds to the C++ standard library header :cppreference-header:`execution`. See :ref:`parallel`, :ref:`parallel_algorithms` and :ref:`executor_parameters` for more information about execution policies and executor parameters. @@ -430,7 +430,7 @@ Classes ``hpx/functional.hpp`` ====================== -The header :hpx-header:`libs/full/include/include,hpx/functional.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/functional.hpp` corresponds to the C++ standard library header :cppreference-header:`functional`. :cpp:class:`hpx::function` is a more efficient and serializable replacement for ``std::function``. @@ -493,7 +493,7 @@ Functions ``hpx/future.hpp`` ================== -The header :hpx-header:`libs/full/include/include,hpx/future.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/future.hpp` corresponds to the C++ standard library header :cppreference-header:`future`. See :ref:`extend_futures` for more information about extensions to futures compared to the C++ standard library. @@ -610,7 +610,7 @@ Functions ``hpx/latch.hpp`` ================= -The header :hpx-header:`libs/full/include/include,hpx/latch.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include,hpx/latch.hpp` corresponds to the C++ standard library header :cppreference-header:`latch`. It contains a local and a distributed latch implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -639,7 +639,7 @@ Classes ``hpx/mutex.hpp`` ================= -The header :hpx-header:`libs/full/include/include,hpx/mutex.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/mutex.hpp` corresponds to the C++ standard library header :cppreference-header:`mutex`. Classes @@ -675,7 +675,7 @@ Functions ``hpx/memory.hpp`` ================== -The header :hpx-header:`libs/full/include/include,hpx/memory.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/memory.hpp` corresponds to the C++ standard library header :cppreference-header:`memory`. It contains parallel versions of the copy, fill, move, and construct helper functions in :cppreference-header:`memory`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -722,7 +722,7 @@ Functions ``hpx/numeric.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/numeric.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/numeric.hpp` corresponds to the C++ standard library header :cppreference-header:`numeric`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -762,7 +762,7 @@ Functions ``hpx/optional.hpp`` ==================== -The header :hpx-header:`libs/full/include/include,hpx/optional.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/optional.hpp` corresponds to the C++ standard library header :cppreference-header:`optional`. :cpp:type:`hpx::optional` is compatible with ``std::optional``. @@ -789,7 +789,7 @@ Classes ``hpx/runtime.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/runtime.hpp` contains functions for accessing +The header :hpx-header:`libs/full/include,hpx/runtime.hpp` contains functions for accessing local and distributed runtime information. Typedefs @@ -849,7 +849,7 @@ Functions ``hpx/source_location.hpp`` =========================== -The header :hpx-header:`libs/full/include/include,hpx/source_location.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/source_location.hpp` corresponds to the C++ standard library header :cppreference-header:`source_location`. Classes @@ -868,7 +868,7 @@ Classes ``hpx/system_error.hpp`` ======================== -The header :hpx-header:`libs/full/include/include,hpx/system_error.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/system_error.hpp` corresponds to the C++ standard library header :cppreference-header:`system_error`. Classes @@ -887,7 +887,7 @@ Classes ``hpx/task_block.hpp`` ====================== -The header :hpx-header:`libs/full/include/include,hpx/task_block.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/task_block.hpp` corresponds to the ``task_block`` feature in |cpp17_n4755|_. See :ref:`using_task_block` for more details on using task blocks. @@ -919,16 +919,16 @@ Functions .. _public_api_header_hpx_task_group: -``hpx/task_group.hpp`` -====================== +``hpx/experimental/task_group.hpp`` +=================================== -The header :hpx-header:`libs/full/include/include,hpx/task_group.hpp` corresponds to the -``task_group`` feature in |oneTBB|_. +The header :hpx-header:`libs/core/include,hpx/experimental/task_group.hpp` +corresponds to the ``task_group`` feature in |oneTBB|_. Classes ------- -.. table:: Classes of header ``hpx/task_group.hpp`` +.. table:: Classes of header ``hpx/experimental/task_group.hpp`` +---------------------------------------------------------+ | Class | @@ -941,7 +941,7 @@ Classes ``hpx/thread.hpp`` ================== -The header :hpx-header:`libs/full/include/include,hpx/thread.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/thread.hpp` corresponds to the C++ standard library header :cppreference-header:`thread`. The functionality in this header is equivalent to the standard library thread functionality, with the exception that the |hpx| equivalents are implemented on top of lightweight threads and the |hpx| runtime. @@ -977,7 +977,7 @@ Functions ``hpx/semaphore.hpp`` ===================== -The header :hpx-header:`libs/full/include/include,hpx/semaphore.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/semaphore.hpp` corresponds to the C++ standard library header :cppreference-header:`semaphore`. Classes @@ -997,7 +997,7 @@ Classes ``hpx/shared_mutex.hpp`` ======================== -The header :hpx-header:`libs/full/include/include,hpx/shared_mutex.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/shared_mutex.hpp` corresponds to the C++ standard library header :cppreference-header:`shared_mutex`. Classes @@ -1016,7 +1016,7 @@ Classes ``hpx/stop_token.hpp`` ====================== -The header :hpx-header:`libs/full/include/include,hpx/stop_token.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/stop_token.hpp` corresponds to the C++ standard library header :cppreference-header:`stop_token`. Constants @@ -1049,7 +1049,7 @@ Classes ``hpx/tuple.hpp`` ================= -The header :hpx-header:`libs/full/include/include,hpx/tuple.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/tuple.hpp` corresponds to the C++ standard library header :cppreference-header:`tuple`. :cpp:class:`hpx::tuple` can be used in CUDA device code, unlike ``std::tuple``. @@ -1097,7 +1097,7 @@ Functions ``hpx/type_traits.hpp`` ======================= -The header :hpx-header:`libs/full/include/include,hpx/type_traits.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/type_traits.hpp` corresponds to the C++ standard library header :cppreference-header:`type_traits`. Classes @@ -1117,7 +1117,7 @@ Classes ``hpx/unwrap.hpp`` ================== -The header :hpx-header:`libs/full/include/include,hpx/unwrap.hpp` contains utilities for +The header :hpx-header:`libs/full/include,hpx/unwrap.hpp` contains utilities for unwrapping futures. Classes diff --git a/libs/core/runtime_local/CMakeLists.txt b/libs/core/runtime_local/CMakeLists.txt index 864a8bc32fc4..b0215a7d8942 100644 --- a/libs/core/runtime_local/CMakeLists.txt +++ b/libs/core/runtime_local/CMakeLists.txt @@ -38,10 +38,6 @@ set(runtime_local_headers hpx/runtime_local/thread_stacktrace.hpp ) -if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) - set(runtime_local_headers ${runtime_local_headers} hpx/init.hpp) -endif() - # cmake-format: off set(runtime_local_compat_headers hpx/custom_exception_info.hpp => hpx/modules/runtime_local.hpp From 4d64e6c75b2e80b86f0e3937a7c8a8794f3c757f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 10 Apr 2023 15:51:24 -0500 Subject: [PATCH 140/333] Cleaning up #includes in hpx/ folder - unifying `` and `` --- cmake/tests/set_thread_name.cpp | 12 ------------ .../tests/regressions/protect_with_nullary_pfo.cpp | 6 ++---- .../include/hpx/io_service/io_service_pool_fwd.hpp | 14 -------------- libs/full/init_runtime/src/hpx_init.cpp | 2 +- 4 files changed, 3 insertions(+), 31 deletions(-) delete mode 100644 cmake/tests/set_thread_name.cpp delete mode 100644 libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp diff --git a/cmake/tests/set_thread_name.cpp b/cmake/tests/set_thread_name.cpp deleted file mode 100644 index d3ab3b4e1fdb..000000000000 --- a/cmake/tests/set_thread_name.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2023 STE||AR Group -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#include - -int main() -{ - pthread_setname_np(pthread_self(), ""); -} diff --git a/libs/core/functional/tests/regressions/protect_with_nullary_pfo.cpp b/libs/core/functional/tests/regressions/protect_with_nullary_pfo.cpp index 7db062326f09..44d2eee73998 100644 --- a/libs/core/functional/tests/regressions/protect_with_nullary_pfo.cpp +++ b/libs/core/functional/tests/regressions/protect_with_nullary_pfo.cpp @@ -6,11 +6,9 @@ // http://www.boost.org/LICENSE_1_0.txt) #include -#include +#include +#include #include -#include -#include -#include #include #include diff --git a/libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp b/libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp deleted file mode 100644 index 9d871fd96cd5..000000000000 --- a/libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2023 Hartmut Kaiser -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#pragma once - -#include - -namespace hpx::util { - - class HPX_CORE_EXPORT io_service_pool; -} // namespace hpx::util diff --git a/libs/full/init_runtime/src/hpx_init.cpp b/libs/full/init_runtime/src/hpx_init.cpp index 67222404d61e..a0966988a586 100644 --- a/libs/full/init_runtime/src/hpx_init.cpp +++ b/libs/full/init_runtime/src/hpx_init.cpp @@ -98,7 +98,7 @@ namespace hpx_startup { std::vector (*user_main_config_function)( std::vector const&) = nullptr; -} +} // namespace hpx_startup /////////////////////////////////////////////////////////////////////////////// namespace hpx::detail { From ee13ebaf9b83726cf70a1acb22fe0d15ffaa8a72 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Thu, 18 May 2023 12:53:35 -0500 Subject: [PATCH 141/333] Re-enable tests Tests were disabled in #5796 --- libs/core/async_cuda/tests/performance/synchronize.cu | 6 ------ libs/core/async_cuda/tests/unit/transform_stream.cu | 6 ------ 2 files changed, 12 deletions(-) diff --git a/libs/core/async_cuda/tests/performance/synchronize.cu b/libs/core/async_cuda/tests/performance/synchronize.cu index d206638da99a..7f16159ec9db 100644 --- a/libs/core/async_cuda/tests/performance/synchronize.cu +++ b/libs/core/async_cuda/tests/performance/synchronize.cu @@ -269,9 +269,3 @@ int main(int argc, char* argv[]) return hpx::local::init(hpx_main, argc, argv, init_args); } -#else -int main(int, char*[]) -{ - return 0; -} -#endif diff --git a/libs/core/async_cuda/tests/unit/transform_stream.cu b/libs/core/async_cuda/tests/unit/transform_stream.cu index d8ad49613be9..7cc1eb94c9b0 100644 --- a/libs/core/async_cuda/tests/unit/transform_stream.cu +++ b/libs/core/async_cuda/tests/unit/transform_stream.cu @@ -302,9 +302,3 @@ int main(int argc, char* argv[]) return hpx::util::report_errors(); } -#else -int main(int, char*[]) -{ - return 0; -} -#endif From 86775628c27a1b9121ddb91eedd125d172467ae1 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Thu, 18 May 2023 13:00:02 -0500 Subject: [PATCH 142/333] Fix compute architecture and drop kepler Kepler support is removed in CUDA 12 so it's time to remove it here as well. --- .jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh b/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh index 3dc9e8a74287..f2c6938e4437 100644 --- a/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh +++ b/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh @@ -4,5 +4,5 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -configuration_slurm_partition="jenkins-cuda" +configuration_slurm_partition="cuda-V100,cuda-A100" configuration_slurm_num_nodes="1" From e88ae851d0ea1da5bbcfd029bc7c12481ec71d66 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Thu, 18 May 2023 14:55:12 -0500 Subject: [PATCH 143/333] Revert "Re-enable tests" This reverts commit 9d5a4a808b9051db7a73dad63076cefa6d7e4b36. Still not working unfortunately (tested with gcc/12 and cuda 12.1) --- libs/core/async_cuda/tests/performance/synchronize.cu | 6 ++++++ libs/core/async_cuda/tests/unit/transform_stream.cu | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/libs/core/async_cuda/tests/performance/synchronize.cu b/libs/core/async_cuda/tests/performance/synchronize.cu index 7f16159ec9db..d206638da99a 100644 --- a/libs/core/async_cuda/tests/performance/synchronize.cu +++ b/libs/core/async_cuda/tests/performance/synchronize.cu @@ -269,3 +269,9 @@ int main(int argc, char* argv[]) return hpx::local::init(hpx_main, argc, argv, init_args); } +#else +int main(int, char*[]) +{ + return 0; +} +#endif diff --git a/libs/core/async_cuda/tests/unit/transform_stream.cu b/libs/core/async_cuda/tests/unit/transform_stream.cu index 7cc1eb94c9b0..d8ad49613be9 100644 --- a/libs/core/async_cuda/tests/unit/transform_stream.cu +++ b/libs/core/async_cuda/tests/unit/transform_stream.cu @@ -302,3 +302,9 @@ int main(int argc, char* argv[]) return hpx::util::report_errors(); } +#else +int main(int, char*[]) +{ + return 0; +} +#endif From 0eb3008192dc046190f3709133b05706e142b10e Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Thu, 18 May 2023 17:10:59 -0500 Subject: [PATCH 144/333] Remove broken node for testing --- .jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh b/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh index f2c6938e4437..3dc9e8a74287 100644 --- a/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh +++ b/.jenkins/lsu/slurm-configuration-gcc-10-cuda-11.sh @@ -4,5 +4,5 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -configuration_slurm_partition="cuda-V100,cuda-A100" +configuration_slurm_partition="jenkins-cuda" configuration_slurm_num_nodes="1" From 4595abfc80d1b832c53671ab2079afec6952c0d8 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Thu, 18 May 2023 19:04:02 -0500 Subject: [PATCH 145/333] Temporarily switch to rocm/4 As all rocm/5 modules are currently broken on the machine --- .jenkins/lsu/env-hipcc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/lsu/env-hipcc.sh b/.jenkins/lsu/env-hipcc.sh index e31a25c4fa26..93fc3184aa37 100644 --- a/.jenkins/lsu/env-hipcc.sh +++ b/.jenkins/lsu/env-hipcc.sh @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -module load rocm +module load rocm/4 module load boost export CXX=hipcc From 430c1b158a486acc8042d2e309587ac4d01e9f2b Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Tue, 23 May 2023 14:25:16 -0500 Subject: [PATCH 146/333] Revert "Temporarily switch to rocm/4" This reverts commit 71c8ac34e334cabdd8bf4b9f872eaa79b8f7f7eb. --- .jenkins/lsu/env-hipcc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/lsu/env-hipcc.sh b/.jenkins/lsu/env-hipcc.sh index 93fc3184aa37..e31a25c4fa26 100644 --- a/.jenkins/lsu/env-hipcc.sh +++ b/.jenkins/lsu/env-hipcc.sh @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -module load rocm/4 +module load rocm module load boost export CXX=hipcc From 8e5bc55df99a60339766dc3faefdcf5f66275701 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Tue, 23 May 2023 23:46:18 -0500 Subject: [PATCH 147/333] Increase test timeout --- .jenkins/lsu-perftests/ctest.cmake | 2 +- .jenkins/lsu/ctest.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/lsu-perftests/ctest.cmake b/.jenkins/lsu-perftests/ctest.cmake index f8cf66b3301f..3acd5888f32c 100644 --- a/.jenkins/lsu-perftests/ctest.cmake +++ b/.jenkins/lsu-perftests/ctest.cmake @@ -8,7 +8,7 @@ # This is a dummy file to trigger the upload of the perftests reports cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -set(CTEST_TEST_TIMEOUT 300) +set(CTEST_TEST_TIMEOUT 900) set(CTEST_BUILD_PARALLELISM 20) set(CTEST_TEST_PARALLELISM 4) set(CTEST_CMAKE_GENERATOR Ninja) diff --git a/.jenkins/lsu/ctest.cmake b/.jenkins/lsu/ctest.cmake index 135f07c12025..f5be5696d025 100644 --- a/.jenkins/lsu/ctest.cmake +++ b/.jenkins/lsu/ctest.cmake @@ -7,7 +7,7 @@ cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -set(CTEST_TEST_TIMEOUT 300) +set(CTEST_TEST_TIMEOUT 900) set(CTEST_BUILD_PARALLELISM 20) set(CTEST_TEST_PARALLELISM 4) set(CTEST_CMAKE_GENERATOR Ninja) From 9a53790d23492b286fc980406766ebd25cf6ae18 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Fri, 26 May 2023 23:43:10 -0500 Subject: [PATCH 148/333] Revert "Increase test timeout" This reverts commit 087074ac3429e67004b70efcd74e74531250825e. --- .jenkins/lsu-perftests/ctest.cmake | 2 +- .jenkins/lsu/ctest.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.jenkins/lsu-perftests/ctest.cmake b/.jenkins/lsu-perftests/ctest.cmake index 3acd5888f32c..f8cf66b3301f 100644 --- a/.jenkins/lsu-perftests/ctest.cmake +++ b/.jenkins/lsu-perftests/ctest.cmake @@ -8,7 +8,7 @@ # This is a dummy file to trigger the upload of the perftests reports cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -set(CTEST_TEST_TIMEOUT 900) +set(CTEST_TEST_TIMEOUT 300) set(CTEST_BUILD_PARALLELISM 20) set(CTEST_TEST_PARALLELISM 4) set(CTEST_CMAKE_GENERATOR Ninja) diff --git a/.jenkins/lsu/ctest.cmake b/.jenkins/lsu/ctest.cmake index f5be5696d025..135f07c12025 100644 --- a/.jenkins/lsu/ctest.cmake +++ b/.jenkins/lsu/ctest.cmake @@ -7,7 +7,7 @@ cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -set(CTEST_TEST_TIMEOUT 900) +set(CTEST_TEST_TIMEOUT 300) set(CTEST_BUILD_PARALLELISM 20) set(CTEST_TEST_PARALLELISM 4) set(CTEST_CMAKE_GENERATOR Ninja) From b5ccb3a0398fc67acf27875c96cc65493c01bdc4 Mon Sep 17 00:00:00 2001 From: Shreyas Atre Date: Thu, 25 May 2023 23:55:20 +0530 Subject: [PATCH 149/333] Cmake Tests: Delete operator check for size_t arg - Clang fails to recognize the global delete operator which has 2 arguments the second one being std::size_t Signed-off-by: Shreyas Atre --- libs/core/type_support/include/hpx/type_support/generator.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/generator.hpp b/libs/core/type_support/include/hpx/type_support/generator.hpp index 75952a74c630..bd7a5fa998f6 100644 --- a/libs/core/type_support/include/hpx/type_support/generator.hpp +++ b/libs/core/type_support/include/hpx/type_support/generator.hpp @@ -64,8 +64,8 @@ namespace hpx { Allocator>::template rebind_alloc; template - concept has_real_pointers = std::is_void_v || - std::is_pointer_v< + concept has_real_pointers = + std::is_void_v || std::is_pointer_v< typename std::allocator_traits::pointer>; // clang-format off From e062de5e2d393c1498336fe413a70b763ac15c74 Mon Sep 17 00:00:00 2001 From: Shreyas Atre Date: Sat, 27 May 2023 00:37:51 +0530 Subject: [PATCH 150/333] Fix clang format formatting for Clang format 11 Signed-off-by: Shreyas Atre --- libs/core/type_support/include/hpx/type_support/generator.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/generator.hpp b/libs/core/type_support/include/hpx/type_support/generator.hpp index bd7a5fa998f6..75952a74c630 100644 --- a/libs/core/type_support/include/hpx/type_support/generator.hpp +++ b/libs/core/type_support/include/hpx/type_support/generator.hpp @@ -64,8 +64,8 @@ namespace hpx { Allocator>::template rebind_alloc; template - concept has_real_pointers = - std::is_void_v || std::is_pointer_v< + concept has_real_pointers = std::is_void_v || + std::is_pointer_v< typename std::allocator_traits::pointer>; // clang-format off From 75d7147127ed58814745dce80414785c4138f34d Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 15 Jun 2023 08:31:01 -0500 Subject: [PATCH 151/333] Resolving merge conflicts --- libs/core/functional/tests/regressions/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/core/functional/tests/regressions/CMakeLists.txt b/libs/core/functional/tests/regressions/CMakeLists.txt index 7c749b67d0ff..677e7fe6b160 100644 --- a/libs/core/functional/tests/regressions/CMakeLists.txt +++ b/libs/core/functional/tests/regressions/CMakeLists.txt @@ -15,8 +15,6 @@ foreach(test ${function_tests}) add_hpx_executable( ${test}_test INTERNAL_FLAGS SOURCES ${sources} ${${test}_FLAGS} - NOLIBS - DEPENDENCIES hpx_core EXCLUDE_FROM_ALL FOLDER "Tests/Regressions/Modules/Core/Functional" ) From 1b1cf55c50beb8b8c96e5b79f67bd3d485dd67b5 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 1 Jun 2023 22:38:15 +0200 Subject: [PATCH 152/333] Add openmp tasks migration guide --- docs/sphinx/manual/migration_guide.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 01a6099b4df3..83027781549e 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -536,7 +536,6 @@ care of synchronization: future_outer.get(); // Wait for the outer task to complete - Task yield ^^^^^^^^^^ From 5c05502c7d5d6ea742747006fa9700d1eccf5e50 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 19 Jun 2023 13:00:48 -0500 Subject: [PATCH 153/333] Update release notes --- docs/sphinx/quickstart.rst | 2 +- docs/sphinx/releases/whats_new_1_9_1.rst | 52 +++++++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/quickstart.rst b/docs/sphinx/quickstart.rst index 17ec0141e4c4..c689c7309fcb 100644 --- a/docs/sphinx/quickstart.rst +++ b/docs/sphinx/quickstart.rst @@ -73,7 +73,7 @@ build an executable using |cmake| and |hpx|: .. code-block:: cmake - cmake_minimum_required(VERSION 3.18) + cmake_minimum_required(VERSION 3.19) project(my_hpx_project CXX) find_package(HPX REQUIRED) add_executable(my_hpx_program main.cpp) diff --git a/docs/sphinx/releases/whats_new_1_9_1.rst b/docs/sphinx/releases/whats_new_1_9_1.rst index a0379f4d25c2..1ce207212654 100644 --- a/docs/sphinx/releases/whats_new_1_9_1.rst +++ b/docs/sphinx/releases/whats_new_1_9_1.rst @@ -18,7 +18,21 @@ This point release fixes a couple of problems reported for the V1.9.0 release. Most importantly, we fixed various occasional hanging during startup and shutdown in distributed scenarios. We also added support for zero-copy serialization on the receiving side to the TCP, MPI, and LCI parcelports. Last but not least, we -have added support for Visual Studio 2019 and gcc using MINGW on Windows. +have added support for Visual Studio 2019 and gcc using MINGW on Windows, and +also support for gcc V13 and clang V15. + +HPX headers are now made consistently named the same as their standard library +counterparts, e.g. `#include ` now corresponds to `#include `. +This significantly simplifies porting existing standards conforming codes to HPX. + +A lot of work has been done to improve and optimize our network communication +layers. Primary focus of this work was on the LCI parcelport, but we have also +cleaned up and improved the MPI parcelport. + +Additionally, we have continued working on our documentation. The main focus +here was on completing the API documentation of the most important API functions. +We have started adding migration guides for people interested in moving their +codes away from other, commonplace parallelization frameworks like OpenMP. Breaking changes @@ -29,7 +43,43 @@ None Closed issues ============= +* :hpx-issue:`6155` - hpxcxx and hpxrun.py do not work if HPX_WITH_TESTS=OFF +* :hpx-issue:`6164` - HPX_WITH_DATAPAR_BACKEND=EVE causes compile errors with C++17 +* :hpx-issue:`6175` - Make sure all our parallel algorithms accept the predicates by value +* :hpx-issue:`6194` - tests.regressions.threads.threads_all_1422 failed at Perlmutter +* :hpx-issue:`6198` - set_intersection/set_difference fails when run with execution::par +* :hpx-issue:`6214` - Broken Links to the Documentation page in readme.rst +* :hpx-issue:`6217` - hpx::make_heap does not terminate when exPolicy is par (or par_unseq) and size of vector is 4 +* :hpx-issue:`6246` - HPX fails to compile under cxx 20 (fresh system) +* :hpx-issue:`6247` - HPX 1.9.0 does not compile with GCC on Windows +* :hpx-issue:`6282` - The "attach-debugger" option is broken on the current master branch. + Closed pull requests ==================== +* :hpx-pr:`6219` - Cleaning up #includes in hpx/ folder +* :hpx-pr:`6223` - Move documentation from README.rst to index.rst files under libs directory +* :hpx-pr:`6229` - Adding zero-copy support on the receiving end of the TCP and MPI parcel ports +* :hpx-pr:`6231` - Remove deprecated email from release procedure +* :hpx-pr:`6235` - Modernize more modules (levels 12-16) +* :hpx-pr:`6236` - Attempt to resolve occasional shutdown hangs in distributed operation +* :hpx-pr:`6239` - Fix Optimizing HPX applications page of Manual +* :hpx-pr:`6241` - LCI parcelport: Refactor, add more variants, zero copy receives. +* :hpx-pr:`6242` - updated deprecated headers +* :hpx-pr:`6243` - Adding github action builders using VS2019 +* :hpx-pr:`6248` - Fix CUDA/HIP Jenkins pipelines +* :hpx-pr:`6250` - Resolve gcc problems on Windows +* :hpx-pr:`6251` - Attempting to fix problems in barrier causing hangs +* :hpx-pr:`6253` - Modernize set_thread_name on Windows +* :hpx-pr:`6256` - Fix nvcc/gcc-10 (Octo-Tiger) compilation issue +* :hpx-pr:`6257` - Cmake Tests: Delete operator check for size_t arg +* :hpx-pr:`6258` - Rewriting wait_some to circumvent data races causing hangs +* :hpx-pr:`6260` - Add migration guide to manual +* :hpx-pr:`6262` - Fixing wrong command line options in local command line handling +* :hpx-pr:`6266` - Attempt to resolve occasional hang in run_loop +* :hpx-pr:`6267` - Attempting to fix migration tests +* :hpx-pr:`6278` - Making sure the future's shared state doesn't go out of scope prematurely +* :hpx-pr:`6279` - Re-expose error names +* :hpx-pr:`6281` - Creating directory for file copy +* :hpx-pr:`6283` - Consistently #include unistd.h for _POSIX_VERSION From 5ca73bc494867a997ccd27582c54d1412ab30dfc Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Wed, 14 Jun 2023 14:21:34 -0500 Subject: [PATCH 154/333] Fixing problems reported by clang-15 --- libs/core/type_support/tests/unit/generator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/type_support/tests/unit/generator.cpp b/libs/core/type_support/tests/unit/generator.cpp index 0bb35a2d0302..d7ab48a41c0c 100644 --- a/libs/core/type_support/tests/unit/generator.cpp +++ b/libs/core/type_support/tests/unit/generator.cpp @@ -116,7 +116,7 @@ namespace tests { hpx::generator const_lvalue_example() { - co_yield X{1}; // OK + co_yield X{1}; // OK X const x{2}; co_yield x; // OK From 2d15803291e8b1850de9688823daa9cf455bff6f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Wed, 14 Jun 2023 14:51:44 -0500 Subject: [PATCH 155/333] Fixing problems reported by gcc-13 --- libs/core/datastructures/tests/unit/bitset_test.hpp | 4 ++++ .../include/hpx/executors/guided_pool_executor.hpp | 4 ++++ libs/core/functional/tests/unit/bind_dm3_test.cpp | 9 +++++++++ libs/core/pack_traversal/tests/unit/pack_traversal.cpp | 8 ++++++++ 4 files changed, 25 insertions(+) diff --git a/libs/core/datastructures/tests/unit/bitset_test.hpp b/libs/core/datastructures/tests/unit/bitset_test.hpp index e9304192ca4e..99cb8f4b20e7 100644 --- a/libs/core/datastructures/tests/unit/bitset_test.hpp +++ b/libs/core/datastructures/tests/unit/bitset_test.hpp @@ -320,6 +320,10 @@ struct bitset_test b = std::move(b); // self assignment check #endif +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 +#pragma GCC diagnostic pop +#endif + // NOLINTNEXTLINE(bugprone-use-after-move) HPX_TEST(b == rhs); } diff --git a/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp b/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp index 924a18bcb1e3..d276f851bbd0 100644 --- a/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp +++ b/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp @@ -196,6 +196,10 @@ namespace hpx::parallel::execution { #ifdef GUIDED_POOL_EXECUTOR_FAKE_NOOP int domain = -1; #else +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Werror=dangling-reference" +#endif // get the argument for the numa hint function from the predecessor future int domain = numa_function_(detail::future_extract_value()( diff --git a/libs/core/functional/tests/unit/bind_dm3_test.cpp b/libs/core/functional/tests/unit/bind_dm3_test.cpp index 2f1f54f91423..57281fe7b067 100644 --- a/libs/core/functional/tests/unit/bind_dm3_test.cpp +++ b/libs/core/functional/tests/unit/bind_dm3_test.cpp @@ -36,8 +36,17 @@ int main() pair_type pair(10, 20); +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Werror=dangling-reference" +#endif + int const& x = hpx::bind(&pair_type::first, placeholders::_1)(pair); +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 +#pragma GCC diagnostic pop +#endif + HPX_TEST_EQ(&pair.first, &x); #endif diff --git a/libs/core/pack_traversal/tests/unit/pack_traversal.cpp b/libs/core/pack_traversal/tests/unit/pack_traversal.cpp index 47038f41533c..fa2d9a5fbf10 100644 --- a/libs/core/pack_traversal/tests/unit/pack_traversal.cpp +++ b/libs/core/pack_traversal/tests/unit/pack_traversal.cpp @@ -486,6 +486,10 @@ static void test_strategic_traverse() { std::unique_ptr ptr(new int(7)); +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Werror=dangling-reference" +#endif std::unique_ptr const& ref = map_pack( [](std::unique_ptr const& ref) -> std::unique_ptr const& { // ... @@ -493,6 +497,10 @@ static void test_strategic_traverse() }, ptr); +#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 +#pragma GCC diagnostic pop +#endif + HPX_TEST_EQ(*ref, 7); *ptr = 0; HPX_TEST_EQ(*ref, 0); From 32c7568feb260683d63801010d7012a4d1df83c5 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 18 Jun 2023 12:16:06 -0500 Subject: [PATCH 156/333] Fixing various feature tests --- .../include/hpx/executors/guided_pool_executor.hpp | 4 ---- libs/core/functional/tests/unit/bind_dm3_test.cpp | 9 --------- libs/core/pack_traversal/tests/unit/pack_traversal.cpp | 8 -------- libs/core/type_support/tests/unit/generator.cpp | 2 +- 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp b/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp index d276f851bbd0..924a18bcb1e3 100644 --- a/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp +++ b/libs/core/executors/include/hpx/executors/guided_pool_executor.hpp @@ -196,10 +196,6 @@ namespace hpx::parallel::execution { #ifdef GUIDED_POOL_EXECUTOR_FAKE_NOOP int domain = -1; #else -#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Werror=dangling-reference" -#endif // get the argument for the numa hint function from the predecessor future int domain = numa_function_(detail::future_extract_value()( diff --git a/libs/core/functional/tests/unit/bind_dm3_test.cpp b/libs/core/functional/tests/unit/bind_dm3_test.cpp index 57281fe7b067..2f1f54f91423 100644 --- a/libs/core/functional/tests/unit/bind_dm3_test.cpp +++ b/libs/core/functional/tests/unit/bind_dm3_test.cpp @@ -36,17 +36,8 @@ int main() pair_type pair(10, 20); -#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Werror=dangling-reference" -#endif - int const& x = hpx::bind(&pair_type::first, placeholders::_1)(pair); -#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 -#pragma GCC diagnostic pop -#endif - HPX_TEST_EQ(&pair.first, &x); #endif diff --git a/libs/core/pack_traversal/tests/unit/pack_traversal.cpp b/libs/core/pack_traversal/tests/unit/pack_traversal.cpp index fa2d9a5fbf10..47038f41533c 100644 --- a/libs/core/pack_traversal/tests/unit/pack_traversal.cpp +++ b/libs/core/pack_traversal/tests/unit/pack_traversal.cpp @@ -486,10 +486,6 @@ static void test_strategic_traverse() { std::unique_ptr ptr(new int(7)); -#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Werror=dangling-reference" -#endif std::unique_ptr const& ref = map_pack( [](std::unique_ptr const& ref) -> std::unique_ptr const& { // ... @@ -497,10 +493,6 @@ static void test_strategic_traverse() }, ptr); -#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 -#pragma GCC diagnostic pop -#endif - HPX_TEST_EQ(*ref, 7); *ptr = 0; HPX_TEST_EQ(*ref, 0); diff --git a/libs/core/type_support/tests/unit/generator.cpp b/libs/core/type_support/tests/unit/generator.cpp index d7ab48a41c0c..0bb35a2d0302 100644 --- a/libs/core/type_support/tests/unit/generator.cpp +++ b/libs/core/type_support/tests/unit/generator.cpp @@ -116,7 +116,7 @@ namespace tests { hpx::generator const_lvalue_example() { - co_yield X{1}; // OK + co_yield X{1}; // OK X const x{2}; co_yield x; // OK From 667cb6ecf6f2a8c8a24b51aaf1923c3e498fec2f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 26 Jun 2023 11:11:59 -0500 Subject: [PATCH 157/333] Adding user supplied on_finalize --- libs/full/init_runtime/src/hpx_init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/full/init_runtime/src/hpx_init.cpp b/libs/full/init_runtime/src/hpx_init.cpp index a0966988a586..67222404d61e 100644 --- a/libs/full/init_runtime/src/hpx_init.cpp +++ b/libs/full/init_runtime/src/hpx_init.cpp @@ -98,7 +98,7 @@ namespace hpx_startup { std::vector (*user_main_config_function)( std::vector const&) = nullptr; -} // namespace hpx_startup +} /////////////////////////////////////////////////////////////////////////////// namespace hpx::detail { From b08bdacff126161c0b91feb64cd6177db577264c Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Mon, 19 Jun 2023 14:04:16 -0500 Subject: [PATCH 158/333] Initial hardcoded version --- libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp | 2 +- libs/core/async_cuda/src/cuda_event_callback.cpp | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp b/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp index e5e617c47171..17c21c35ba76 100644 --- a/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp +++ b/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp @@ -131,7 +131,7 @@ namespace hpx { namespace cuda { namespace experimental { } future_data(init_no_addref no_addref, other_allocator const& alloc, - cudaStream_t stream) + cudaStream_t stream, int device) : lcos::detail::future_data_allocator(no_addref, alloc) , rt_(hpx::get_runtime_ptr()) diff --git a/libs/core/async_cuda/src/cuda_event_callback.cpp b/libs/core/async_cuda/src/cuda_event_callback.cpp index 90698b402a98..e54daf518a9b 100644 --- a/libs/core/async_cuda/src/cuda_event_callback.cpp +++ b/libs/core/async_cuda/src/cuda_event_callback.cpp @@ -116,6 +116,7 @@ namespace hpx { namespace cuda { namespace experimental { namespace detail { HPX_THROW_EXCEPTION(hpx::error::invalid_status, "add_event_callback", "could not get an event"); } + check_cuda_error(cudaSetDevice(device)); check_cuda_error(cudaEventRecord(event, stream)); detail::add_to_event_callback_queue( @@ -160,9 +161,6 @@ namespace hpx { namespace cuda { namespace experimental { namespace detail { debug::dec<3>(get_number_of_active_events())); } - // Grab the handle to the event pool so we can return completed events - cuda_event_pool& pool = - hpx::cuda::experimental::cuda_event_pool::get_event_pool(); // Iterate over our list of events and see if any have completed event_callback_vector.erase( From 0d9a705f371736572fc57c05aad64e5d3230589d Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Mon, 19 Jun 2023 15:22:04 -0500 Subject: [PATCH 159/333] Cleanup --- libs/core/async_cuda/src/cuda_event_callback.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/core/async_cuda/src/cuda_event_callback.cpp b/libs/core/async_cuda/src/cuda_event_callback.cpp index e54daf518a9b..04af46cf9fc4 100644 --- a/libs/core/async_cuda/src/cuda_event_callback.cpp +++ b/libs/core/async_cuda/src/cuda_event_callback.cpp @@ -161,6 +161,9 @@ namespace hpx { namespace cuda { namespace experimental { namespace detail { debug::dec<3>(get_number_of_active_events())); } + // Grab the handle to the event pool so we can return completed events + cuda_event_pool& pool = + hpx::cuda::experimental::cuda_event_pool::get_event_pool(); // Iterate over our list of events and see if any have completed event_callback_vector.erase( From bc31b537a2a57bc2d91004de0e62ade47e31b3ac Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Wed, 21 Jun 2023 14:07:16 -0500 Subject: [PATCH 160/333] Fix unused parameter warning --- libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp b/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp index 17c21c35ba76..e5e617c47171 100644 --- a/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp +++ b/libs/core/async_cuda/include/hpx/async_cuda/cuda_future.hpp @@ -131,7 +131,7 @@ namespace hpx { namespace cuda { namespace experimental { } future_data(init_no_addref no_addref, other_allocator const& alloc, - cudaStream_t stream, int device) + cudaStream_t stream) : lcos::detail::future_data_allocator(no_addref, alloc) , rt_(hpx::get_runtime_ptr()) From c38dcc215c77706a7e0db61024ae41f757619dc7 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Tue, 27 Jun 2023 11:14:46 -0500 Subject: [PATCH 161/333] Fix assert --- libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp b/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp index ce4db339969f..b6afa226fbb2 100644 --- a/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp +++ b/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp @@ -1,7 +1,6 @@ // Copyright (c) 2023 Gregor Daiß // Copyright (c) 2020 John Biddiscombe // Copyright (c) 2020 Teodor Nikolov -// // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From ff37ae09c6d0f6a89d3c9112cb623099e5aa5d96 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Tue, 27 Jun 2023 14:54:41 -0500 Subject: [PATCH 162/333] Remove superfluous cudaSetDevice --- libs/core/async_cuda/src/cuda_event_callback.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/async_cuda/src/cuda_event_callback.cpp b/libs/core/async_cuda/src/cuda_event_callback.cpp index 04af46cf9fc4..90698b402a98 100644 --- a/libs/core/async_cuda/src/cuda_event_callback.cpp +++ b/libs/core/async_cuda/src/cuda_event_callback.cpp @@ -116,7 +116,6 @@ namespace hpx { namespace cuda { namespace experimental { namespace detail { HPX_THROW_EXCEPTION(hpx::error::invalid_status, "add_event_callback", "could not get an event"); } - check_cuda_error(cudaSetDevice(device)); check_cuda_error(cudaEventRecord(event, stream)); detail::add_to_event_callback_queue( From 66febef97fee1bb88516c4825c67254a2a8718c3 Mon Sep 17 00:00:00 2001 From: Gregor Daiss Date: Wed, 28 Jun 2023 14:18:32 -0500 Subject: [PATCH 163/333] Put event pool singleton access definition in src --- libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp b/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp index b6afa226fbb2..ce4db339969f 100644 --- a/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp +++ b/libs/core/async_cuda/include/hpx/async_cuda/cuda_event.hpp @@ -1,6 +1,7 @@ // Copyright (c) 2023 Gregor Daiß // Copyright (c) 2020 John Biddiscombe // Copyright (c) 2020 Teodor Nikolov +// // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From 34d4cf57eb8b82e5724e0ec15507bb641f8da7fa Mon Sep 17 00:00:00 2001 From: STE||AR Group Date: Fri, 4 Aug 2023 18:34:38 -0400 Subject: [PATCH 164/333] Final patch release additions --- CITATION.cff | 6 +++--- cmake/packaging/rpm/Changelog.txt | 3 +++ docs/sphinx/releases/whats_new_1_9_1.rst | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 2a034898c87e..2ab1d9bb6a65 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -7,7 +7,7 @@ # Metadata for citation of this software according to the CFF format (https://citation-file-format.github.io/) cff-version: 1.2.0 message: If you use this software, please cite it using these metadata. -title: 'STEllAR-GROUP/hpx: HPX V1.9.0: The C++ Standards Library for Parallelism and +title: 'STEllAR-GROUP/hpx: HPX V1.9.1: The C++ Standards Library for Parallelism and Concurrency' doi: 10.5281/zenodo.5185328 authors: @@ -84,7 +84,7 @@ authors: - given-names: Zahra family-names: Khatami affiliation: Louisiana State University -version: 1.9.0 -date-released: 2023-05-02 +version: 1.9.1 +date-released: 2023-08-04 repository-code: https://github.com/STEllAR-GROUP/hpx license: BSL-1.0 diff --git a/cmake/packaging/rpm/Changelog.txt b/cmake/packaging/rpm/Changelog.txt index d36d4893e21e..231fcee5d395 100644 --- a/cmake/packaging/rpm/Changelog.txt +++ b/cmake/packaging/rpm/Changelog.txt @@ -1,3 +1,6 @@ +* Fri Aug 04 2023 STE||AR Group 1.9.1 +- HPX Release 1.9.1 (https://hpx-docs.stellar-group.org/tags/1.9.1/html/releases/whats_new_1_9_1.html) + * Tue May 02 2023 STE||AR Group 1.9.0 - HPX Release 1.9.0 (https://hpx-docs.stellar-group.org/tags/1.9.0/html/releases/whats_new_1_9_0.html) diff --git a/docs/sphinx/releases/whats_new_1_9_1.rst b/docs/sphinx/releases/whats_new_1_9_1.rst index 1ce207212654..34c56d86a857 100644 --- a/docs/sphinx/releases/whats_new_1_9_1.rst +++ b/docs/sphinx/releases/whats_new_1_9_1.rst @@ -8,7 +8,7 @@ .. _hpx_1_9_1: =========================== -|hpx| V1.9.1 (TBD) +|hpx| V1.9.1 (August 4, 2023) =========================== General changes From ed4ca47ae3432404b43ac7797477a0f379390e26 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 5 Aug 2023 11:44:22 -0500 Subject: [PATCH 165/333] Fixing linker problem on FreeBSD --- .../include/hpx/hpx_init_impl.hpp | 19 ++++++++++--------- .../include/hpx/hpx_start_impl.hpp | 19 ++++++++++--------- .../hpx/init_runtime/detail/run_or_start.hpp | 4 ++-- libs/full/init_runtime/src/hpx_init.cpp | 10 +++++----- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/libs/full/init_runtime/include/hpx/hpx_init_impl.hpp b/libs/full/init_runtime/include/hpx/hpx_init_impl.hpp index cc3570264c40..5fc0817d44d3 100644 --- a/libs/full/init_runtime/include/hpx/hpx_init_impl.hpp +++ b/libs/full/init_runtime/include/hpx/hpx_init_impl.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2018 Mikael Simberg -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // Copyright (c) 2011 Bryce Lelbach // // SPDX-License-Identifier: BSL-1.0 @@ -9,6 +9,7 @@ #pragma once #include +#include #include #include #include @@ -26,8 +27,7 @@ #include #if defined(__FreeBSD__) -extern HPX_EXPORT char** freebsd_environ; -extern char** environ; +extern char** environ; // available in executables only #endif #if defined(HPX_WINDOWS) && defined(HPX_HAVE_APEX) @@ -56,7 +56,8 @@ namespace hpx { inline int init(std::function f, int argc, char** argv, init_params const& params) { - return detail::init_impl(HPX_MOVE(f), argc, argv, params, HPX_PREFIX); + return detail::init_impl( + HPX_MOVE(f), argc, argv, params, HPX_PREFIX, environ); } /// \brief Main entry point for launching the HPX runtime system. @@ -71,7 +72,7 @@ namespace hpx { std::function main_f = hpx::bind_back(detail::init_helper, HPX_MOVE(f)); return detail::init_impl( - HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX); + HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX, environ); } /// \brief Main entry point for launching the HPX runtime system. @@ -82,8 +83,8 @@ namespace hpx { /// function given by \p f as a HPX thread. inline int init(int argc, char** argv, init_params const& params) { - return detail::init_impl( - hpx_startup::get_main_func(), argc, argv, params, HPX_PREFIX); + return detail::init_impl(hpx_startup::get_main_func(), argc, argv, + params, HPX_PREFIX, environ); } /// \brief Main entry point for launching the HPX runtime system. @@ -97,7 +98,7 @@ namespace hpx { { hpx::function main_f; return detail::init_impl( - HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX); + HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX, environ); } /// \brief Main entry point for launching the HPX runtime system. @@ -109,6 +110,6 @@ namespace hpx { { return detail::init_impl(hpx_startup::get_main_func(), hpx::local::detail::dummy_argc, hpx::local::detail::dummy_argv, - params, HPX_PREFIX); + params, HPX_PREFIX, environ); } } // namespace hpx diff --git a/libs/full/init_runtime/include/hpx/hpx_start_impl.hpp b/libs/full/init_runtime/include/hpx/hpx_start_impl.hpp index baebdab1b64c..c89de39cf65b 100644 --- a/libs/full/init_runtime/include/hpx/hpx_start_impl.hpp +++ b/libs/full/init_runtime/include/hpx/hpx_start_impl.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2018 Mikael Simberg -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -8,6 +8,7 @@ #pragma once #include +#include #include #include #include @@ -24,8 +25,7 @@ #include #if defined(__FreeBSD__) -extern HPX_EXPORT char** freebsd_environ; -extern char** environ; +extern char** environ; // available in executables only #endif #if defined(HPX_WINDOWS) && defined(HPX_HAVE_APEX) @@ -57,7 +57,8 @@ namespace hpx { std::function f, int argc, char** argv, init_params const& params) { - return detail::start_impl(HPX_MOVE(f), argc, argv, params, HPX_PREFIX); + return detail::start_impl( + HPX_MOVE(f), argc, argv, params, HPX_PREFIX, environ); } /// \brief Main non-blocking entry point for launching the HPX runtime system. @@ -74,7 +75,7 @@ namespace hpx { hpx::function main_f = hpx::bind_back(detail::init_helper, HPX_MOVE(f)); return detail::start_impl( - HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX); + HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX, environ); } /// \brief Main non-blocking entry point for launching the HPX runtime system. @@ -87,8 +88,8 @@ namespace hpx { /// with the runtime system's execution. inline bool start(int argc, char** argv, init_params const& params) { - return detail::start_impl( - hpx_startup::get_main_func(), argc, argv, params, HPX_PREFIX); + return detail::start_impl(hpx_startup::get_main_func(), argc, argv, + params, HPX_PREFIX, environ); } /// \brief Main non-blocking entry point for launching the HPX runtime system. @@ -104,7 +105,7 @@ namespace hpx { { hpx::function main_f; return detail::start_impl( - HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX); + HPX_MOVE(main_f), argc, argv, params, HPX_PREFIX, environ); } /// \brief Main non-blocking entry point for launching the HPX runtime system. @@ -119,6 +120,6 @@ namespace hpx { { return detail::start_impl(hpx_startup::get_main_func(), hpx::local::detail::dummy_argc, hpx::local::detail::dummy_argv, - params, HPX_PREFIX); + params, HPX_PREFIX, environ); } } // namespace hpx diff --git a/libs/full/init_runtime/include/hpx/init_runtime/detail/run_or_start.hpp b/libs/full/init_runtime/include/hpx/init_runtime/detail/run_or_start.hpp index 14279896ff1e..431d161331b7 100644 --- a/libs/full/init_runtime/include/hpx/init_runtime/detail/run_or_start.hpp +++ b/libs/full/init_runtime/include/hpx/init_runtime/detail/run_or_start.hpp @@ -22,12 +22,12 @@ namespace hpx { HPX_EXPORT int init_impl( hpx::function const& f, int argc, char** argv, init_params const& params, - char const* hpx_prefix); + char const* hpx_prefix, char** env); HPX_EXPORT bool start_impl( hpx::function const& f, int argc, char** argv, init_params const& params, - char const* hpx_prefix); + char const* hpx_prefix, char** env); } // namespace detail /// \endcond } // namespace hpx diff --git a/libs/full/init_runtime/src/hpx_init.cpp b/libs/full/init_runtime/src/hpx_init.cpp index 67222404d61e..ef823d5e0a52 100644 --- a/libs/full/init_runtime/src/hpx_init.cpp +++ b/libs/full/init_runtime/src/hpx_init.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // Copyright (c) 2017 Shoshana Jakobovits // Copyright (c) 2010-2011 Phillip LeBlanc, Dylan Stark // Copyright (c) 2011 Bryce Lelbach @@ -109,7 +109,7 @@ namespace hpx::detail { int init_impl( hpx::function const& f, int argc, char** argv, init_params const& params, - char const* hpx_prefix) + char const* hpx_prefix, [[maybe_unused]] char** env) { if (argc == 0 || argv == nullptr) { @@ -127,7 +127,7 @@ namespace hpx::detail { #endif util::set_hpx_prefix(hpx_prefix); #if defined(__FreeBSD__) - freebsd_environ = environ; + freebsd_environ = env; #endif // set a handler for std::abort, std::at_quick_exit, and std::atexit std::signal(SIGABRT, detail::on_abort); @@ -142,7 +142,7 @@ namespace hpx::detail { bool start_impl( hpx::function const& f, int argc, char** argv, init_params const& params, - char const* hpx_prefix) + char const* hpx_prefix, [[maybe_unused]] char** env) { if (argc == 0 || argv == nullptr) { @@ -160,7 +160,7 @@ namespace hpx::detail { #endif util::set_hpx_prefix(hpx_prefix); #if defined(__FreeBSD__) - freebsd_environ = environ; + freebsd_environ = env; #endif // set a handler for std::abort, std::at_quick_exit, and std::atexit std::signal(SIGABRT, detail::on_abort); From 4ed7564f504a4a4765b8878d769bd9ba42767033 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 10 Aug 2023 10:52:40 -0500 Subject: [PATCH 166/333] Generate git tag if needed but it is not available --- CMakeLists.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d32ef46c522..9363c522969e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2223,6 +2223,21 @@ hpx_add_config_define(HPX_HAVE_GIT_COMMIT "\"${HPX_WITH_GIT_COMMIT}\"") hpx_include(SetOutputPaths) +# ############################################################################## +# Fixing git tag, if necessary +hpx_info("HPX_WITH_GIT_TAG: " ${HPX_WITH_GIT_TAG}) +hpx_info("HPX_WITH_GIT_BRANCH: " ${HPX_WITH_GIT_BRANCH}) + +if(HPX_WITH_GIT_BRANCH AND ((NOT HPX_WITH_GIT_TAG) OR "${HPX_WITH_GIT_TAG}x" + STREQUAL "x") +) + string(REGEX MATCH "^release-[0-9]+.[0-9]+.X$" matched ${HPX_WITH_GIT_BRANCH}) + if(matched) + set(HPX_WITH_GIT_TAG "v${HPX_VERSION}") + hpx_info("Setting HPX_WITH_GIT_TAG to: " "${HPX_WITH_GIT_TAG}") + endif() +endif() + # ############################################################################## # Add custom targets for tests # ############################################################################## From cde5d8fca2219336d69d5a85c139908ff1b4b6f5 Mon Sep 17 00:00:00 2001 From: Panos Date: Fri, 11 Aug 2023 16:06:50 +0300 Subject: [PATCH 167/333] Ensure hpx_main is a proper thread_function --- libs/core/runtime_local/src/runtime_local.cpp | 12 +++++++----- .../runtime_distributed/src/runtime_distributed.cpp | 13 ++++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/libs/core/runtime_local/src/runtime_local.cpp b/libs/core/runtime_local/src/runtime_local.cpp index 2aa33b83e009..0bc3eb1e47b8 100644 --- a/libs/core/runtime_local/src/runtime_local.cpp +++ b/libs/core/runtime_local/src/runtime_local.cpp @@ -1460,11 +1460,13 @@ namespace hpx { lbt_ << "(1st stage) runtime::start: launching run_helper " "HPX thread"; - threads::thread_init_data data( - hpx::bind(&runtime::run_helper, this, func, std::ref(result_), true, - &detail::handle_print_bind), - "run_helper", threads::thread_priority::normal, - threads::thread_schedule_hint(0), threads::thread_stacksize::large); + threads::thread_function_type thread_func = + threads::make_thread_function(hpx::bind(&runtime::run_helper, this, + func, std::ref(result_), true, &detail::handle_print_bind)); + + threads::thread_init_data data(HPX_MOVE(thread_func), "run_helper", + threads::thread_priority::normal, threads::thread_schedule_hint(0), + threads::thread_stacksize::large); this->runtime::starting(); threads::thread_id_ref_type id = threads::invalid_thread_id; diff --git a/libs/full/runtime_distributed/src/runtime_distributed.cpp b/libs/full/runtime_distributed/src/runtime_distributed.cpp index 2f7731914334..4238b8d869f8 100644 --- a/libs/full/runtime_distributed/src/runtime_distributed.cpp +++ b/libs/full/runtime_distributed/src/runtime_distributed.cpp @@ -472,11 +472,14 @@ namespace hpx { lbt_ << "(1st stage) runtime_distributed::start: launching " "run_helper HPX thread"; - threads::thread_init_data data( - hpx::bind(&runtime_distributed::run_helper, this, func, - std::ref(result_)), - "run_helper", threads::thread_priority::normal, - threads::thread_schedule_hint(0), threads::thread_stacksize::large); + threads::thread_function_type thread_func = + threads::make_thread_function( + hpx::bind(&runtime_distributed::run_helper, this, func, + std::ref(result_))); + + threads::thread_init_data data(HPX_MOVE(thread_func), "run_helper", + threads::thread_priority::normal, threads::thread_schedule_hint(0), + threads::thread_stacksize::large); this->runtime::starting(); threads::thread_id_ref_type id = threads::invalid_thread_id; From 8b421553ffe3deb4d1a5184932dff8cb1871a696 Mon Sep 17 00:00:00 2001 From: Panos Date: Fri, 11 Aug 2023 22:23:28 +0300 Subject: [PATCH 168/333] Add regression test for hpx_main exit callbacks --- tests/regressions/threads/CMakeLists.txt | 1 + .../threads/main_thread_exit_callbacks.cpp | 55 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 tests/regressions/threads/main_thread_exit_callbacks.cpp diff --git a/tests/regressions/threads/CMakeLists.txt b/tests/regressions/threads/CMakeLists.txt index 4970747154ce..73920cd63342 100644 --- a/tests/regressions/threads/CMakeLists.txt +++ b/tests/regressions/threads/CMakeLists.txt @@ -7,6 +7,7 @@ set(tests block_os_threads_1036 + main_thread_exit_callbacks run_as_hpx_thread_exceptions_3304 run_as_os_thread_lockup_2991 stackless_self_4155 diff --git a/tests/regressions/threads/main_thread_exit_callbacks.cpp b/tests/regressions/threads/main_thread_exit_callbacks.cpp new file mode 100644 index 000000000000..4a6451236cd0 --- /dev/null +++ b/tests/regressions/threads/main_thread_exit_callbacks.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2023 Panos Syskakis +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include + +#include +#include + +bool callback_called(false); + +int hpx_main() +{ + hpx::threads::thread_id_type id = hpx::threads::get_self_id(); + hpx::threads::add_thread_exit_callback(id, [id]() { + hpx::threads::thread_id_type id1 = hpx::threads::get_self_id(); + HPX_TEST_EQ(id1, id); + + callback_called = true; + }); + + return hpx::finalize(); +} + +int main(int argc, char* argv[]) +{ + // Test local runtime + { + hpx::init_params iparams; + iparams.mode = hpx::runtime_mode::local; + callback_called = false; + HPX_TEST_EQ_MSG(hpx::init(argc, argv, iparams), 0, + "HPX main exited with non-zero status"); + HPX_TEST(callback_called); + } + +#if defined(HPX_HAVE_DISTRIBUTED_RUNTIME) + // Test distributed runtime + { + hpx::init_params iparams; + iparams.mode = hpx::runtime_mode::console; + callback_called = false; + HPX_TEST_EQ_MSG(hpx::init(argc, argv, iparams), 0, + "HPX main exited with non-zero status"); + + HPX_TEST(callback_called); + } +#endif + return hpx::util::report_errors(); +} From 3f969e1cd77032c2fbc868cea8633e0b9afd79b2 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 18 Aug 2023 17:49:26 -0500 Subject: [PATCH 169/333] Don't register components if in local mode --- libs/full/init_runtime/src/hpx_init.cpp | 62 ++++++++++++++----------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/libs/full/init_runtime/src/hpx_init.cpp b/libs/full/init_runtime/src/hpx_init.cpp index ef823d5e0a52..e4c88961c70a 100644 --- a/libs/full/init_runtime/src/hpx_init.cpp +++ b/libs/full/init_runtime/src/hpx_init.cpp @@ -130,11 +130,17 @@ namespace hpx::detail { freebsd_environ = env; #endif // set a handler for std::abort, std::at_quick_exit, and std::atexit - std::signal(SIGABRT, detail::on_abort); - std::atexit(detail::on_exit); + [[maybe_unused]] auto const prev_signal = + std::signal(SIGABRT, detail::on_abort); + HPX_ASSERT(prev_signal != SIG_ERR); + + [[maybe_unused]] auto const ret_at_exit = std::atexit(detail::on_exit); + HPX_ASSERT(ret_at_exit == 0); + #if defined(HPX_HAVE_CXX11_STD_QUICK_EXIT) - [[maybe_unused]] int const ret = std::at_quick_exit(detail::on_exit); - HPX_ASSERT(ret == 0); + [[maybe_unused]] auto const ret_at_quick_exit = + std::at_quick_exit(detail::on_exit); + HPX_ASSERT(ret_at_quick_exit == 0); #endif return detail::run_or_start(f, argc, argv, params, true); } @@ -163,11 +169,17 @@ namespace hpx::detail { freebsd_environ = env; #endif // set a handler for std::abort, std::at_quick_exit, and std::atexit - std::signal(SIGABRT, detail::on_abort); - std::atexit(detail::on_exit); + [[maybe_unused]] auto const prev_signal = + std::signal(SIGABRT, detail::on_abort); + HPX_ASSERT(prev_signal != SIG_ERR); + + [[maybe_unused]] auto const ret_atexit = std::atexit(detail::on_exit); + HPX_ASSERT(ret_atexit == 0); + #if defined(HPX_HAVE_CXX11_STD_QUICK_EXIT) - [[maybe_unused]] int const ret = std::at_quick_exit(detail::on_exit); - HPX_ASSERT(ret == 0); + [[maybe_unused]] auto const ret_at_quick_exit = + std::at_quick_exit(detail::on_exit); + HPX_ASSERT(ret_at_quick_exit == 0); #endif return 0 == detail::run_or_start(f, argc, argv, params, false); } @@ -870,15 +882,15 @@ namespace hpx { hpx_startup::user_main_config(params.cfg), f}; #endif + std::vector< + std::shared_ptr> + component_registries; + // scope exception handling to resource partitioner initialization // any exception thrown during run_or_start below are handled // separately try { - std::vector< - std::shared_ptr> - component_registries; - result = cmdline.call( params.desc_cmdline, argc, argv, component_registries); @@ -903,13 +915,6 @@ namespace hpx { hpx::resource::detail::make_partitioner( params.rp_mode, cmdline.rtcfg_, affinity_data); - for (auto& registry : component_registries) - { - hpx::register_startup_function([registry]() { - registry->register_component_type(); - }); - } - activate_global_options(cmdline, argc, argv); // check whether HPX should be exited at this point @@ -964,6 +969,13 @@ namespace hpx { default: { #if defined(HPX_HAVE_DISTRIBUTED_RUNTIME) + for (auto const& registry : component_registries) + { + hpx::register_startup_function([registry]() { + registry->register_component_type(); + }); + } + LPROGRESS_ << "creating distributed runtime"; rt.reset(new hpx::runtime_distributed(cmdline.rtcfg_, &hpx::detail::pre_main, &hpx::detail::post_main)); @@ -1011,7 +1023,7 @@ namespace hpx { } catch (hpx::util::bad_lexical_cast const&) { - ; // do nothing + // do nothing } } return default_; @@ -1112,9 +1124,8 @@ namespace hpx { if (std::abs(shutdown_timeout + 1.0) < 1e-16) shutdown_timeout = detail::get_option("hpx.shutdown_timeout", -1.0); - components::server::runtime_support* p = - static_cast( - get_runtime_distributed().get_runtime_support_lva()); + auto* p = static_cast( + get_runtime_distributed().get_runtime_support_lva()); if (nullptr == p) { @@ -1147,9 +1158,8 @@ namespace hpx { } #if defined(HPX_HAVE_DISTRIBUTED_RUNTIME) - components::server::runtime_support* p = - static_cast( - get_runtime_distributed().get_runtime_support_lva()); + auto* p = static_cast( + get_runtime_distributed().get_runtime_support_lva()); if (nullptr == p) { From d8223bba820037a2123852a446b7e51c84c25908 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 10 Aug 2023 19:35:12 -0500 Subject: [PATCH 170/333] Fixing cyclic dependencies - Fixing cyclic dependencies related to parcelport symbols - Fixing cyclic dependencies in naming and agas modules - Adding linux static github builder - flyby: fixing compression filter linking order --- .github/workflows/linux_release_static.yml | 47 ++++++++++++++ cmake/FindHwloc.cmake | 5 +- cmake/HPX_AddModule.cmake | 11 ++-- cmake/HPX_ParcelPorts.cmake | 26 +------- .../binary_filter/bzip2/CMakeLists.txt | 7 ++- .../binary_filter/snappy/CMakeLists.txt | 7 ++- .../binary_filter/zlib/CMakeLists.txt | 7 ++- libs/CMakeLists.txt | 13 ++-- libs/core/CMakeLists.txt | 5 ++ libs/core/include_local/CMakeLists.txt | 20 +++--- libs/core/init_runtime_local/CMakeLists.txt | 4 +- libs/create_module_skeleton.py | 11 ++-- libs/full/CMakeLists.txt | 12 +++- libs/full/agas/src/route.cpp | 28 ++++++--- libs/full/agas_base/CMakeLists.txt | 1 + .../agas_base/include/hpx/agas_base/route.hpp | 21 +++++++ .../agas_base/server/primary_namespace.hpp | 8 ++- .../src/server/primary_namespace_server.cpp | 14 +++++ libs/full/init_runtime/CMakeLists.txt | 4 +- libs/full/init_runtime/src/hpx_init.cpp | 9 +++ libs/full/modules.rst | 20 +++++- .../include/hpx/naming/credit_handling.hpp | 5 +- libs/full/naming/src/credit_handling.cpp | 62 +++++++++++-------- .../include/hpx/naming_base/id_type.hpp | 45 +++++++------- libs/full/naming_base/src/id_type.cpp | 26 ++++---- libs/full/parcelports/CMakeLists.txt | 58 +++++++++++++++++ .../templates/static_parcelports.hpp.in | 17 +++-- .../hpx/parcelports/init_all_parcelports.hpp | 19 ++++++ .../parcelports/src/static_parcelports.cpp | 35 +++++++++++ libs/full/parcelset/CMakeLists.txt | 1 + .../hpx/parcelset/init_parcelports.hpp | 22 +++++++ libs/full/parcelset/src/parcelhandler.cpp | 2 +- .../plugin_factories/parcelport_factory.hpp | 2 +- 33 files changed, 430 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/linux_release_static.yml create mode 100644 libs/full/agas_base/include/hpx/agas_base/route.hpp create mode 100644 libs/full/parcelports/CMakeLists.txt rename {cmake => libs/full/parcelports/cmake}/templates/static_parcelports.hpp.in (62%) create mode 100644 libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp create mode 100644 libs/full/parcelports/src/static_parcelports.cpp create mode 100644 libs/full/parcelset/include/hpx/parcelset/init_parcelports.hpp diff --git a/.github/workflows/linux_release_static.yml b/.github/workflows/linux_release_static.yml new file mode 100644 index 000000000000..6ff33a5fb9d3 --- /dev/null +++ b/.github/workflows/linux_release_static.yml @@ -0,0 +1,47 @@ +# Copyright (c) 2020 ETH Zurich +# Copyright (c) 2023 The STE||AR Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +name: Linux CI (Release, Static) + +on: [pull_request] + +jobs: + build: + runs-on: ubuntu-latest + container: stellargroup/build_env:16 + + steps: + - uses: actions/checkout@v3 + - name: Configure + shell: bash + run: | + cmake \ + . \ + -Bbuild \ + -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DHPX_WITH_STATIC_LINKING=ON \ + -DHPX_WITH_MALLOC=system \ + -DHPX_WITH_FETCH_ASIO=ON \ + -DHPX_WITH_EXAMPLES=ON \ + -DHPX_WITH_TESTS=ON \ + -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ + -DHPX_WITH_CHECK_MODULE_DEPENDENCIES=On + - name: Build + shell: bash + run: | + cmake --build build --target all + cmake --build build --target examples + - name: Test + shell: bash + run: | + cd build + ctest \ + --output-on-failure \ + --tests-regex tests.examples \ + --exclude-regex tests.examples.transpose.transpose_block_numa \ + --exclude-regex tests.examples.quickstart.distributed.tcp.custom_serialization diff --git a/cmake/FindHwloc.cmake b/cmake/FindHwloc.cmake index cdede78fb68b..89dadf132013 100644 --- a/cmake/FindHwloc.cmake +++ b/cmake/FindHwloc.cmake @@ -26,7 +26,7 @@ if(NOT TARGET Hwloc::hwloc) find_library( HWLOC_LIBRARY - NAMES hwloc libhwloc + NAMES libhwloc.so hwloc HINTS ${HWLOC_ROOT} ENV HWLOC_ROOT @@ -70,7 +70,8 @@ if(NOT TARGET Hwloc::hwloc) add_library(Hwloc::hwloc INTERFACE IMPORTED) target_include_directories(Hwloc::hwloc SYSTEM INTERFACE ${HWLOC_INCLUDE_DIR}) - target_link_libraries(Hwloc::hwloc INTERFACE ${HWLOC_LIBRARIES}) + target_link_libraries(Hwloc::hwloc INTERFACE ${HWLOC_LIBRARIES}) mark_as_advanced(HWLOC_ROOT HWLOC_LIBRARY HWLOC_INCLUDE_DIR) + endif() diff --git a/cmake/HPX_AddModule.cmake b/cmake/HPX_AddModule.cmake index 3efec41ec1f1..22510e5117be 100644 --- a/cmake/HPX_AddModule.cmake +++ b/cmake/HPX_AddModule.cmake @@ -176,10 +176,9 @@ function(add_hpx_module libname modulename) endforeach() else() foreach(file_to_generate ${${modulename}_GENERATED_HEADERS}) - if(EXISTS ${file_to_generate}) - hpx_warn("Removing zombie generated header: ${file_to_generate}") - file(REMOVE ${file_to_generate}) - endif() + set(generated_headers ${generated_headers} + ${generated_file_base}/${file_to_generate} + ) endforeach() endif() endif() @@ -284,8 +283,8 @@ function(add_hpx_module libname modulename) list(FIND _hpx_${libname}_modules ${dep} dep_index) if(${dep_index} EQUAL -1) hpx_error( - "The module ${dep} should not be be listed in MODULE_DEPENDENCIES " - "for module hpx_${modulename}" + "The module hpx_${dep} should not be be listed in MODULE_DEPENDENCIES " + "for '${libname}' module hpx_${modulename}" ) endif() endif() diff --git a/cmake/HPX_ParcelPorts.cmake b/cmake/HPX_ParcelPorts.cmake index 0c7aab45bb31..8dbcafc6df1c 100644 --- a/cmake/HPX_ParcelPorts.cmake +++ b/cmake/HPX_ParcelPorts.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Hartmut Kaiser +# Copyright (c) 2021-2023 Hartmut Kaiser # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -9,26 +9,6 @@ if(HPX_WITH_DISTRIBUTED_RUNTIME) ${HPX_STATIC_PARCELPORT_PLUGINS} ) - # add_subdirectory is called before to insure HPX_STATIC_PARCELPORT_PLUGINS - # cache variable is filled - set(_parcelport_export) - set(_parcelport_init) - - foreach(parcelport ${HPX_STATIC_PARCELPORT_PLUGINS}) - set(_parcelport_export - "${_parcelport_export}HPX_EXPORT hpx::plugins::parcelport_factory_base *${parcelport}_factory_init(\n" - ) - set(_parcelport_export - "${_parcelport_export} std::vector& factories);\n" - ) - set(_parcelport_init - "${_parcelport_init} ${parcelport}_factory_init(factories);\n" - ) - endforeach() - - configure_file( - "${PROJECT_SOURCE_DIR}/cmake/templates/static_parcelports.hpp.in" - "${PROJECT_BINARY_DIR}/libs/full/parcelset/include/hpx/parcelset/static_parcelports.hpp" - @ONLY - ) + # handle parcelports module to create proper dependencies + add_subdirectory(libs/full/parcelports) endif() diff --git a/components/parcel_plugins/binary_filter/bzip2/CMakeLists.txt b/components/parcel_plugins/binary_filter/bzip2/CMakeLists.txt index cd0ed71697ec..33d173ab75af 100644 --- a/components/parcel_plugins/binary_filter/bzip2/CMakeLists.txt +++ b/components/parcel_plugins/binary_filter/bzip2/CMakeLists.txt @@ -30,14 +30,15 @@ add_hpx_library( "hpx/binary_filter/bzip2_serialization_filter.hpp" "hpx/binary_filter/bzip2_serialization_filter_registration.hpp" PREPEND_HEADER_ROOT INSTALL_HEADERS - FOLDER "Core/Plugins/Compression" - DEPENDENCIES ${BZIP2_LIBRARIES} ${HPX_WITH_UNITY_BUILD_OPTION} + FOLDER "Core/Plugins/Compression" ${HPX_WITH_UNITY_BUILD_OPTION} ) target_include_directories( compression_bzip2 SYSTEM PRIVATE ${BZIP2_INCLUDE_DIR} ) -target_link_libraries(compression_bzip2 PUBLIC Boost::iostreams) +target_link_libraries( + compression_bzip2 PUBLIC Boost::iostreams ${BZIP2_LIBRARIES} +) add_hpx_pseudo_dependencies( components.parcel_plugins.binary_filter.bzip2 compression_bzip2 diff --git a/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt b/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt index ebc41f37b119..6550e27e3ca1 100644 --- a/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt +++ b/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt @@ -30,8 +30,7 @@ add_hpx_library( "hpx/binary_filter/snappy_serialization_filter.hpp" "hpx/binary_filter/snappy_serialization_filter_registration.hpp" PREPEND_HEADER_ROOT INSTALL_HEADERS - FOLDER "Core/Plugins/Compression" - DEPENDENCIES ${SNAPPY_LIBRARY} ${HPX_WITH_UNITY_BUILD_OPTION} + FOLDER "Core/Plugins/Compression" ${HPX_WITH_UNITY_BUILD_OPTION} ) target_include_directories( @@ -39,7 +38,9 @@ target_include_directories( ) target_link_directories(compression_snappy PRIVATE ${SNAPPY_LIBRARY_DIR}) -target_link_libraries(compression_snappy PUBLIC Boost::iostreams) +target_link_libraries( + compression_snappy PUBLIC Boost::iostreams ${SNAPPY_LIBRARY} +) add_hpx_pseudo_dependencies( components.parcel_plugins.binary_filter.snappy compression_snappy diff --git a/components/parcel_plugins/binary_filter/zlib/CMakeLists.txt b/components/parcel_plugins/binary_filter/zlib/CMakeLists.txt index 1b324f17e1c7..6dba673e7225 100644 --- a/components/parcel_plugins/binary_filter/zlib/CMakeLists.txt +++ b/components/parcel_plugins/binary_filter/zlib/CMakeLists.txt @@ -30,12 +30,13 @@ add_hpx_library( "hpx/binary_filter/zlib_serialization_filter.hpp" "hpx/binary_filter/zlib_serialization_filter_registration.hpp" PREPEND_HEADER_ROOT INSTALL_HEADERS - FOLDER "Core/Plugins/Compression" - DEPENDENCIES ${ZLIB_LIBRARIES} ${HPX_WITH_UNITY_BUILD_OPTION} + FOLDER "Core/Plugins/Compression" ${HPX_WITH_UNITY_BUILD_OPTION} ) target_include_directories(compression_zlib SYSTEM PRIVATE ${ZLIB_INCLUDE_DIRS}) -target_link_libraries(compression_zlib PUBLIC Boost::iostreams) +target_link_libraries( + compression_zlib PUBLIC Boost::iostreams ${ZLIB_LIBRARIES} +) add_hpx_pseudo_dependencies( components.parcel_plugins.binary_filter.zlib compression_zlib diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 0f37a7ed1d71..b09723ad28dc 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -100,13 +100,7 @@ target_include_directories( ) target_link_libraries(hpx_full PUBLIC hpx_base_libraries) - -if((NOT HPX_WITH_STATIC_LINKING) AND (("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - OR (APPLE)) -) - set_target_properties(hpx_full PROPERTIES POSITION_INDEPENDENT_CODE ON) -endif() - +set_target_properties(hpx_full PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_definitions(hpx_full PRIVATE HPX_COMPONENT_NAME=hpx HPX_EXPORTS) # ############################################################################## @@ -340,6 +334,11 @@ foreach(lib ${HPX_LIBS}) endif() add_subdirectory(${lib}) + + set(_hpx_${lib}_modules + ${_hpx_${lib}_modules} + PARENT_SCOPE + ) endforeach() # add_hpx_module populates HPX_ENABLED_MODULES diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index ddd39a0a0baa..3e6550e93f25 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -94,3 +94,8 @@ hpx_info(" Configuring libhpx_core modules:") foreach(module ${_hpx_core_modules}) add_subdirectory(${module}) endforeach() + +set(_hpx_core_modules + ${_hpx_core_modules} + PARENT_SCOPE +) diff --git a/libs/core/include_local/CMakeLists.txt b/libs/core/include_local/CMakeLists.txt index 05875d8e45f2..a3036f1d7471 100644 --- a/libs/core/include_local/CMakeLists.txt +++ b/libs/core/include_local/CMakeLists.txt @@ -32,15 +32,17 @@ set(include_local_headers hpx/experimental/task_group.hpp ) -set(generated_include_local_headers - hpx/algorithm.hpp - hpx/barrier.hpp - hpx/channel.hpp - hpx/compute.hpp - hpx/future.hpp - hpx/latch.hpp - hpx/runtime.hpp -) +if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) + set(generated_include_local_headers + hpx/algorithm.hpp + hpx/barrier.hpp + hpx/channel.hpp + hpx/compute.hpp + hpx/future.hpp + hpx/latch.hpp + hpx/runtime.hpp + ) +endif() # The headers in hpx/local/ were deprecated in HPX V1.10.0 # cmake-format: off diff --git a/libs/core/init_runtime_local/CMakeLists.txt b/libs/core/init_runtime_local/CMakeLists.txt index c106a9f1fefd..edbd3da3aa73 100644 --- a/libs/core/init_runtime_local/CMakeLists.txt +++ b/libs/core/init_runtime_local/CMakeLists.txt @@ -12,7 +12,9 @@ set(init_runtime_local_headers hpx/init_runtime_local/detail/init_logging.hpp hpx/init_runtime_local/init_runtime_local.hpp ) -set(generated_init_runtime_local_headers hpx/init.hpp) +if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) + set(generated_init_runtime_local_headers hpx/init.hpp) +endif() set(init_runtime_local_sources init_logging.cpp init_runtime_local.cpp) diff --git a/libs/create_module_skeleton.py b/libs/create_module_skeleton.py index 9ca789c583f1..e254f101d4a1 100755 --- a/libs/create_module_skeleton.py +++ b/libs/create_module_skeleton.py @@ -2,6 +2,7 @@ ''' Copyright (c) 2019-2020 ETH Zurich Copyright (c) 2018 Thomas Heller +Copyright (c) 2022-2023 Hartmut Kaiser SPDX-License-Identifier: BSL-1.0 Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -24,14 +25,14 @@ header_str = '=' * len(module_name) -cmake_root_header = f'''# Copyright (c) 2019-2021 The STE||AR-Group +cmake_root_header = f'''# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) ''' -cmake_header = f'''# Copyright (c) 2020-2021 The STE||AR-Group +cmake_header = f'''# Copyright (c) 2020-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -40,7 +41,7 @@ readme_template = f''' .. - Copyright (c) 2020-2021 The STE||AR-Group + Copyright (c) 2020-2023 The STE||AR-Group SPDX-License-Identifier: BSL-1.0 Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -57,7 +58,7 @@ ''' index_rst = f'''.. - Copyright (c) 2020-2021 The STE||AR-Group + Copyright (c) 2020-2023 The STE||AR-Group SPDX-License-Identifier: BSL-1.0 Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -255,7 +256,7 @@ def mkdir(path): header_underline_str = '=' * len(header_name_str) modules_rst = f'''.. - Copyright (c) 2018-2021 The STE||AR-Group + Copyright (c) 2018-2023 The STE||AR-Group SPDX-License-Identifier: BSL-1.0 Distributed under the Boost Software License, Version 1.0. (See accompanying diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index c63bbce9c8f0..e4a7eb24b923 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -36,6 +36,7 @@ set(_hpx_full_modules parcelport_libfabric parcelport_mpi parcelport_tcp + parcelports parcelset parcelset_base performance_counters @@ -52,5 +53,14 @@ hpx_info("") hpx_info(" Configuring libhpx modules:") foreach(module ${_hpx_full_modules}) - add_subdirectory(${module}) + # the parcelports module is handled explicitly after all other modules have + # been processed (see HPX_ParcelPorts.cmake) + if(NOT (${module} STREQUAL "parcelports")) + add_subdirectory(${module}) + endif() endforeach() + +set(_hpx_full_modules + ${_hpx_full_modules} + PARENT_SCOPE +) diff --git a/libs/full/agas/src/route.cpp b/libs/full/agas/src/route.cpp index 3fde16520a9f..a9e237b7fab6 100644 --- a/libs/full/agas/src/route.cpp +++ b/libs/full/agas/src/route.cpp @@ -10,6 +10,7 @@ #if defined(HPX_HAVE_NETWORKING) #include #include +#include #include #include #include @@ -41,32 +42,28 @@ HPX_PLAIN_ACTION_ID(hpx::detail::update_agas_cache, update_agas_cache_action, namespace hpx::agas::server { - void primary_namespace::route(parcelset::parcel&& p) + void route_impl(primary_namespace& server, parcelset::parcel&& p) { - LPT_(debug).format("primary_namespace::route: {}", p.parcel_id()); - - util::scoped_timer> update( - counter_data_.route_.time_, counter_data_.route_.enabled_); - counter_data_.increment_route_count(); + LPT_(debug).format("agas::server::route_impl: {}", p.parcel_id()); naming::gid_type const& gid = p.destination(); naming::address& addr = p.addr(); - resolved_type cache_address; + primary_namespace::resolved_type cache_address; // resolve destination addresses, we should be able to resolve all of // them, otherwise it's an error { - std::unique_lock l(mutex_); + std::unique_lock l(server.mutex()); error_code& ec = throws; // wait for any migration to be completed if (naming::detail::is_migratable(gid)) { - wait_for_migration_locked(l, gid, ec); + server.wait_for_migration_locked(l, gid, ec); } - cache_address = resolve_gid_locked(l, gid, ec); + cache_address = server.resolve_gid_locked(l, gid, ec); if (ec || hpx::get<0>(cache_address) == naming::invalid_gid) { @@ -133,6 +130,17 @@ namespace hpx::agas::server { } } } + + /////////////////////////////////////////////////////////////////////////// + struct init_route_function + { + init_route_function() + { + server::route = &route_impl; + } + }; + + init_route_function init; } // namespace hpx::agas::server #endif diff --git a/libs/full/agas_base/CMakeLists.txt b/libs/full/agas_base/CMakeLists.txt index 76288c7dbbb6..8fb09230926a 100644 --- a/libs/full/agas_base/CMakeLists.txt +++ b/libs/full/agas_base/CMakeLists.txt @@ -20,6 +20,7 @@ set(agas_base_headers hpx/agas_base/gva.hpp hpx/agas_base/locality_namespace.hpp hpx/agas_base/primary_namespace.hpp + hpx/agas_base/route.hpp hpx/agas_base/server/component_namespace.hpp hpx/agas_base/server/locality_namespace.hpp hpx/agas_base/server/primary_namespace.hpp diff --git a/libs/full/agas_base/include/hpx/agas_base/route.hpp b/libs/full/agas_base/include/hpx/agas_base/route.hpp new file mode 100644 index 000000000000..ec3b75eb42d6 --- /dev/null +++ b/libs/full/agas_base/include/hpx/agas_base/route.hpp @@ -0,0 +1,21 @@ +// Copyright (c) 2007-2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) +#include +#include + +namespace hpx::agas::server { + + extern HPX_EXPORT void (*route)( + primary_namespace& server, parcelset::parcel&& p); +} // namespace hpx::agas::server + +#endif diff --git a/libs/full/agas_base/include/hpx/agas_base/server/primary_namespace.hpp b/libs/full/agas_base/include/hpx/agas_base/server/primary_namespace.hpp index 0c39dc21db0b..8d0489e131a2 100644 --- a/libs/full/agas_base/include/hpx/agas_base/server/primary_namespace.hpp +++ b/libs/full/agas_base/include/hpx/agas_base/server/primary_namespace.hpp @@ -126,6 +126,11 @@ namespace hpx::agas::server { using resolved_type = hpx::tuple; + mutex_type& mutex() + { + return mutex_; + } + private: // REVIEW: Separate mutexes might reduce contention here. This has to be // investigated carefully. @@ -241,6 +246,7 @@ namespace hpx::agas::server { char const* func_name); #endif + public: // helper function void wait_for_migration_locked(std::unique_lock& l, naming::gid_type const& id, error_code& ec); @@ -295,10 +301,10 @@ namespace hpx::agas::server { std::pair allocate( std::uint64_t count); - private: resolved_type resolve_gid_locked(std::unique_lock& l, naming::gid_type const& gid, error_code& ec); + private: resolved_type resolve_gid_locked_non_local( std::unique_lock& l, naming::gid_type const& gid, error_code& ec); diff --git a/libs/full/agas_base/src/server/primary_namespace_server.cpp b/libs/full/agas_base/src/server/primary_namespace_server.cpp index f91819e3917e..a0a56c2f2e39 100644 --- a/libs/full/agas_base/src/server/primary_namespace_server.cpp +++ b/libs/full/agas_base/src/server/primary_namespace_server.cpp @@ -7,6 +7,7 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include +#include #include #include #include @@ -1119,6 +1120,19 @@ namespace hpx::agas::server { return resolved_type(naming::invalid_gid, gva(), naming::invalid_gid); } +#if defined(HPX_HAVE_NETWORKING) + void (*route)(primary_namespace& server, parcelset::parcel&& p) = nullptr; + + void primary_namespace::route(parcelset::parcel&& p) + { + util::scoped_timer> update( + counter_data_.route_.time_, counter_data_.route_.enabled_); + counter_data_.increment_route_count(); + + (*server::route)(*this, HPX_MOVE(p)); + } +#endif + // access current counter values std::int64_t primary_namespace::counter_data::get_bind_gid_count(bool reset) { diff --git a/libs/full/init_runtime/CMakeLists.txt b/libs/full/init_runtime/CMakeLists.txt index 7b3edd57771c..71307fbbc85f 100644 --- a/libs/full/init_runtime/CMakeLists.txt +++ b/libs/full/init_runtime/CMakeLists.txt @@ -31,8 +31,8 @@ if(HPX_WITH_DISTRIBUTED_RUNTIME) set(init_runtime_sources ${init_runtime_sources} pre_main.cpp) set(init_runtime_optional_module_dependencies - hpx_async_distributed hpx_collectives hpx_naming hpx_performance_counters - hpx_runtime_distributed + hpx_async_distributed hpx_collectives hpx_naming hpx_parcelports + hpx_performance_counters hpx_runtime_distributed ) endif() diff --git a/libs/full/init_runtime/src/hpx_init.cpp b/libs/full/init_runtime/src/hpx_init.cpp index e4c88961c70a..cb0fc1eab038 100644 --- a/libs/full/init_runtime/src/hpx_init.cpp +++ b/libs/full/init_runtime/src/hpx_init.cpp @@ -67,6 +67,7 @@ #include #include #if defined(HPX_HAVE_NETWORKING) +#include #include #include #endif @@ -124,6 +125,10 @@ namespace hpx::detail { // application apex::version(); #endif +#endif +#if defined(HPX_HAVE_NETWORKING) + // force linking parcelports + hpx::parcelset::init_all_parcelports(); #endif util::set_hpx_prefix(hpx_prefix); #if defined(__FreeBSD__) @@ -163,6 +168,10 @@ namespace hpx::detail { // application apex::version(); #endif +#endif +#if defined(HPX_HAVE_NETWORKING) + // force linking parcelports + hpx::parcelset::init_all_parcelports(); #endif util::set_hpx_prefix(hpx_prefix); #if defined(__FreeBSD__) diff --git a/libs/full/modules.rst b/libs/full/modules.rst index 7d41a956c030..0f025215b8a5 100644 --- a/libs/full/modules.rst +++ b/libs/full/modules.rst @@ -14,20 +14,36 @@ Main |hpx| modules .. toctree:: :maxdepth: 2 + /libs/full/actions/docs/index.rst + /libs/full/actions_base/docs/index.rst + /libs/full/agas/docs/index.rst /libs/full/agas_base/docs/index.rst + /libs/full/async_colocated/docs/index.rst /libs/full/async_distributed/docs/index.rst /libs/full/checkpoint/docs/index.rst /libs/full/checkpoint_base/docs/index.rst /libs/full/collectives/docs/index.rst /libs/full/command_line_handling/docs/index.rst + /libs/full/components/docs/index.rst + /libs/full/components_base/docs/index.rst /libs/full/compute/docs/index.rst + /libs/full/distribution_policies/docs/index.rst /libs/full/executors_distributed/docs/index.rst /libs/full/include/docs/index.rst - /libs/full/lci_base/docs/index.rst + /libs/full/init_runtime/docs/index.rst /libs/full/lcos_distributed/docs/index.rst + /libs/full/naming/docs/index.rst /libs/full/naming_base/docs/index.rst + /libs/full/parcelport_lci/docs/index.rst + /libs/full/parcelport_libfabric/docs/index.rst + /libs/full/parcelport_mpi/docs/index.rst + /libs/full/parcelport_tcp/docs/index.rst + /libs/full/parcelset/docs/index.rst + /libs/full/parcelset_base/docs/index.rst /libs/full/performance_counters/docs/index.rst + /libs/full/plugin_factories/docs/index.rst /libs/full/resiliency_distributed/docs/index.rst + /libs/full/runtime_components/docs/index.rst + /libs/full/runtime_distributed/docs/index.rst /libs/full/segmented_algorithms/docs/index.rst /libs/full/statistics/docs/index.rst - /libs/full/version/docs/index.rst diff --git a/libs/full/naming/include/hpx/naming/credit_handling.hpp b/libs/full/naming/include/hpx/naming/credit_handling.hpp index 45750c60b92b..b083861e9ad4 100644 --- a/libs/full/naming/include/hpx/naming/credit_handling.hpp +++ b/libs/full/naming/include/hpx/naming/credit_handling.hpp @@ -39,7 +39,8 @@ namespace hpx::naming { gid_type& id, std::int64_t debit); HPX_EXPORT std::int64_t fill_credit_for_gid(gid_type& id, - std::int64_t credits = std::int64_t(HPX_GLOBALCREDIT_INITIAL)); + std::int64_t credits = static_cast( + HPX_GLOBALCREDIT_INITIAL)); /////////////////////////////////////////////////////////////////////// HPX_EXPORT gid_type move_gid(gid_type& id); @@ -58,7 +59,7 @@ namespace hpx::naming { std::unique_lock& l, gid_type& id); /////////////////////////////////////////////////////////////////////// - HPX_EXPORT void decrement_refcnt(id_type_impl* gid) noexcept; + HPX_EXPORT void decrement_refcnt(id_type_impl const* gid) noexcept; /////////////////////////////////////////////////////////////////////// // credit management (called during serialization), this function diff --git a/libs/full/naming/src/credit_handling.cpp b/libs/full/naming/src/credit_handling.cpp index cceb6175de92..230faac199e8 100644 --- a/libs/full/naming/src/credit_handling.cpp +++ b/libs/full/naming/src/credit_handling.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // Copyright (c) 2011 Bryce Lelbach // // SPDX-License-Identifier: BSL-1.0 @@ -114,7 +114,7 @@ namespace hpx::naming { /////////////////////////////////////////////////////////////////////////// namespace detail { - void decrement_refcnt(id_type_impl* p) noexcept + void decrement_refcnt(id_type_impl const* p) noexcept { // do nothing if it's too late in the game if (!get_runtime_ptr()) @@ -136,7 +136,7 @@ namespace hpx::naming { if (gid_was_split(*p) || !agas::resolve_cached(*p, addr)) { // decrement global reference count for the given gid, - std::int64_t credits = detail::get_credit_from_gid(*p); + std::int64_t const credits = get_credit_from_gid(*p); HPX_ASSERT(0 != credits); if (get_runtime_ptr()) // -V547 @@ -206,13 +206,13 @@ namespace hpx::naming { // Get the current credit for our gid. If no other concurrent // split has happened since we invoked incref below, the credit // of this gid is equal to 2, otherwise it is larger. - std::int64_t src_credit = get_credit_from_gid(gid); + std::int64_t const src_credit = get_credit_from_gid(gid); HPX_ASSERT(src_credit >= 2); - std::int64_t split_credit = + constexpr std::int64_t split_credit = static_cast(HPX_GLOBALCREDIT_INITIAL) - 2; std::int64_t new_credit = src_credit + split_credit; - std::int64_t overflow_credit = new_credit - + std::int64_t const overflow_credit = new_credit - static_cast(HPX_GLOBALCREDIT_INITIAL); HPX_ASSERT(overflow_credit >= 0); @@ -258,7 +258,8 @@ namespace hpx::naming { // credit is guaranteed to arrive only after we incremented the // credit successfully in agas. HPX_ASSERT(get_log2credit_from_gid(gid) > 0); - std::int16_t src_log2credits = get_log2credit_from_gid(gid); + std::int16_t const src_log2credits = + get_log2credit_from_gid(gid); // Credit exhaustion - we need to get more. if (src_log2credits == 1) @@ -279,11 +280,11 @@ namespace hpx::naming { // We add HPX_GLOBALCREDIT_INITIAL credits for the new gid // and HPX_GLOBALCREDIT_INITIAL - 2 for the old one. - std::int64_t new_credit = 2 * + constexpr std::int64_t new_credit = 2 * (static_cast(HPX_GLOBALCREDIT_INITIAL) - 1); - naming::gid_type new_gid = gid; // strips lock-bit + naming::gid_type const new_gid = gid; // strips lock-bit HPX_ASSERT(new_gid != invalid_gid); return agas::incref(new_gid, new_credit) .then(hpx::launch::sync, @@ -338,7 +339,7 @@ namespace hpx::naming { { HPX_ASSERT_OWNS_LOCK(l); - std::uint16_t log2credits = get_log2credit_from_gid(id); + std::int16_t const log2credits = get_log2credit_from_gid(id); HPX_ASSERT(log2credits > 0); gid_type newid = id; // strips lock-bit @@ -362,21 +363,19 @@ namespace hpx::naming { std::int64_t replenish_credits_locked( std::unique_lock& l, gid_type& gid) { - std::int64_t added_credit = 0; - HPX_ASSERT(0 == get_credit_from_gid(gid)); - added_credit = naming::detail::fill_credit_for_gid(gid); + std::int64_t const added_credit = + naming::detail::fill_credit_for_gid(gid); naming::detail::set_credit_split_mask_for_gid(gid); - gid_type unlocked_gid = gid; // strips lock-bit + gid_type const unlocked_gid = gid; // strips lock-bit - std::int64_t result = 0; + std::int64_t result; { hpx::unlock_guard> ul(l); result = agas::incref(launch::sync, unlocked_gid, added_credit); } - return result; } @@ -403,10 +402,10 @@ namespace hpx::naming { std::int64_t fill_credit_for_gid(gid_type& id, std::int64_t credits) { - std::int64_t c = get_credit_from_gid(id); + std::int64_t const c = get_credit_from_gid(id); HPX_ASSERT(c <= credits); - std::int64_t added = credits - c; + std::int64_t const added = credits - c; set_credit_for_gid(id, credits); return added; @@ -419,7 +418,7 @@ namespace hpx::naming { HPX_ASSERT(detail::gid_was_split(gid)); // decrement global reference count for the given gid, - std::int64_t credits = detail::get_credit_from_gid(gid); + std::int64_t const credits = detail::get_credit_from_gid(gid); HPX_ASSERT(0 != credits); // Fire-and-forget semantics. @@ -432,7 +431,7 @@ namespace hpx::naming { // custom deleter for managed gid_types, will be called when the last // copy of the corresponding hpx::id_type goes out of scope - void gid_managed_deleter(id_type_impl* p) noexcept + void gid_managed_deleter_impl(id_type_impl const* p) noexcept { // a credit of zero means the component is not (globally) reference // counted @@ -450,11 +449,23 @@ namespace hpx::naming { // custom deleter for unmanaged gid_types, will be called when the last // copy of the corresponding hpx::id_type goes out of scope - void gid_unmanaged_deleter(id_type_impl* p) noexcept + void gid_unmanaged_deleter_impl(id_type_impl const* p) noexcept { delete p; // delete local gid representation only } + // break cyclic dependency with naming_base + struct init_deleter_functions + { + init_deleter_functions() + { + gid_managed_deleter = &gid_managed_deleter_impl; + gid_unmanaged_deleter = &gid_unmanaged_deleter_impl; + } + }; + + init_deleter_functions init; + /////////////////////////////////////////////////////////////////////// // prepare the given id, note: this function modifies the passed id void handle_credit_splitting(serialization::output_archive& ar, @@ -551,7 +562,8 @@ namespace hpx::naming { { preprocess_gid(id_impl, ar); - gid_serialization_data data{id_impl, type}; + gid_serialization_data const data{ + static_cast(id_impl), type}; ar << data; return; } @@ -569,7 +581,7 @@ namespace hpx::naming { gid_type new_gid; if (hpx::id_type::management_type::unmanaged == type) { - new_gid = id_impl; + new_gid = static_cast(id_impl); } else if (hpx::id_type::management_type::managed_move_credit == type) { @@ -587,12 +599,12 @@ namespace hpx::naming { } #if defined(HPX_DEBUG) - auto* split_gids = ar.try_get_extra_data< + auto const* split_gids = ar.try_get_extra_data< serialization::detail::preprocess_gid_types>(); HPX_ASSERT(!split_gids || !split_gids->has_gid(id_impl)); #endif - gid_serialization_data data{new_gid, type}; + gid_serialization_data const data{new_gid, type}; ar << data; } diff --git a/libs/full/naming_base/include/hpx/naming_base/id_type.hpp b/libs/full/naming_base/include/hpx/naming_base/id_type.hpp index 08d956b5d087..cfeef69c6d83 100644 --- a/libs/full/naming_base/include/hpx/naming_base/id_type.hpp +++ b/libs/full/naming_base/include/hpx/naming_base/id_type.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -32,8 +32,10 @@ namespace hpx { HPX_EXPORT void intrusive_ptr_add_ref(id_type_impl* p) noexcept; HPX_EXPORT void intrusive_ptr_release(id_type_impl* p) noexcept; - HPX_EXPORT void gid_managed_deleter(id_type_impl* p) noexcept; - HPX_EXPORT void gid_unmanaged_deleter(id_type_impl* p) noexcept; + extern HPX_EXPORT void (*gid_managed_deleter)( + id_type_impl const* p) noexcept; + extern HPX_EXPORT void (*gid_unmanaged_deleter)( + id_type_impl const* p) noexcept; } // namespace naming::detail /////////////////////////////////////////////////////////////////////////// @@ -96,6 +98,8 @@ namespace hpx { id_type& operator=(id_type const& o) = default; id_type& operator=(id_type&& o) noexcept = default; + ~id_type() = default; + naming::gid_type& get_gid(); naming::gid_type const& get_gid() const; @@ -104,7 +108,7 @@ namespace hpx { management_type get_management_type() const noexcept; id_type& operator++(); - id_type operator++(int); + id_type operator++(int) const; explicit operator bool() const noexcept; @@ -121,11 +125,11 @@ namespace hpx { // access the internal parts of the gid std::uint64_t get_msb() const; - void set_msb(std::uint64_t msb); + void set_msb(std::uint64_t msb) const; std::uint64_t get_lsb() const; - void set_lsb(std::uint64_t lsb); - void set_lsb(void* lsb); + void set_lsb(std::uint64_t lsb) const; + void set_lsb(void* lsb) const; // Convert this id into an unmanaged one (in-place) - Use with maximum // care, or better, don't use this at all. @@ -164,14 +168,14 @@ namespace hpx { inline id_type get_id_from_locality_id( std::uint32_t locality_id) noexcept { - return id_type((std::uint64_t(locality_id) + 1) + return {(static_cast(locality_id) + 1) << naming::gid_type::locality_id_shift, - 0, id_type::management_type::unmanaged); + 0, id_type::management_type::unmanaged}; } inline std::uint32_t get_locality_id_from_id(id_type const& id) noexcept { - return std::uint32_t( + return static_cast( id.get_msb() >> naming::gid_type::locality_id_shift) - 1; } @@ -200,10 +204,13 @@ namespace hpx { struct id_type_impl : gid_type { public: - HPX_NON_COPYABLE(id_type_impl); + id_type_impl(id_type_impl const&) = delete; + id_type_impl(id_type_impl&&) = delete; + id_type_impl& operator=(id_type_impl const&) = delete; + id_type_impl& operator=(id_type_impl&&) = delete; private: - using deleter_type = void (*)(detail::id_type_impl*) noexcept; + using deleter_type = void (*)(detail::id_type_impl const*) noexcept; static deleter_type get_deleter( id_type::management_type t) noexcept; @@ -245,6 +252,8 @@ namespace hpx { { } + ~id_type_impl() = default; + constexpr id_type::management_type get_management_type() const noexcept { @@ -282,12 +291,6 @@ namespace hpx { } private: - // custom deleter for id_type_impl - friend HPX_EXPORT void gid_managed_deleter( - id_type_impl* p) noexcept; - friend HPX_EXPORT void gid_unmanaged_deleter( - id_type_impl* p) noexcept; - // reference counting friend HPX_EXPORT void intrusive_ptr_add_ref( id_type_impl* p) noexcept; @@ -389,7 +392,7 @@ namespace hpx { { return gid_->get_msb(); } - inline void id_type::set_msb(std::uint64_t msb) + inline void id_type::set_msb(std::uint64_t msb) const { gid_->set_msb(msb); } @@ -398,11 +401,11 @@ namespace hpx { { return gid_->get_lsb(); } - inline void id_type::set_lsb(std::uint64_t lsb) + inline void id_type::set_lsb(std::uint64_t lsb) const { gid_->set_lsb(lsb); } - inline void id_type::set_lsb(void* lsb) + inline void id_type::set_lsb(void* lsb) const { gid_->set_lsb(lsb); } diff --git a/libs/full/naming_base/src/id_type.cpp b/libs/full/naming_base/src/id_type.cpp index fa270c9428f8..94ef2ff94fa3 100644 --- a/libs/full/naming_base/src/id_type.cpp +++ b/libs/full/naming_base/src/id_type.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -18,6 +18,9 @@ namespace hpx { /////////////////////////////////////////////////////////////////////////// namespace naming::detail { + void (*gid_managed_deleter)(id_type_impl const* p) noexcept = nullptr; + void (*gid_unmanaged_deleter)(id_type_impl const* p) noexcept = nullptr; + util::internal_allocator id_type_impl::alloc_; id_type_impl::deleter_type id_type_impl::get_deleter( @@ -26,18 +29,18 @@ namespace hpx { switch (t) { case hpx::id_type::management_type::unmanaged: - return &detail::gid_unmanaged_deleter; + return gid_unmanaged_deleter; case hpx::id_type::management_type::managed: [[fallthrough]]; case hpx::id_type::management_type::managed_move_credit: - return &detail::gid_managed_deleter; + return gid_managed_deleter; default: HPX_ASSERT(false); // invalid management type - return &detail::gid_unmanaged_deleter; + break; } - return nullptr; + return gid_unmanaged_deleter; } // support functions for hpx::intrusive_ptr @@ -62,9 +65,10 @@ namespace hpx { return *this; } - id_type id_type::operator++(int) // post-increment + id_type id_type::operator++(int) const + // post-increment { - return id_type((*gid_)++, management_type::unmanaged); + return {(*gid_)++, management_type::unmanaged}; } // comparison is required as well @@ -138,10 +142,10 @@ namespace hpx::traits { hpx::id_type get_remote_result::call( naming::gid_type const& rhs) { - bool has_credits = naming::detail::has_credits(rhs); - return hpx::id_type(rhs, + bool const has_credits = naming::detail::has_credits(rhs); + return {rhs, has_credits ? hpx::id_type::management_type::managed : - hpx::id_type::management_type::unmanaged); + hpx::id_type::management_type::unmanaged}; } // we need to specialize this template to allow for automatic conversion of @@ -154,7 +158,7 @@ namespace hpx::traits { result.reserve(rhs.size()); for (naming::gid_type const& r : rhs) { - bool has_credits = naming::detail::has_credits(r); + bool const has_credits = naming::detail::has_credits(r); result.emplace_back(r, has_credits ? hpx::id_type::management_type::managed : hpx::id_type::management_type::unmanaged); diff --git a/libs/full/parcelports/CMakeLists.txt b/libs/full/parcelports/CMakeLists.txt new file mode 100644 index 000000000000..c775de66a0af --- /dev/null +++ b/libs/full/parcelports/CMakeLists.txt @@ -0,0 +1,58 @@ +# Copyright (c) 2019-2023 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) + return() +endif() + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") + +set(parcelports_headers hpx/parcelports/init_all_parcelports.hpp) + +set(parcelports_sources static_parcelports.cpp) + +set(parcelports_generated_headers hpx/parcelports/static_parcelports.hpp) + +set(parcelport_module_dependencies) +set(parcelport_export_declarations) +set(parcelport_init_definitions) + +foreach(parcelport ${HPX_STATIC_PARCELPORT_PLUGINS}) + set(parcelport_module_dependencies ${parcelport_module_dependencies} + hpx_${parcelport} + ) + + # generate header defining initialization functions for parcelports + set(parcelport_export_declarations + "${parcelport_export_declarations}HPX_EXPORT hpx::plugins::parcelport_factory_base *\n" + ) + set(parcelport_export_declarations + "${parcelport_export_declarations} ${parcelport}_factory_init(\n" + ) + set(parcelport_export_declarations + "${parcelport_export_declarations} std::vector& factories);\n" + ) + set(parcelport_init_definitions + "${parcelport_init_definitions} ${parcelport}_factory_init(factories);\n" + ) +endforeach() + +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/templates/static_parcelports.hpp.in" + "${CMAKE_CURRENT_BINARY_DIR}/include/hpx/parcelports/static_parcelports.hpp" + @ONLY +) + +include(HPX_AddModule) +add_hpx_module( + full parcelports + GLOBAL_HEADER_GEN OFF + GENERATED_HEADERS ${parcelports_generated_headers} + SOURCES ${parcelports_sources} + HEADERS ${parcelports_headers} + DEPENDENCIES hpx_core + MODULE_DEPENDENCIES hpx_parcelset ${parcelport_module_dependencies} +) diff --git a/cmake/templates/static_parcelports.hpp.in b/libs/full/parcelports/cmake/templates/static_parcelports.hpp.in similarity index 62% rename from cmake/templates/static_parcelports.hpp.in rename to libs/full/parcelports/cmake/templates/static_parcelports.hpp.in index 4f068f446455..92c74024334b 100644 --- a/cmake/templates/static_parcelports.hpp.in +++ b/libs/full/parcelports/cmake/templates/static_parcelports.hpp.in @@ -1,5 +1,5 @@ // Copyright (c) 2012-2015 Thomas Heller -// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -9,16 +9,23 @@ #pragma once -#include +#include + +#if defined(HPX_HAVE_NETWORKING) #include #include -@_parcelport_export@ +@parcelport_export_declarations@ namespace hpx::parcelset { - void init_static_parcelport_factories( + // force linking with this module + HPX_EXPORT void init_parcel_ports(); + + inline void init_static_parcelport_factories_impl( std::vector& factories) { -@_parcelport_init@ } +@parcelport_init_definitions@ } } + +#endif diff --git a/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp b/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp new file mode 100644 index 000000000000..554173676522 --- /dev/null +++ b/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp @@ -0,0 +1,19 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) + +namespace hpx::parcelset +{ + // force linking with this module + HPX_EXPORT void init_all_parcelports(); +} + +#endif diff --git a/libs/full/parcelports/src/static_parcelports.cpp b/libs/full/parcelports/src/static_parcelports.cpp new file mode 100644 index 000000000000..7258b6171cd9 --- /dev/null +++ b/libs/full/parcelports/src/static_parcelports.cpp @@ -0,0 +1,35 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) +#include +#include +#include +#include + +#include + +namespace hpx::parcelset { + + void (*init_static_parcelport_factories)( + std::vector& factories) = nullptr; + + struct HPX_EXPORT init_parcelports + { + init_parcelports() + { + init_static_parcelport_factories = + init_static_parcelport_factories_impl; + } + } init; + + // force linking with this module + void init_all_parcelports() {} +} // namespace hpx::parcelset + +#endif diff --git a/libs/full/parcelset/CMakeLists.txt b/libs/full/parcelset/CMakeLists.txt index 3b331756c5a8..cc49c3f793aa 100644 --- a/libs/full/parcelset/CMakeLists.txt +++ b/libs/full/parcelset/CMakeLists.txt @@ -14,6 +14,7 @@ set(parcelset_headers hpx/parcelset/detail/parcel_await.hpp hpx/parcelset/detail/message_handler_interface_functions.hpp hpx/parcelset/encode_parcels.hpp + hpx/parcelset/init_parcelports.hpp hpx/parcelset/message_handler_fwd.hpp hpx/parcelset/parcel.hpp hpx/parcelset/parcelhandler.hpp diff --git a/libs/full/parcelset/include/hpx/parcelset/init_parcelports.hpp b/libs/full/parcelset/include/hpx/parcelset/init_parcelports.hpp new file mode 100644 index 000000000000..02420f7fe211 --- /dev/null +++ b/libs/full/parcelset/include/hpx/parcelset/init_parcelports.hpp @@ -0,0 +1,22 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) +#include + +#include + +namespace hpx::parcelset { + + extern HPX_EXPORT void (*init_static_parcelport_factories)( + std::vector& factories); +} + +#endif diff --git a/libs/full/parcelset/src/parcelhandler.cpp b/libs/full/parcelset/src/parcelhandler.cpp index 8e073b362af6..216bca8e52c4 100644 --- a/libs/full/parcelset/src/parcelhandler.cpp +++ b/libs/full/parcelset/src/parcelhandler.cpp @@ -33,9 +33,9 @@ #include #include +#include #include #include -#include #include #include #include diff --git a/libs/full/plugin_factories/include/hpx/plugin_factories/parcelport_factory.hpp b/libs/full/plugin_factories/include/hpx/plugin_factories/parcelport_factory.hpp index 763aab44c689..3e9922a46a2d 100644 --- a/libs/full/plugin_factories/include/hpx/plugin_factories/parcelport_factory.hpp +++ b/libs/full/plugin_factories/include/hpx/plugin_factories/parcelport_factory.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2014 Thomas Heller -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // Copyright (c) 2020 Google // // SPDX-License-Identifier: BSL-1.0 From 439ff28fe378a32bfbbcac0b18fea8b142405ed9 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 18 Aug 2023 15:18:07 -0500 Subject: [PATCH 171/333] Fixing test failure - flyby: remove stubs from example --- .../cancelable_action/cancelable_action.cpp | 10 ++--- .../cancelable_action/cancelable_action.hpp | 22 +++++----- .../server/cancelable_action.hpp | 37 ++++++++++------ .../stubs/cancelable_action.hpp | 43 ------------------- .../cancelable_action_client.cpp | 5 ++- .../hpx/parcelports/init_all_parcelports.hpp | 6 +-- 6 files changed, 47 insertions(+), 76 deletions(-) delete mode 100644 examples/cancelable_action/cancelable_action/stubs/cancelable_action.hpp diff --git a/examples/cancelable_action/cancelable_action/cancelable_action.cpp b/examples/cancelable_action/cancelable_action/cancelable_action.cpp index 189b352e556c..8c176f414cc2 100644 --- a/examples/cancelable_action/cancelable_action/cancelable_action.cpp +++ b/examples/cancelable_action/cancelable_action/cancelable_action.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2007-2012 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include + #if !defined(HPX_COMPUTE_DEVICE_CODE) -#include #include -#include +#include #include "server/cancelable_action.hpp" @@ -17,8 +17,8 @@ HPX_REGISTER_COMPONENT_MODULE() /////////////////////////////////////////////////////////////////////////////// -typedef hpx::components::component - cancelable_action_component_type; +using cancelable_action_component_type = + hpx::components::component; HPX_REGISTER_COMPONENT(cancelable_action_component_type, cancelable_action) diff --git a/examples/cancelable_action/cancelable_action/cancelable_action.hpp b/examples/cancelable_action/cancelable_action/cancelable_action.hpp index cd660ff0e4ab..372ac05e1ebd 100644 --- a/examples/cancelable_action/cancelable_action/cancelable_action.hpp +++ b/examples/cancelable_action/cancelable_action/cancelable_action.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2011 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -11,21 +11,21 @@ #include #include -#include "stubs/cancelable_action.hpp" +#include "server/cancelable_action.hpp" #include namespace examples { + /////////////////////////////////////////////////////////////////////////// // Client side representation for for the \a server::cancelable_action // component. class cancelable_action : public hpx::components::client_base + server::cancelable_action> { - typedef hpx::components::client_base - base_type; + using base_type = hpx::components::client_base; public: // Default construct an empty client side representation (not @@ -45,16 +45,18 @@ namespace examples { } /////////////////////////////////////////////////////////////////////// - void do_it(hpx::error_code& ec = hpx::throws) + void do_it(hpx::error_code& ec = hpx::throws) const { + using action_type = server::cancelable_action::do_it_action; HPX_ASSERT(this->get_id()); - this->base_type::do_it(this->get_id(), ec); + hpx::async(this->get_id()).get(ec); } - void cancel_it() + void cancel_it() const { + using action_type = server::cancelable_action::cancel_it_action; HPX_ASSERT(this->get_id()); - this->base_type::cancel_it(this->get_id()); + hpx::post(this->get_id()); } }; } // namespace examples diff --git a/examples/cancelable_action/cancelable_action/server/cancelable_action.hpp b/examples/cancelable_action/cancelable_action/server/cancelable_action.hpp index b3b9de3e62f7..9c131d256650 100644 --- a/examples/cancelable_action/cancelable_action/server/cancelable_action.hpp +++ b/examples/cancelable_action/cancelable_action/server/cancelable_action.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -7,18 +7,22 @@ #pragma once #include + #if !defined(HPX_COMPUTE_DEVICE_CODE) -#include +#include + #include #include #include #include #include +#include #include /////////////////////////////////////////////////////////////////////////////// -namespace examples { namespace server { +namespace examples::server { + /////////////////////////////////////////////////////////////////////////// inline void delay(int c) { @@ -50,24 +54,29 @@ namespace examples { namespace server { explicit reset_id(cancelable_action& this_) : outer_(this_) { - std::lock_guard l(outer_.mtx_); - hpx::thread::id old_value = outer_.id_; + auto const mtx = outer_.mtx_; + std::lock_guard l(*mtx); + + [[maybe_unused]] hpx::thread::id const old_value = outer_.id_; outer_.id_ = hpx::this_thread::get_id(); HPX_ASSERT(old_value == hpx::thread::id()); - HPX_UNUSED(old_value); } ~reset_id() { - hpx::thread::id old_value = outer_.id_; + [[maybe_unused]] hpx::thread::id const old_value = outer_.id_; outer_.id_ = hpx::thread::id(); HPX_ASSERT(old_value != hpx::thread::id()); - HPX_UNUSED(old_value); } cancelable_action& outer_; }; public: + cancelable_action() + : mtx_(std::make_shared()) + { + } + // Do some lengthy work void do_it() { @@ -85,15 +94,17 @@ namespace examples { namespace server { } // Cancel the lengthy action above - void cancel_it() + void cancel_it() const { // Make sure id_ has been set hpx::util::yield_while([this]() { - std::lock_guard l(mtx_); + auto const mtx = mtx_; + std::lock_guard l(*mtx); return id_ == hpx::thread::id(); }); - std::lock_guard l(mtx_); + auto const mtx = mtx_; + std::lock_guard l(*mtx); HPX_ASSERT(id_ != hpx::thread::id()); hpx::thread::interrupt(id_); } @@ -103,10 +114,10 @@ namespace examples { namespace server { cancelable_action, cancel_it, cancel_it_action) private: - hpx::mutex mtx_; + std::shared_ptr mtx_; hpx::thread::id id_; }; -}} // namespace examples::server +} // namespace examples::server /////////////////////////////////////////////////////////////////////////////// HPX_REGISTER_ACTION_DECLARATION( diff --git a/examples/cancelable_action/cancelable_action/stubs/cancelable_action.hpp b/examples/cancelable_action/cancelable_action/stubs/cancelable_action.hpp deleted file mode 100644 index e7253a683c72..000000000000 --- a/examples/cancelable_action/cancelable_action/stubs/cancelable_action.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2007-2012 Hartmut Kaiser -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#pragma once - -#include -#if !defined(HPX_COMPUTE_DEVICE_CODE) -#include - -#include "../server/cancelable_action.hpp" - -/////////////////////////////////////////////////////////////////////////////// -namespace examples { namespace stubs { - /////////////////////////////////////////////////////////////////////////// - struct cancelable_action - : hpx::components::stub_base - { - // Do some lengthy work - static hpx::future do_it_async(hpx::id_type const& gid) - { - typedef server::cancelable_action::do_it_action action_type; - return hpx::async(gid); - } - - static void do_it( - hpx::id_type const& gid, hpx::error_code& ec = hpx::throws) - { - do_it_async(gid).get(ec); - } - - // Cancel the lengthy action above - static void cancel_it(hpx::id_type const& gid) - { - typedef server::cancelable_action::cancel_it_action action_type; - hpx::post(gid); - } - }; -}} // namespace examples::stubs - -#endif diff --git a/examples/cancelable_action/cancelable_action_client.cpp b/examples/cancelable_action/cancelable_action_client.cpp index 5d025fa10e2b..11e166230083 100644 --- a/examples/cancelable_action/cancelable_action_client.cpp +++ b/examples/cancelable_action/cancelable_action_client.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2007-2012 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include + #if !defined(HPX_COMPUTE_DEVICE_CODE) #include #include @@ -12,7 +13,7 @@ #include "cancelable_action/cancelable_action.hpp" /////////////////////////////////////////////////////////////////////////////// -void interrupt_do_it(examples::cancelable_action ca) +void interrupt_do_it(examples::cancelable_action const& ca) { // wait for one second before interrupting the (possibly remote) operation hpx::this_thread::sleep_for(std::chrono::seconds(1)); diff --git a/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp b/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp index 554173676522..2c4c2eb190db 100644 --- a/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp +++ b/libs/full/parcelports/include/hpx/parcelports/init_all_parcelports.hpp @@ -10,10 +10,10 @@ #if defined(HPX_HAVE_NETWORKING) -namespace hpx::parcelset -{ +namespace hpx::parcelset { + // force linking with this module HPX_EXPORT void init_all_parcelports(); -} +} // namespace hpx::parcelset #endif From f0285592a120e0f09cde4b6fbc916c03d5757a2f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 19 Aug 2023 09:59:19 -0500 Subject: [PATCH 172/333] Adding POSITION_INDEPENDENT_CODE=ON to all HPX targets --- cmake/HPX_SetupTarget.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/HPX_SetupTarget.cmake b/cmake/HPX_SetupTarget.cmake index aec75f68959e..5b3f8c61950b 100644 --- a/cmake/HPX_SetupTarget.cmake +++ b/cmake/HPX_SetupTarget.cmake @@ -216,6 +216,8 @@ function(hpx_setup_target target) hpx_debug("setup_target.${target} UNITY_BUILD: OFF") endif() + set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON) + get_target_property(target_EXCLUDE_FROM_ALL ${target} EXCLUDE_FROM_ALL) if(target_EXPORT AND NOT target_EXCLUDE_FROM_ALL) From b97d698ac080389b8a05d76bad7fd1efcf1c68ac Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Wed, 19 Jul 2023 17:49:57 -0500 Subject: [PATCH 173/333] Adapting build system for CMake V3.27 --- CMakeLists.txt | 33 ++++--- cmake/FindAmplifier.cmake | 53 ++++++----- cmake/FindAsio.cmake | 46 +++++---- cmake/FindBreathe.cmake | 17 +++- cmake/FindHwloc.cmake | 67 +++++++------ cmake/FindJemalloc.cmake | 95 ++++++++++--------- cmake/FindLibSigSegv.cmake | 64 +++++++------ cmake/FindLibfabric.cmake | 39 +++++--- cmake/FindMSR.cmake | 43 +++++---- cmake/FindOrangeFS.cmake | 45 +++++---- cmake/FindPAPI.cmake | 71 -------------- cmake/FindPWR.cmake | 73 +++++++------- cmake/FindPapi.cmake | 88 +++++++++++++++++ cmake/FindQThreads.cmake | 45 +++++---- cmake/FindRDMA_CM.cmake | 30 ++++-- cmake/FindRdmacm.cmake | 37 +++++--- cmake/FindSnappy.cmake | 49 ++++++---- cmake/FindSphinx.cmake | 17 +++- cmake/FindTBB.cmake | 41 ++++---- cmake/FindTBBmalloc.cmake | 69 ++++++++------ cmake/FindTCMalloc.cmake | 61 +++++++----- cmake/FindValgrind.cmake | 27 ++++-- cmake/HPX_AddTest.cmake | 6 +- cmake/HPX_Documentation.cmake | 8 +- cmake/HPX_SetupAllocator.cmake | 22 ++--- cmake/HPX_SetupApex.cmake | 51 +++++++--- cmake/HPX_SetupAsio.cmake | 20 +++- cmake/HPX_SetupBoost.cmake | 21 ++-- cmake/HPX_SetupBoostFilesystem.cmake | 7 +- cmake/HPX_SetupBoostIostreams.cmake | 6 +- cmake/HPX_SetupBoostRegex.cmake | 6 +- cmake/HPX_SetupEve.cmake | 24 +++-- cmake/HPX_SetupHIP.cmake | 2 +- cmake/HPX_SetupHwloc.cmake | 4 +- cmake/HPX_SetupLCI.cmake | 16 +++- cmake/HPX_SetupPapi.cmake | 6 +- cmake/HPX_SetupSVE.cmake | 24 +++-- cmake/HPX_SetupValgrind.cmake | 4 +- cmake/installed_hpx.cmake | 19 +++- cmake/templates/HPXConfig.cmake.in | 46 ++++----- cmake/templates/HPXMacros.cmake.in | 8 +- cmake/templates/cmake_variables.rst.in | 12 +-- cmake/templates/hpxcxx.in | 2 +- cmake/templates/hpxrun.py.in | 2 +- cmake/toolchains/BGION-gcc.cmake | 4 +- .../binary_filter/snappy/CMakeLists.txt | 11 ++- docs/CMakeLists.txt | 4 +- docs/sphinx/contributing/documentation.rst | 6 +- docs/sphinx/manual/building_hpx.rst | 30 +++--- docs/sphinx/manual/creating_hpx_projects.rst | 20 ++-- .../manual/optimizing_hpx_applications.rst | 2 +- docs/sphinx/manual/troubleshooting.rst | 4 +- examples/1d_stencil/README.APEX | 2 +- libs/CMakeLists.txt | 2 +- .../cmake/HPX_SetupJSON.cmake | 19 +++- .../parcelport_libfabric/cmake/FindPMI.cmake | 66 +++++++------ tests/performance/local/CMakeLists.txt | 10 +- .../performance/local/htts_v2/CMakeLists.txt | 8 +- tools/build_boost.sh | 4 +- 59 files changed, 967 insertions(+), 651 deletions(-) delete mode 100644 cmake/FindPAPI.cmake create mode 100644 cmake/FindPapi.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 9363c522969e..28cf94607398 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1281,7 +1281,7 @@ hpx_option( ) if(HPX_WITH_EXAMPLES_TBB) find_package(TBB) - if(NOT TBB_FOUND) + if(NOT Tbb_FOUND) set(HPX_WITH_EXAMPLES_TBB OFF) endif() endif() @@ -1293,7 +1293,7 @@ hpx_option( ) if(HPX_WITH_EXAMPLES_QTHREADS) find_package(QThreads) - if(NOT QTHREADS_FOUND) + if(NOT Qthreads_FOUND) set(HPX_WITH_EXAMPLES_QTHREADS OFF) endif() endif() @@ -1306,7 +1306,7 @@ hpx_option( ) if(HPX_WITH_EXAMPLES_HDF5) find_package(HDF5 COMPONENTS CXX) - if(NOT HDF5_FOUND) + if(NOT Hdf5_FOUND) set(HPX_WITH_EXAMPLES_HDF5 OFF) endif() endif() @@ -1321,7 +1321,7 @@ if(NOT "${HPX_PLATFORM_UC}" STREQUAL "BLUEGENEQ") ) if(HPX_WITH_EXAMPLES_QT4) find_package(Qt4) - if(NOT QT4_FOUND) + if(NOT Qt4_FOUND) set(HPX_WITH_EXAMPLES_QT4 OFF) endif() endif() @@ -2333,12 +2333,23 @@ add_subdirectory(components) # ############################################################################## # Tests # ############################################################################## -find_package(PythonInterp) +if(POLICY CMP0148) + find_package(Python COMPONENTS Interpreter) + if(NOT Python_FOUND) + hpx_warn( + "A python interpreter could not be found. The test suite can not be run automatically." + ) + endif() -if(NOT PYTHONINTERP_FOUND) - hpx_warn( - "A python interpreter could not be found. The test suite can not be run automatically." - ) +else() + find_package(PythonInterp) + if(NOT PYTHONINTERP_FOUND) + hpx_warn( + "A python interpreter could not be found. The test suite can not be run automatically." + ) + else() + set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) + endif() endif() if(HPX_WITH_TESTS) @@ -2364,7 +2375,7 @@ if(HPX_WITH_TOOLS OR HPX_WITH_TESTS_BENCHMARKS) endif() # Configure hpxrun.py -if(PYTHONINTERP_FOUND) +if(PYTHONINTERP_FOUND OR Python_FOUND) configure_file( "${PROJECT_SOURCE_DIR}/cmake/templates/hpxrun.py.in" "${PROJECT_BINARY_DIR}/bin/hpxrun.py" @ONLY @@ -2489,7 +2500,7 @@ endif() # ############################################################################## # installation instructions # ############################################################################## -if(PYTHONINTERP_FOUND) +if(PYTHONINTERP_FOUND OR Python_FOUND) install( FILES "${PROJECT_BINARY_DIR}/bin/hpxrun.py" DESTINATION ${CMAKE_INSTALL_BINDIR} diff --git a/cmake/FindAmplifier.cmake b/cmake/FindAmplifier.cmake index abdacd13ac93..41ffbf74434e 100644 --- a/cmake/FindAmplifier.cmake +++ b/cmake/FindAmplifier.cmake @@ -1,64 +1,73 @@ # Copyright (c) 2014 Thomas Heller -# Copyright (c) 2012 Hartmut Kaiser +# Copyright (c) 2012-2023 Hartmut Kaiser # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(NOT TARGET Amplifier::amplifier) + # compatibility with older CMake versions + if(AMPLIFIER_ROOT AND NOT Amplifier_ROOT) + set(Amplifier_ROOT + ${AMPLIFIER_ROOT} + CACHE PATH "Amplifier base directory" + ) + unset(AMPLIFIER_ROOT CACHE) + endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_AMPLIFIER QUIET amplifier) find_path( - AMPLIFIER_INCLUDE_DIR ittnotify.h - HINTS ${AMPLIFIER_ROOT} ENV AMPLIFIER_ROOT ${PC_AMPLIFIER_INCLUDEDIR} - ${PC_AMPLIFIER_INCLUDE_DIRS} + Amplifier_INCLUDE_DIR ittnotify.h + HINTS ${Amplifier_ROOT} ENV AMPLIFIER_ROOT ${PC_Amplifier_INCLUDEDIR} + ${PC_Amplifier_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - AMPLIFIER_LIBRARY + Amplifier_LIBRARY NAMES ittnotify libittnotify - HINTS ${AMPLIFIER_ROOT} ENV AMPLIFIER_ROOT ${PC_AMPLIFIER_LIBDIR} - ${PC_AMPLIFIER_LIBRARY_DIRS} + HINTS ${Amplifier_ROOT} ENV AMPLIFIER_ROOT ${PC_Amplifier_LIBDIR} + ${PC_Amplifier_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) - # Set AMPLIFIER_ROOT in case the other hints are used - if(AMPLIFIER_ROOT) + # Set Amplifier_ROOT in case the other hints are used + if(Amplifier_ROOT) # The call to file is for compatibility for windows paths - file(TO_CMAKE_PATH ${AMPLIFIER_ROOT} AMPLIFIER_ROOT) + file(TO_CMAKE_PATH ${Amplifier_ROOT} Amplifier_ROOT) elseif("$ENV{AMPLIFIER_ROOT}") - file(TO_CMAKE_PATH $ENV{AMPLIFIER_ROOT} AMPLIFIER_ROOT) + file(TO_CMAKE_PATH $ENV{AMPLIFIER_ROOT} Amplifier_ROOT) else() - file(TO_CMAKE_PATH "${AMPLIFIER_INCLUDE_DIR}" AMPLIFIER_INCLUDE_DIR) - string(REPLACE "/include" "" AMPLIFIER_ROOT "${AMPLIFIER_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Amplifier_INCLUDE_DIR}" Amplifier_INCLUDE_DIR) + string(REPLACE "/include" "" Amplifier_ROOT "${Amplifier_INCLUDE_DIR}") endif() - set(AMPLIFIER_LIBRARIES ${AMPLIFIER_LIBRARY}) - set(AMPLIFIER_INCLUDE_DIRS ${AMPLIFIER_INCLUDE_DIR}) + set(Amplifier_LIBRARIES ${Amplifier_LIBRARY}) + set(Amplifier_INCLUDE_DIRS ${Amplifier_INCLUDE_DIR}) find_package_handle_standard_args( - Amplifier DEFAULT_MSG AMPLIFIER_LIBRARY AMPLIFIER_INCLUDE_DIR + Amplifier DEFAULT_MSG Amplifier_LIBRARY Amplifier_INCLUDE_DIR ) get_property( _type - CACHE AMPLIFIER_ROOT + CACHE Amplifier_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE AMPLIFIER_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Amplifier_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE AMPLIFIER_ROOT PROPERTY TYPE PATH) + set_property(CACHE Amplifier_ROOT PROPERTY TYPE PATH) endif() endif() - mark_as_advanced(AMPLIFIER_ROOT AMPLIFIER_LIBRARY AMPLIFIER_INCLUDE_DIR) + mark_as_advanced(Amplifier_ROOT Amplifier_LIBRARY Amplifier_INCLUDE_DIR) add_library(Amplifier::amplifier INTERFACE IMPORTED) target_include_directories( - Amplifier::amplifier SYSTEM INTERFACE ${AMPLIFIER_INCLUDE_DIR} + Amplifier::amplifier SYSTEM INTERFACE ${Amplifier_INCLUDE_DIR} ) - target_link_libraries(Amplifier::amplifier INTERFACE ${AMPLIFIER_LIBRARIES}) + target_link_libraries(Amplifier::amplifier INTERFACE ${Amplifier_LIBRARIES}) endif() diff --git a/cmake/FindAsio.cmake b/cmake/FindAsio.cmake index d66bcff352e5..76fea2033ddd 100644 --- a/cmake/FindAsio.cmake +++ b/cmake/FindAsio.cmake @@ -8,57 +8,67 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(NOT TARGET Asio::asio) + # compatibility with older CMake versions + if(ASIO_ROOT AND NOT Asio_ROOT) + set(Asio_ROOT + ${ASIO_ROOT} + CACHE PATH "Asio base directory" + ) + unset(ASIO_ROOT CACHE) + endif() + find_path( - ASIO_INCLUDE_DIR asio.hpp - HINTS "${ASIO_ROOT}" ENV ASIO_ROOT "${HPX_ASIO_ROOT}" + Asio_INCLUDE_DIR asio.hpp + HINTS "${Asio_ROOT}" ENV ASIO_ROOT "${HPX_ASIO_ROOT}" PATH_SUFFIXES include ) - if(NOT ASIO_INCLUDE_DIR) + if(NOT Asio_INCLUDE_DIR) hpx_error( - "Could not find Asio. Set ASIO_ROOT as a CMake or environment variable to point to the Asio root install directory. Alternatively, set HPX_WITH_FETCH_ASIO=ON to fetch Asio using CMake's FetchContent (when using this option Asio will be installed together with HPX, be careful about conflicts with separately installed versions of Asio)." + "Could not find Asio. Set Asio_ROOT as a CMake or environment variable to point to the Asio root install directory. Alternatively, set HPX_WITH_FETCH_ASIO=ON to fetch Asio using CMake's FetchContent (when using this option Asio will be installed together with HPX, be careful about conflicts with separately installed versions of Asio)." ) endif() - # Set ASIO_ROOT in case the other hints are used - if(ASIO_ROOT) + # Set Asio_ROOT in case the other hints are used + if(Asio_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${ASIO_ROOT} ASIO_ROOT) + file(TO_CMAKE_PATH ${Asio_ROOT} Asio_ROOT) elseif("$ENV{ASIO_ROOT}") - file(TO_CMAKE_PATH $ENV{ASIO_ROOT} ASIO_ROOT) + file(TO_CMAKE_PATH $ENV{ASIO_ROOT} Asio_ROOT) else() - file(TO_CMAKE_PATH "${ASIO_INCLUDE_DIR}" ASIO_INCLUDE_DIR) - string(REPLACE "/include" "" ASIO_ROOT "${ASIO_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Asio_INCLUDE_DIR}" Asio_INCLUDE_DIR) + string(REPLACE "/include" "" Asio_ROOT "${Asio_INCLUDE_DIR}") endif() - if(ASIO_INCLUDE_DIR AND EXISTS "${ASIO_INCLUDE_DIR}/asio/version.hpp") + if(Asio_INCLUDE_DIR AND EXISTS "${Asio_INCLUDE_DIR}/asio/version.hpp") # Matches a line of the form: # # #define ASIO_VERSION XXYYZZ // XX.YY.ZZ # # with arbitrary whitespace between the tokens file( - STRINGS "${ASIO_INCLUDE_DIR}/asio/version.hpp" ASIO_VERSION_DEFINE_LINE + STRINGS "${Asio_INCLUDE_DIR}/asio/version.hpp" Asio_VERSION_DEFINE_LINE REGEX "#define[ \t]+ASIO_VERSION[ \t]+[0-9]+[ \t]+//[ \t]+[0-9]+\.[0-9]+\.[0-9]+[ \t]*" ) # Extracts the dotted version number after the comment as - # ASIO_VERSION_STRING + # Asio_VERSION_STRING string(REGEX REPLACE "#define ASIO_VERSION [0-9]+ // ([0-9]+\.[0-9]+\.[0-9]+)" - "\\1" ASIO_VERSION_STRING "${ASIO_VERSION_DEFINE_LINE}" + "\\1" Asio_VERSION_STRING "${Asio_VERSION_DEFINE_LINE}" ) endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args( Asio - REQUIRED_VARS ASIO_INCLUDE_DIR - VERSION_VAR ASIO_VERSION_STRING + REQUIRED_VARS Asio_INCLUDE_DIR + VERSION_VAR Asio_VERSION_STRING + FOUND_VAR Asio_FOUND ) add_library(Asio::asio INTERFACE IMPORTED) - target_include_directories(Asio::asio SYSTEM INTERFACE ${ASIO_INCLUDE_DIR}) + target_include_directories(Asio::asio SYSTEM INTERFACE ${Asio_INCLUDE_DIR}) - mark_as_advanced(ASIO_ROOT ASIO_INCLUDE_DIR ASIO_VERSION_STRING) + mark_as_advanced(Asio_ROOT Asio_INCLUDE_DIR Asio_VERSION_STRING) endif() diff --git a/cmake/FindBreathe.cmake b/cmake/FindBreathe.cmake index f8fc8bc14d74..a43963c2bba5 100644 --- a/cmake/FindBreathe.cmake +++ b/cmake/FindBreathe.cmake @@ -4,16 +4,25 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(BREATHE_APIDOC_ROOT AND NOT Breathe_APIDOC_ROOT) + set(Breathe_APIDOC_ROOT + ${BREATHE_APIDOC_ROOT} + CACHE PATH "Breathe base directory" + ) + unset(BREATHE_ROOT CACHE) +endif() + find_program( - BREATHE_APIDOC_EXECUTABLE + Breathe_APIDOC_EXECUTABLE NAMES breathe-apidoc - PATHS ${BREATHE_APIDOC_ROOT} ENV BREATHE_APIDOC_ROOT + PATHS ${Breathe_APIDOC_ROOT} ENV Breathe_APIDOC_ROOT DOC "Path to breathe-apidoc executable" ) -if(BREATHE_APIDOC_EXECUTABLE) +if(Breathe_APIDOC_EXECUTABLE) include(FindPackageHandleStandardArgs) find_package_handle_standard_args( - Breathe DEFAULT_MESSAGE BREATHE_APIDOC_EXECUTABLE + Breathe DEFAULT_MESSAGE Breathe_APIDOC_EXECUTABLE ) endif() diff --git a/cmake/FindHwloc.cmake b/cmake/FindHwloc.cmake index 89dadf132013..7760048c948f 100644 --- a/cmake/FindHwloc.cmake +++ b/cmake/FindHwloc.cmake @@ -1,5 +1,5 @@ # Copyright (c) 2014 Thomas Heller -# Copyright (c) 2007-2012 Hartmut Kaiser +# Copyright (c) 2007-2023 Hartmut Kaiser # Copyright (c) 2010-2011 Matt Anderson # Copyright (c) 2011 Bryce Lelbach # @@ -8,70 +8,79 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(NOT TARGET Hwloc::hwloc) + # compatibility with older CMake versions + if(HWLOC_ROOT AND NOT Hwloc_ROOT) + set(Hwloc_ROOT + ${HWLOC_ROOT} + CACHE PATH "Hwloc base directory" + ) + unset(HWLOC_ROOT CACHE) + endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_HWLOC QUIET hwloc) find_path( - HWLOC_INCLUDE_DIR hwloc.h - HINTS ${HWLOC_ROOT} + Hwloc_INCLUDE_DIR hwloc.h + HINTS ${Hwloc_ROOT} ENV HWLOC_ROOT ${HPX_HWLOC_ROOT} - ${PC_HWLOC_MINIMAL_INCLUDEDIR} - ${PC_HWLOC_MINIMAL_INCLUDE_DIRS} - ${PC_HWLOC_INCLUDEDIR} - ${PC_HWLOC_INCLUDE_DIRS} + ${PC_Hwloc_MINIMAL_INCLUDEDIR} + ${PC_Hwloc_MINIMAL_INCLUDE_DIRS} + ${PC_Hwloc_INCLUDEDIR} + ${PC_Hwloc_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - HWLOC_LIBRARY + Hwloc_LIBRARY NAMES libhwloc.so hwloc - HINTS ${HWLOC_ROOT} + HINTS ${Hwloc_ROOT} ENV HWLOC_ROOT - ${HPX_HWLOC_ROOT} - ${PC_HWLOC_MINIMAL_LIBDIR} - ${PC_HWLOC_MINIMAL_LIBRARY_DIRS} - ${PC_HWLOC_LIBDIR} - ${PC_HWLOC_LIBRARY_DIRS} + ${HPX_Hwloc_ROOT} + ${PC_Hwloc_MINIMAL_LIBDIR} + ${PC_Hwloc_MINIMAL_LIBRARY_DIRS} + ${PC_Hwloc_LIBDIR} + ${PC_Hwloc_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) - # Set HWLOC_ROOT in case the other hints are used - if(HWLOC_ROOT) + # Set Hwloc_ROOT in case the other hints are used + if(Hwloc_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${HWLOC_ROOT} HWLOC_ROOT) + file(TO_CMAKE_PATH ${Hwloc_ROOT} Hwloc_ROOT) elseif("$ENV{HWLOC_ROOT}") - file(TO_CMAKE_PATH $ENV{HWLOC_ROOT} HWLOC_ROOT) + file(TO_CMAKE_PATH $ENV{HWLOC_ROOT} Hwloc_ROOT) else() - file(TO_CMAKE_PATH "${HWLOC_INCLUDE_DIR}" HWLOC_INCLUDE_DIR) - string(REPLACE "/include" "" HWLOC_ROOT "${HWLOC_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Hwloc_INCLUDE_DIR}" Hwloc_INCLUDE_DIR) + string(REPLACE "/include" "" Hwloc_ROOT "${Hwloc_INCLUDE_DIR}") endif() - set(HWLOC_LIBRARIES ${HWLOC_LIBRARY}) - set(HWLOC_INCLUDE_DIRS ${HWLOC_INCLUDE_DIR}) + set(Hwloc_LIBRARIES ${Hwloc_LIBRARY}) + set(Hwloc_INCLUDE_DIRS ${Hwloc_INCLUDE_DIR}) find_package_handle_standard_args( - Hwloc DEFAULT_MSG HWLOC_LIBRARY HWLOC_INCLUDE_DIR + Hwloc DEFAULT_MSG Hwloc_LIBRARY Hwloc_INCLUDE_DIR ) get_property( _type - CACHE HWLOC_ROOT + CACHE Hwloc_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE HWLOC_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Hwloc_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE HWLOC_ROOT PROPERTY TYPE PATH) + set_property(CACHE Hwloc_ROOT PROPERTY TYPE PATH) endif() endif() add_library(Hwloc::hwloc INTERFACE IMPORTED) - target_include_directories(Hwloc::hwloc SYSTEM INTERFACE ${HWLOC_INCLUDE_DIR}) - - target_link_libraries(Hwloc::hwloc INTERFACE ${HWLOC_LIBRARIES}) + target_include_directories(Hwloc::hwloc SYSTEM INTERFACE ${Hwloc_INCLUDE_DIR}) + target_link_libraries(Hwloc::hwloc INTERFACE ${Hwloc_LIBRARIES}) mark_as_advanced(HWLOC_ROOT HWLOC_LIBRARY HWLOC_INCLUDE_DIR) + mark_as_advanced(Hwloc_ROOT Hwloc_LIBRARY Hwloc_INCLUDE_DIR) endif() diff --git a/cmake/FindJemalloc.cmake b/cmake/FindJemalloc.cmake index facccbbb7ca5..e828a39b5170 100644 --- a/cmake/FindJemalloc.cmake +++ b/cmake/FindJemalloc.cmake @@ -1,5 +1,5 @@ # Copyright (c) 2014 Thomas Heller -# Copyright (c) 2007-2012 Hartmut Kaiser +# Copyright (c) 2007-2023 Hartmut Kaiser # Copyright (c) 2010-2011 Matt Anderson # Copyright (c) 2011 Bryce Lelbach # @@ -7,19 +7,28 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(JEMALLOC_ROOT AND NOT Jemalloc_ROOT) + set(Jemalloc_ROOT + ${JEMALLOC_ROOT} + CACHE PATH "Jemalloc base directory" + ) + unset(JEMALLOC_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_JEMALLOC QUIET jemalloc) find_path( - JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h - HINTS ${JEMALLOC_ROOT} + Jemalloc_INCLUDE_DIR jemalloc/jemalloc.h + HINTS ${Jemalloc_ROOT} ENV - JEMALLOC_ROOT - ${HPX_JEMALLOC_ROOT} - ${PC_JEMALLOC_MINIMAL_INCLUDEDIR} - ${PC_JEMALLOC_MINIMAL_INCLUDE_DIRS} - ${PC_JEMALLOC_INCLUDEDIR} - ${PC_JEMALLOC_INCLUDE_DIRS} + Jemalloc_ROOT + ${HPX_Jemalloc_ROOT} + ${PC_Jemalloc_MINIMAL_INCLUDEDIR} + ${PC_Jemalloc_MINIMAL_INCLUDE_DIRS} + ${PC_Jemalloc_INCLUDEDIR} + ${PC_Jemalloc_INCLUDE_DIRS} PATH_SUFFIXES include ) @@ -27,68 +36,68 @@ if(MSVC) # MSVC needs additional header files provided by jemalloc to compensate for # missing posix headers find_path( - JEMALLOC_ADDITIONAL_INCLUDE_DIR msvc_compat/strings.h - HINTS ${JEMALLOC_ROOT} + Jemalloc_ADDITIONAL_INCLUDE_DIR msvc_compat/strings.h + HINTS ${Jemalloc_ROOT} ENV - JEMALLOC_ROOT - ${HPX_JEMALLOC_ROOT} - ${PC_JEMALLOC_MINIMAL_INCLUDEDIR} - ${PC_JEMALLOC_MINIMAL_INCLUDE_DIRS} - ${PC_JEMALLOC_INCLUDEDIR} - ${PC_JEMALLOC_INCLUDE_DIRS} + Jemalloc_ROOT + ${HPX_Jemalloc_ROOT} + ${PC_Jemalloc_MINIMAL_INCLUDEDIR} + ${PC_Jemalloc_MINIMAL_INCLUDE_DIRS} + ${PC_Jemalloc_INCLUDEDIR} + ${PC_Jemalloc_INCLUDE_DIRS} PATH_SUFFIXES include ) endif() -# Set JEMALLOC_ROOT in case the other hints are used -if(JEMALLOC_ROOT) +# Set Jemalloc_ROOT in case the other hints are used +if(Jemalloc_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${JEMALLOC_ROOT} JEMALLOC_ROOT) + file(TO_CMAKE_PATH ${Jemalloc_ROOT} Jemalloc_ROOT) elseif("$ENV{JEMALLOC_ROOT}") - file(TO_CMAKE_PATH $ENV{JEMALLOC_ROOT} JEMALLOC_ROOT) + file(TO_CMAKE_PATH $ENV{JEMALLOC_ROOT} Jemalloc_ROOT) else() - file(TO_CMAKE_PATH "${JEMALLOC_INCLUDE_DIR}" JEMALLOC_INCLUDE_DIR) - string(REPLACE "/include" "" JEMALLOC_ROOT "${JEMALLOC_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Jemalloc_INCLUDE_DIR}" Jemalloc_INCLUDE_DIR) + string(REPLACE "/include" "" Jemalloc_ROOT "${Jemalloc_INCLUDE_DIR}") endif() find_library( - JEMALLOC_LIBRARY + Jemalloc_LIBRARY NAMES jemalloc libjemalloc - HINTS ${JEMALLOC_ROOT} + HINTS ${Jemalloc_ROOT} ENV - JEMALLOC_ROOT - ${HPX_JEMALLOC_ROOT} - ${PC_JEMALLOC_MINIMAL_LIBDIR} - ${PC_JEMALLOC_MINIMAL_LIBRARY_DIRS} - ${PC_JEMALLOC_LIBDIR} - ${PC_JEMALLOC_LIBRARY_DIRS} + Jemalloc_ROOT + ${HPX_Jemalloc_ROOT} + ${PC_Jemalloc_MINIMAL_LIBDIR} + ${PC_Jemalloc_MINIMAL_LIBRARY_DIRS} + ${PC_Jemalloc_LIBDIR} + ${PC_Jemalloc_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set JEMALLOC_ROOT in case the other hints are used -if(NOT JEMALLOC_ROOT AND "$ENV{JEMALLOC_ROOT}") - set(JEMALLOC_ROOT $ENV{JEMALLOC_ROOT}) -elseif(NOT JEMALLOC_ROOT) - string(REPLACE "/include" "" JEMALLOC_ROOT "${JEMALLOC_INCLUDE_DIR}") +# Set Jemalloc_ROOT in case the other hints are used +if(NOT Jemalloc_ROOT AND "$ENV{JEMALLOC_ROOT}") + set(Jemalloc_ROOT $ENV{JEMALLOC_ROOT}) +elseif(NOT Jemalloc_ROOT) + string(REPLACE "/include" "" Jemalloc_ROOT "${Jemalloc_INCLUDE_DIR}") endif() -set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY}) -set(JEMALLOC_INCLUDE_DIRS ${JEMALLOC_INCLUDE_DIR}) +set(Jemalloc_LIBRARIES ${Jemalloc_LIBRARY}) +set(Jemalloc_INCLUDE_DIRS ${Jemalloc_INCLUDE_DIR}) find_package_handle_standard_args( - Jemalloc DEFAULT_MSG JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR + Jemalloc DEFAULT_MSG Jemalloc_LIBRARY Jemalloc_INCLUDE_DIR ) get_property( _type - CACHE JEMALLOC_ROOT + CACHE Jemalloc_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE JEMALLOC_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Jemalloc_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE JEMALLOC_ROOT PROPERTY TYPE PATH) + set_property(CACHE Jemalloc_ROOT PROPERTY TYPE PATH) endif() endif() -mark_as_advanced(JEMALLOC_ROOT JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR) +mark_as_advanced(Jemalloc_ROOT Jemalloc_LIBRARY Jemalloc_INCLUDE_DIR) diff --git a/cmake/FindLibSigSegv.cmake b/cmake/FindLibSigSegv.cmake index 150c31965660..e37bac02a0b6 100644 --- a/cmake/FindLibSigSegv.cmake +++ b/cmake/FindLibSigSegv.cmake @@ -1,64 +1,74 @@ # Copyright (c) 2017 Abhimanyu Rawat +# # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(LIBSIGSEGV_ROOT AND NOT Libsigsegv_ROOT) + set(Libsigsegv_ROOT + ${LIBSIGSEGV_ROOT} + CACHE PATH "Libsigsegv base directory" + ) + unset(LIBSIGSEGV_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_LIBSIGSEGV QUIET libsigsegv) find_path( - LIBSIGSEGV_INCLUDE_DIR sigsegv.h - HINTS ${LIBSIGSEGV_ROOT} + Libsigsegv_INCLUDE_DIR sigsegv.h + HINTS ${Libsigsegv_ROOT} ENV - LIBSIGSEGV_ROOT - ${PC_LIBSIGSEGV_MINIMAL_INCLUDEDIR} - ${PC_LIBSIGSEGV_MINIMAL_INCLUDE_DIRS} - ${PC_LIBSIGSEGV_INCLUDEDIR} - ${PC_LIBSIGSEGV_INCLUDE_DIRS} + Libsigsegv_ROOT + ${PC_Libsigsegv_MINIMAL_INCLUDEDIR} + ${PC_Libsigsegv_MINIMAL_INCLUDE_DIRS} + ${PC_Libsigsegv_INCLUDEDIR} + ${PC_Libsigsegv_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - LIBSIGSEGV_LIBRARY + Libsigsegv_LIBRARY NAMES sigsegv libsigsegv - HINTS ${LIBSIGSEGV_ROOT} + HINTS ${Libsigsegv_ROOT} ENV - LIBSIGSEGV_ROOT - ${PC_LIBSIGSEGV_MINIMAL_LIBDIR} - ${PC_LIBSIGSEGV_MINIMAL_LIBRARY_DIRS} - ${PC_LIBSIGSEGV_LIBDIR} - ${PC_LIBSIGSEGV_LIBRARY_DIRS} + Libsigsegv_ROOT + ${PC_Libsigsegv_MINIMAL_LIBDIR} + ${PC_Libsigsegv_MINIMAL_LIBRARY_DIRS} + ${PC_Libsigsegv_LIBDIR} + ${PC_Libsigsegv_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set LIBSIGSEGV_ROOT in case the other hints are used -if(LIBSIGSEGV_ROOT) +# Set Libsigsegv_ROOT in case the other hints are used +if(Libsigsegv_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${LIBSIGSEGV_ROOT} LIBSIGSEGV_ROOT) + file(TO_CMAKE_PATH ${Libsigsegv_ROOT} Libsigsegv_ROOT) elseif("$ENV{LIBSIGSEGV_ROOT}") - file(TO_CMAKE_PATH $ENV{LIBSIGSEGV_ROOT} LIBSIGSEGV_ROOT) + file(TO_CMAKE_PATH $ENV{LIBSIGSEGV_ROOT} Libsigsegv_ROOT) else() - file(TO_CMAKE_PATH "${LIBSIGSEGV_INCLUDE_DIR}" LIBSIGSEGV_INCLUDE_DIR) - string(REPLACE "/include" "" LIBSIGSEGV_ROOT "${LIBSIGSEGV_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Libsigsegv_INCLUDE_DIR}" Libsigsegv_INCLUDE_DIR) + string(REPLACE "/include" "" Libsigsegv_ROOT "${Libsigsegv_INCLUDE_DIR}") endif() -set(LIBSIGSEGV_LIBRARIES ${LIBSIGSEGV_LIBRARY}) -set(LIBSIGSEGV_INCLUDE_DIRS ${LIBSIGSEGV_INCLUDE_DIR}) +set(Libsigsegv_LIBRARIES ${Libsigsegv_LIBRARY}) +set(Libsigsegv_INCLUDE_DIRS ${Libsigsegv_INCLUDE_DIR}) find_package_handle_standard_args( - LibSigSegv DEFAULT_MSG LIBSIGSEGV_LIBRARY LIBSIGSEGV_INCLUDE_DIR + LibSigSegv DEFAULT_MSG Libsigsegv_LIBRARY Libsigsegv_INCLUDE_DIR ) get_property( _type - CACHE LIBSIGSEGV_ROOT + CACHE Libsigsegv_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE LIBSIGSEGV_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Libsigsegv_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE LIBSIGSEGV_ROOT PROPERTY TYPE PATH) + set_property(CACHE Libsigsegv_ROOT PROPERTY TYPE PATH) endif() endif() -mark_as_advanced(LIBSIGSEGV_ROOT LIBSIGSEGV_LIBRARY LIBSIGSEGV_INCLUDE_DIR) +mark_as_advanced(Libsigsegv_ROOT Libsigsegv_LIBRARY Libsigsegv_INCLUDE_DIR) diff --git a/cmake/FindLibfabric.cmake b/cmake/FindLibfabric.cmake index d5a9ae06408a..ce6a3a247835 100644 --- a/cmake/FindLibfabric.cmake +++ b/cmake/FindLibfabric.cmake @@ -4,42 +4,51 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(LIBFABRIC_ROOT AND NOT Libfabric_ROOT) + set(Libfabric_ROOT + ${LIBFABRIC_ROOT} + CACHE PATH "Libfabric base directory" + ) + unset(LIBFABRIC_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_LIBFABRIC QUIET libfabric) find_path( - LIBFABRIC_INCLUDE_DIR rdma/fabric.h - HINTS ${LIBFABRIC_ROOT} ENV LIBFABRIC_ROOT ${LIBFABRIC_DIR} ENV LIBFABRIC_DIR + Libfabric_INCLUDE_DIR rdma/fabric.h + HINTS ${Libfabric_ROOT} ENV LIBFABRIC_ROOT ${Libfabric_DIR} ENV LIBFABRIC_DIR PATH_SUFFIXES include ) find_library( - LIBFABRIC_LIBRARY + Libfabric_LIBRARY NAMES fabric - HINTS ${LIBFABRIC_ROOT} ENV LIBFABRIC_ROOT + HINTS ${Libfabric_ROOT} ENV LIBFABRIC_ROOT PATH_SUFFIXES lib lib64 ) -# Set LIBFABRIC_ROOT in case the other hints are used -if(LIBFABRIC_ROOT) +# Set Libfabric_ROOT in case the other hints are used +if(Libfabric_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${LIBFABRIC_ROOT} LIBFABRIC_ROOT) + file(TO_CMAKE_PATH ${Libfabric_ROOT} Libfabric_ROOT) elseif("$ENV{LIBFABRIC_ROOT}") - file(TO_CMAKE_PATH $ENV{LIBFABRIC_ROOT} LIBFABRIC_ROOT) + file(TO_CMAKE_PATH $ENV{LIBFABRIC_ROOT} Libfabric_ROOT) else() - file(TO_CMAKE_PATH "${LIBFABRIC_INCLUDE_DIR}" LIBFABRIC_INCLUDE_DIR) - string(REPLACE "/include" "" LIBFABRIC_ROOT "${LIBFABRIC_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Libfabric_INCLUDE_DIR}" Libfabric_INCLUDE_DIR) + string(REPLACE "/include" "" Libfabric_ROOT "${Libfabric_INCLUDE_DIR}") endif() -if(NOT LIBFABRIC_INCLUDE_DIR OR NOT LIBFABRIC_LIBRARY) - hpx_error("Could not find LIBFABRIC_INCLUDE_DIR or LIBFABRIC_LIBRARY please \ - set the LIBFABRIC_ROOT environment variable" +if(NOT Libfabric_INCLUDE_DIR OR NOT Libfabric_LIBRARY) + hpx_error("Could not find Libfabric_INCLUDE_DIR or Libfabric_LIBRARY please \ + set the Libfabric_ROOT environment variable" ) endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args( - Libfabric DEFAULT_MSG LIBFABRIC_LIBRARY LIBFABRIC_INCLUDE_DIR + Libfabric DEFAULT_MSG Libfabric_LIBRARY Libfabric_INCLUDE_DIR ) -mark_as_advanced(LIBFABRIC_ROOT) +mark_as_advanced(Libfabric_ROOT) diff --git a/cmake/FindMSR.cmake b/cmake/FindMSR.cmake index 13edbd17aa0e..fb7c2a41526c 100644 --- a/cmake/FindMSR.cmake +++ b/cmake/FindMSR.cmake @@ -8,48 +8,57 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(MSR_ROOT AND NOT Msr_ROOT) + set(Msr_ROOT + ${MSR_ROOT} + CACHE PATH "Msr base directory" + ) + unset(MSR_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_MSR QUIET libmsr) find_path( - MSR_INCLUDE_DIR msr_core.h - HINTS ${MSR_ROOT} ENV MSR_ROOT ${PC_MSR_INCLUDEDIR} ${PC_MSR_INCLUDE_DIRS} + Msr_INCLUDE_DIR msr_core.h + HINTS ${Msr_ROOT} ENV MSR_ROOT ${PC_Msr_INCLUDEDIR} ${PC_Msr_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - MSR_LIBRARY + Msr_LIBRARY NAMES msr libmsr - HINTS ${MSR_ROOT} ENV MSR_ROOT ${PC_MSR_LIBDIR} ${PC_MSR_LIBRARY_DIRS} + HINTS ${Msr_ROOT} ENV MSR_ROOT ${PC_Msr_LIBDIR} ${PC_Msr_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set MSR_ROOT in case the other hints are used -if(MSR_ROOT) +# Set Msr_ROOT in case the other hints are used +if(Msr_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${MSR_ROOT} MSR_ROOT) + file(TO_CMAKE_PATH ${Msr_ROOT} Msr_ROOT) elseif("$ENV{MSR_ROOT}") - file(TO_CMAKE_PATH $ENV{MSR_ROOT} MSR_ROOT) + file(TO_CMAKE_PATH $ENV{MSR_ROOT} Msr_ROOT) else() - file(TO_CMAKE_PATH "${MSR_INCLUDE_DIR}" MSR_INCLUDE_DIR) - string(REPLACE "/include" "" MSR_ROOT "${MSR_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Msr_INCLUDE_DIR}" Msr_INCLUDE_DIR) + string(REPLACE "/include" "" Msr_ROOT "${Msr_INCLUDE_DIR}") endif() -set(MSR_LIBRARIES ${MSR_LIBRARY}) -set(MSR_INCLUDE_DIRS ${MSR_INCLUDE_DIR}) +set(Msr_LIBRARIES ${Msr_LIBRARY}) +set(Msr_INCLUDE_DIRS ${Msr_INCLUDE_DIR}) -find_package_handle_standard_args(MSR DEFAULT_MSG MSR_LIBRARY MSR_INCLUDE_DIR) +find_package_handle_standard_args(MSR DEFAULT_MSG Msr_LIBRARY Msr_INCLUDE_DIR) get_property( _type - CACHE MSR_ROOT + CACHE Msr_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE MSR_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Msr_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE MSR_ROOT PROPERTY TYPE PATH) + set_property(CACHE Msr_ROOT PROPERTY TYPE PATH) endif() endif() -mark_as_advanced(MSR_ROOT MSR_LIBRARY MSR_INCLUDE_DIR) +mark_as_advanced(Msr_ROOT Msr_LIBRARY Msr_INCLUDE_DIR) diff --git a/cmake/FindOrangeFS.cmake b/cmake/FindOrangeFS.cmake index df968570217d..36caaaaa0f26 100644 --- a/cmake/FindOrangeFS.cmake +++ b/cmake/FindOrangeFS.cmake @@ -5,44 +5,53 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(ORANGEFS_ROOT AND NOT Orangefs_ROOT) + set(Orangefs_ROOT + ${ORANGEFS_ROOT} + CACHE PATH "OrangeFS base directory" + ) + unset(ORANGEFS_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) -pkg_check_modules(PC_ORANGEFS QUIET orangefs) +pkg_check_modules(PC_Orangefs QUIET orangefs) find_path( - ORANGEFS_INCLUDE_DIR + Orangefs_INCLUDE_DIR NAMES pxfs.h orange.h - HINTS ${ORANGEFS_ROOT} ENV ORANGEFS_ROOT ${PC_ORANGEFS_INCLUDEDIR} - ${PC_ORANGEFS_INCLUDE_DIRS} + HINTS ${Orangefs_ROOT} ENV Orangefs_ROOT ${PC_Orangefs_INCLUDEDIR} + ${PC_Orangefs_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - ORANGEFS_LIBRARY + Orangefs_LIBRARY NAMES pvfs2 # orangefs pvfs2 orangefsposix - HINTS ${ORANGEFS_ROOT} ENV ORANGEFS_ROOT ${PC_ORANGEFS_LIBDIR} - ${PC_ORANGEFS_LIBRARY_DIRS} + HINTS ${Orangefs_ROOT} ENV ORANGEFS_ROOT ${PC_Orangefs_LIBDIR} + ${PC_Orangefs_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set ORANGEFS_ROOT in case the other hints are used -if(ORANGEFS_ROOT) +# Set Orangefs_ROOT in case the other hints are used +if(Orangefs_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${ORANGEFS_ROOT} ORANGEFS_ROOT) + file(TO_CMAKE_PATH ${Orangefs_ROOT} Orangefs_ROOT) elseif("$ENV{ORANGEFS_ROOT}") - file(TO_CMAKE_PATH $ENV{ORANGEFS_ROOT} ORANGEFS_ROOT) + file(TO_CMAKE_PATH $ENV{ORANGEFS_ROOT} Orangefs_ROOT) else() - file(TO_CMAKE_PATH "${ORANGEFS_INCLUDE_DIR}" ORANGEFS_INCLUDE_DIR) - string(REPLACE "/include" "" ORANGEFS_ROOT "${ORANGEFS_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Orangefs_INCLUDE_DIR}" Orangefs_INCLUDE_DIR) + string(REPLACE "/include" "" Orangefs_ROOT "${Orangefs_INCLUDE_DIR}") endif() -set(ORANGEFS_LIBRARIES ${ORANGEFS_LIBRARY}) -set(ORANGEFS_INCLUDE_DIRS ${ORANGEFS_INCLUDE_DIR}) +set(Orangefs_LIBRARIES ${Orangefs_LIBRARY}) +set(Orangefs_INCLUDE_DIRS ${Orangefs_INCLUDE_DIR}) find_package_handle_standard_args( - OrangeFS DEFAULT_MSG ORANGEFS_LIBRARY ORANGEFS_INCLUDE_DIR + OrangeFS DEFAULT_MSG Orangefs_LIBRARY Orangefs_INCLUDE_DIR ) -foreach(v ORANGEFS_ROOT) +foreach(v Orangefs_ROOT) get_property( _type CACHE ${v} @@ -56,4 +65,4 @@ foreach(v ORANGEFS_ROOT) endif() endforeach() -mark_as_advanced(ORANGEFS_ROOT ORANGEFS_LIBRARY ORANGEFS_INCLUDE_DIR) +mark_as_advanced(Orangefs_ROOT Orangefs_LIBRARY Orangefs_INCLUDE_DIR) diff --git a/cmake/FindPAPI.cmake b/cmake/FindPAPI.cmake deleted file mode 100644 index 9f6d906b023e..000000000000 --- a/cmake/FindPAPI.cmake +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2014 Thomas Heller -# Copyright (c) 2007-2011 Hartmut Kaiser -# Copyright (c) 2011 Bryce Lelbach -# Copyright (c) 2011 Maciej Brodowicz -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -if(NOT TARGET Papi::papi) - find_package(PkgConfig QUIET) - pkg_check_modules(PC_PAPI QUIET papi) - - find_path( - PAPI_INCLUDE_DIR papi.h - HINTS ${PAPI_ROOT} ENV PAPI_ROOT ${HPX_PAPI_ROOT} ${PC_PAPI_INCLUDEDIR} - ${PC_PAPI_INCLUDE_DIRS} - PATH_SUFFIXES include - ) - - find_library( - PAPI_LIBRARY - NAMES papi libpapi - HINTS ${PAPI_ROOT} ENV PAPI_ROOT ${HPX_PAPI_ROOT} ${PC_PAPI_LIBDIR} - ${PC_PAPI_LIBRARY_DIRS} - PATH_SUFFIXES lib lib64 - ) - - # Set PAPI_ROOT in case the other hints are used - if(NOT PAPI_ROOT AND "$ENV{PAPI_ROOT}") - set(PAPI_ROOT $ENV{PAPI_ROOT}) - elseif(NOT PAPI_ROOT) - string(REPLACE "/include" "" PAPI_ROOT "${PAPI_INCLUDE_DIR}") - endif() - - # Set PAPI_ROOT in case the other hints are used - if(PAPI_ROOT) - # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${PAPI_ROOT} PAPI_ROOT) - elseif("$ENV{PAPI_ROOT}") - file(TO_CMAKE_PATH $ENV{PAPI_ROOT} PAPI_ROOT) - else() - file(TO_CMAKE_PATH "${PAPI_INCLUDE_DIR}" PAPI_INCLUDE_DIR) - string(REPLACE "/include" "" PAPI_ROOT "${PAPI_INCLUDE_DIR}") - endif() - - set(PAPI_LIBRARIES ${PAPI_LIBRARY}) - set(PAPI_INCLUDE_DIRS ${PAPI_INCLUDE_DIR}) - - find_package_handle_standard_args( - PAPI DEFAULT_MSG PAPI_LIBRARY PAPI_INCLUDE_DIR - ) - - get_property( - _type - CACHE PAPI_ROOT - PROPERTY TYPE - ) - if(_type) - set_property(CACHE PAPI_ROOT PROPERTY ADVANCED 1) - if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE PAPI_ROOT PROPERTY TYPE PATH) - endif() - endif() - - mark_as_advanced(PAPI_ROOT PAPI_LIBRARY PAPI_INCLUDE_DIR) - - add_library(Papi::papi INTERFACE IMPORTED) - target_include_directories(Papi::papi SYSTEM INTERFACE ${PAPI_INCLUDE_DIR}) - target_link_libraries(Papi::papi INTERFACE ${PAPI_LIBRARY}) -endif() diff --git a/cmake/FindPWR.cmake b/cmake/FindPWR.cmake index edcadc73cf79..0007fa6008dc 100644 --- a/cmake/FindPWR.cmake +++ b/cmake/FindPWR.cmake @@ -8,70 +8,79 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(NOT TARGET PWR::pwr) + # compatibility with older CMake versions + if(PWR_ROOT AND NOT Pwr_ROOT) + set(Pwr_ROOT + ${PWR_ROOT} + CACHE PATH "PWR base directory" + ) + unset(PWR_ROOT CACHE) + endif() + find_package(PkgConfig QUIET) - pkg_check_modules(PC_PWR QUIET pwr) - if(NOT PC_PWR_FOUND) - pkg_check_modules(PC_PWR pwrapi QUIET) + pkg_check_modules(PC_Pwr QUIET pwr) + if(NOT PC_Pwr_FOUND) + pkg_check_modules(PC_Pwr pwrapi QUIET) endif() find_path( - PWR_INCLUDE_DIR pwr.h - HINTS ${PWR_ROOT} + Pwr_INCLUDE_DIR pwr.h + HINTS ${Pwr_ROOT} ENV PWR_ROOT - ${HPX_PWR_ROOT} - ${PC_PWR_MINIMAL_INCLUDEDIR} - ${PC_PWR_MINIMAL_INCLUDE_DIRS} - ${PC_PWR_INCLUDEDIR} - ${PC_PWR_INCLUDE_DIRS} + ${HPX_Pwr_ROOT} + ${PC_Pwr_MINIMAL_INCLUDEDIR} + ${PC_Pwr_MINIMAL_INCLUDE_DIRS} + ${PC_Pwr_INCLUDEDIR} + ${PC_Pwr_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - PWR_LIBRARY + Pwr_LIBRARY NAMES pwr libpwr - HINTS ${PWR_ROOT} + HINTS ${Pwr_ROOT} ENV PWR_ROOT - ${HPX_PWR_ROOT} - ${PC_PWR_MINIMAL_LIBDIR} - ${PC_PWR_MINIMAL_LIBRARY_DIRS} - ${PC_PWR_LIBDIR} - ${PC_PWR_LIBRARY_DIRS} + ${HPX_Pwr_ROOT} + ${PC_Pwr_MINIMAL_LIBDIR} + ${PC_Pwr_MINIMAL_LIBRARY_DIRS} + ${PC_Pwr_LIBDIR} + ${PC_Pwr_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) - # Set PWR_ROOT in case the other hints are used - if(PWR_ROOT) + # Set Pwr_ROOT in case the other hints are used + if(Pwr_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${PWR_ROOT} PWR_ROOT) + file(TO_CMAKE_PATH ${Pwr_ROOT} Pwr_ROOT) elseif("$ENV{PWR_ROOT}") - file(TO_CMAKE_PATH $ENV{PWR_ROOT} PWR_ROOT) + file(TO_CMAKE_PATH $ENV{PWR_ROOT} Pwr_ROOT) else() - file(TO_CMAKE_PATH "${PWR_INCLUDE_DIR}" PWR_INCLUDE_DIR) - string(REPLACE "/include" "" PWR_ROOT "${PWR_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Pwr_INCLUDE_DIR}" Pwr_INCLUDE_DIR) + string(REPLACE "/include" "" Pwr_ROOT "${Pwr_INCLUDE_DIR}") endif() - set(PWR_LIBRARIES ${PWR_LIBRARY}) - set(PWR_INCLUDE_DIRS ${PWR_INCLUDE_DIR}) + set(Pwr_LIBRARIES ${Pwr_LIBRARY}) + set(Pwr_INCLUDE_DIRS ${Pwr_INCLUDE_DIR}) - find_package_handle_standard_args(PWR DEFAULT_MSG PWR_LIBRARY PWR_INCLUDE_DIR) + find_package_handle_standard_args(PWR DEFAULT_MSG Pwr_LIBRARY Pwr_INCLUDE_DIR) get_property( _type - CACHE PWR_ROOT + CACHE Pwr_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE PWR_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Pwr_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE PWR_ROOT PROPERTY TYPE PATH) + set_property(CACHE Pwr_ROOT PROPERTY TYPE PATH) endif() endif() add_library(PWR::pwr INTERFACE IMPORTED) - target_include_directories(PWR::pwr SYSTEM INTERFACE ${PWR_INCLUDE_DIR}) - target_link_libraries(PWR::pwr INTERFACE ${PWR_LIBRARIES}) + target_include_directories(PWR::pwr SYSTEM INTERFACE ${Pwr_INCLUDE_DIR}) + target_link_libraries(PWR::pwr INTERFACE ${Pwr_LIBRARIES}) - mark_as_advanced(PWR_ROOT PWR_LIBRARY PWR_INCLUDE_DIR) + mark_as_advanced(Pwr_ROOT Pwr_LIBRARY Pwr_INCLUDE_DIR) endif() diff --git a/cmake/FindPapi.cmake b/cmake/FindPapi.cmake new file mode 100644 index 000000000000..b0a4dbf4cf2e --- /dev/null +++ b/cmake/FindPapi.cmake @@ -0,0 +1,88 @@ +# Copyright (c) 2014 Thomas Heller +# Copyright (c) 2007-2023 Hartmut Kaiser +# Copyright (c) 2011 Bryce Lelbach +# Copyright (c) 2011 Maciej Brodowicz +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT TARGET Papi::papi) + # compatibility with older CMake versions + if(PAPI_ROOT AND NOT Papi_ROOT) + set(Papi_ROOT + ${PAPI_ROOT} + CACHE PATH "PAPI base directory" + ) + unset(PAPI_ROOT CACHE) + endif() + + find_package(PkgConfig QUIET) + pkg_check_modules(PC_Papi QUIET papi) + + find_path( + Papi_INCLUDE_DIR papi.h + HINTS ${Papi_ROOT} ENV PAPI_ROOT ${HPX_PAPI_ROOT} ${PC_Papi_INCLUDEDIR} + ${PC_Papi_INCLUDE_DIRS} + PATH_SUFFIXES include + ) + + find_library( + Papi_LIBRARY + NAMES papi libpapi + HINTS ${Papi_ROOT} ENV PAPI_ROOT ${HPX_PAPI_ROOT} ${PC_Papi_LIBDIR} + ${PC_Papi_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64 + ) + + # Set Papi_ROOT in case the other hints are used + if(NOT Papi_ROOT AND "$ENV{PAPI_ROOT}") + set(Papi_ROOT $ENV{PAPI_ROOT}) + elseif(NOT Papi_ROOT) + string(REPLACE "/include" "" Papi_ROOT "${Papi_INCLUDE_DIR}") + endif() + + # Set Papi_ROOT in case the other hints are used + if(Papi_ROOT) + # The call to file is for compatibility with windows paths + file(TO_CMAKE_PATH ${Papi_ROOT} Papi_ROOT) + elseif("$ENV{PAPI_ROOT}") + file(TO_CMAKE_PATH $ENV{PAPI_ROOT} Papi_ROOT) + else() + file(TO_CMAKE_PATH "${Papi_INCLUDE_DIR}" Papi_INCLUDE_DIR) + string(REPLACE "/include" "" Papi_ROOT "${Papi_INCLUDE_DIR}") + endif() + + set(Papi_LIBRARIES ${Papi_LIBRARY}) + set(Papi_INCLUDE_DIRS ${Papi_INCLUDE_DIR}) + + find_package_handle_standard_args( + Papi + REQUIRED_VARS Papi_LIBRARY Papi_INCLUDE_DIR + FOUND_VAR Papi_FOUND + ) + + if(NOT Papi_FOUND) + message(FATAL "The PAPI library could not be found using Papi_LIBRARY: " + ${Papi_LIBRARY} "and Papi_INCLUDE_DIR: " ${Papi_INCLUDE_DIR} + ) + endif() + + get_property( + _type + CACHE Papi_ROOT + PROPERTY TYPE + ) + if(_type) + set_property(CACHE Papi_ROOT PROPERTY ADVANCED 1) + if("x${_type}" STREQUAL "xUNINITIALIZED") + set_property(CACHE Papi_ROOT PROPERTY TYPE PATH) + endif() + endif() + + mark_as_advanced(Papi_ROOT Papi_LIBRARY Papi_INCLUDE_DIR) + + add_library(Papi::papi INTERFACE IMPORTED) + target_include_directories(Papi::papi SYSTEM INTERFACE ${Papi_INCLUDE_DIR}) + target_link_libraries(Papi::papi INTERFACE ${Papi_LIBRARY}) +endif() diff --git a/cmake/FindQThreads.cmake b/cmake/FindQThreads.cmake index d731aab6b8ba..08e2342521fb 100644 --- a/cmake/FindQThreads.cmake +++ b/cmake/FindQThreads.cmake @@ -5,43 +5,52 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(QTHREADS_ROOT AND NOT Qthreads_ROOT) + set(Qthreads_ROOT + ${QTHREADS_ROOT} + CACHE PATH "QThreads base directory" + ) + unset(QTHREADS_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) -pkg_check_modules(PC_QTHREADS QUIET swarm) +pkg_check_modules(PC_Qthreads QUIET swarm) find_path( - QTHREADS_INCLUDE_DIR qthread/qthread.h - HINTS ${QTHREADS_ROOT} ENV QTHREADS_ROOT ${PC_QTHREADS_INCLUDEDIR} - ${PC_QTHREADS_INCLUDE_DIRS} + Qthreads_INCLUDE_DIR qthread/qthread.h + HINTS ${Qthreads_ROOT} ENV QTHREADS_ROOT ${PC_Qthreads_INCLUDEDIR} + ${PC_Qthreads_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - QTHREADS_LIBRARY + Qthreads_LIBRARY NAMES qthread libqthread - HINTS ${QTHREADS_ROOT} ENV QTHREADS_ROOT ${PC_QTHREADS_LIBDIR} - ${PC_QTHREADS_LIBRARY_DIRS} + HINTS ${Qthreads_ROOT} ENV QTHREADS_ROOT ${PC_Qthreads_LIBDIR} + ${PC_Qthreads_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set QTHREADS_ROOT in case the other hints are used -if(QTHREADS_ROOT) +# Set Qthreads_ROOT in case the other hints are used +if(Qthreads_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${QTHREADS_ROOT} QTHREADS_ROOT) + file(TO_CMAKE_PATH ${Qthreads_ROOT} Qthreads_ROOT) elseif("$ENV{QTHREADS_ROOT}") - file(TO_CMAKE_PATH $ENV{QTHREADS_ROOT} QTHREADS_ROOT) + file(TO_CMAKE_PATH $ENV{QTHREADS_ROOT} Qthreads_ROOT) else() - file(TO_CMAKE_PATH "${QTHREADS_INCLUDE_DIR}" QTHREADS_INCLUDE_DIR) - string(REPLACE "/include" "" QTHREADS_ROOT "${QTHREADS_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Qthreads_INCLUDE_DIR}" Qthreads_INCLUDE_DIR) + string(REPLACE "/include" "" Qthreads_ROOT "${Qthreads_INCLUDE_DIR}") endif() -set(QTHREADS_LIBRARIES ${QTHREADS_LIBRARY}) -set(QTHREADS_INCLUDE_DIRS ${QTHREADS_INCLUDE_DIR}) +set(Qthreads_LIBRARIES ${Qthreads_LIBRARY}) +set(Qthreads_INCLUDE_DIRS ${Qthreads_INCLUDE_DIR}) find_package_handle_standard_args( - QThreads DEFAULT_MSG QTHREADS_LIBRARY QTHREADS_INCLUDE_DIR + QThreads DEFAULT_MSG Qthreads_LIBRARY Qthreads_INCLUDE_DIR ) -foreach(v QTHREADS_ROOT) +foreach(v Qthreads_ROOT) get_property( _type CACHE ${v} @@ -55,4 +64,4 @@ foreach(v QTHREADS_ROOT) endif() endforeach() -mark_as_advanced(QTHREADS_ROOT QTHREADS_LIBRARY QTHREADS_INCLUDE_DIR) +mark_as_advanced(Qthreads_ROOT Qthreads_LIBRARY Qthreads_INCLUDE_DIR) diff --git a/cmake/FindRDMA_CM.cmake b/cmake/FindRDMA_CM.cmake index 1a33f36e5db3..3f0d210a4213 100644 --- a/cmake/FindRDMA_CM.cmake +++ b/cmake/FindRDMA_CM.cmake @@ -6,28 +6,38 @@ # # - Try to find RDMA CM # Once done this will define -# RDMA_CM_FOUND - System has RDMA CM -# RDMA_CM_INCLUDE_DIRS - The RDMA CM include directories -# RDMA_CM_LIBRARIES - The libraries needed to use RDMA CM +# Rdma_CM_FOUND - System has RDMA CM +# Rdma_CM_INCLUDE_DIRS - The RDMA CM include directories +# Rdma_CM_LIBRARIES - The libraries needed to use RDMA CM -find_path(RDMA_CM_INCLUDE_DIR rdma_cma.h HINTS /usr/local/include +# compatibility with older CMake versions +if(RDMA_ROOT AND NOT Rdma_ROOT) + set(Rdma_ROOT + ${RDMA_ROOT} + CACHE PATH "RDMA base directory" + ) + unset(RDMA_ROOT CACHE) +endif() + +find_path(Rdma_CM_INCLUDE_DIR rdma_cma.h HINTS /usr/local/include /usr/include/rdma ) find_library( - RDMA_CM_LIBRARY + Rdma_CM_LIBRARY NAMES rdmacm PATHS /usr/local/lib /usr/lib ) -set(RDMA_CM_INCLUDE_DIRS ${RDMA_CM_INCLUDE_DIR}) -set(RDMA_CM_LIBRARIES ${RDMA_CM_LIBRARY}) +set(Rdma_CM_INCLUDE_DIRS ${Rdma_CM_INCLUDE_DIR}) +set(Rdma_CM_LIBRARIES ${Rdma_CM_LIBRARY}) include(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set RDMA_CM_FOUND to TRUE if all + +# handle the QUIETLY and REQUIRED arguments and set Rdma_CM_FOUND to TRUE if all # listed variables are TRUE find_package_handle_standard_args( - RDMA_CM DEFAULT_MSG RDMA_CM_INCLUDE_DIR RDMA_CM_LIBRARY + Rdma_CM DEFAULT_MSG Rdma_CM_INCLUDE_DIR Rdma_CM_LIBRARY ) -mark_as_advanced(RDMA_CM_INCLUDE_DIR RDMA_CM_LIBRARY) +mark_as_advanced(Rdma_CM_INCLUDE_DIR Rdma_CM_LIBRARY) diff --git a/cmake/FindRdmacm.cmake b/cmake/FindRdmacm.cmake index 2eb9896125e6..5513ea4bd646 100644 --- a/cmake/FindRdmacm.cmake +++ b/cmake/FindRdmacm.cmake @@ -7,38 +7,47 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(RDMACM_ROOT AND NOT Rdmacm_ROOT) + set(Rdmacm_ROOT + ${RDMACM_ROOT} + CACHE PATH "RDMACM base directory" + ) + unset(RDMACM_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) -pkg_check_modules(PC_RDMACM QUIET libibverbs) +pkg_check_modules(PC_Rdmacm QUIET libibverbs) find_path( - RDMACM_INCLUDE_DIR rdma/rdma_cma.h - HINTS ${RDMACM_ROOT} ENV RDMACM_ROOT ${PC_RDMACM_INCLUDEDIR} - ${PC_RDMACM_INCLUDE_DIRS} + Rdmacm_INCLUDE_DIR rdma/rdma_cma.h + HINTS ${Rdmacm_ROOT} ENV RDMACM_ROOT ${PC_Rdmacm_INCLUDEDIR} + ${PC_Rdmacm_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - RDMACM_LIBRARY + Rdmacm_LIBRARY NAMES rdmacm librdmacm - HINTS ${RDMACM_ROOT} ENV RDMACM_ROOT ${PC_RDMACM_LIBDIR} - ${PC_RDMACM_LIBRARY_DIRS} + HINTS ${Rdmacm_ROOT} ENV RDMACM_ROOT ${PC_Rdmacm_LIBDIR} + ${PC_Rdmacm_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -set(RDMACM_LIBRARIES - ${RDMACM_LIBRARY} +set(Rdmacm_LIBRARIES + ${Rdmacm_LIBRARY} CACHE INTERNAL "" ) -set(RDMACM_INCLUDE_DIRS - ${RDMACM_INCLUDE_DIR} +set(Rdmacm_INCLUDE_DIRS + ${Rdmacm_INCLUDE_DIR} CACHE INTERNAL "" ) find_package_handle_standard_args( - Rdmacm DEFAULT_MSG RDMACM_LIBRARY RDMACM_INCLUDE_DIR + Rdmacm DEFAULT_MSG Rdmacm_LIBRARY Rdmacm_INCLUDE_DIR ) -foreach(v RDMACM_ROOT) +foreach(v Rdmacm_ROOT) get_property( _type CACHE ${v} @@ -52,4 +61,4 @@ foreach(v RDMACM_ROOT) endif() endforeach() -mark_as_advanced(RDMACM_ROOT RDMACM_LIBRARY RDMACM_INCLUDE_DIR) +mark_as_advanced(Rdmacm_ROOT Rdmacm_LIBRARY Rdmacm_INCLUDE_DIR) diff --git a/cmake/FindSnappy.cmake b/cmake/FindSnappy.cmake index 58cee9ca47e0..0374be7dff41 100644 --- a/cmake/FindSnappy.cmake +++ b/cmake/FindSnappy.cmake @@ -1,55 +1,64 @@ # Copyright (c) 2014 Thomas Heller -# Copyright (c) 2013 Hartmut Kaiser +# Copyright (c) 2013-2023 Hartmut Kaiser # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(SNAPPY_ROOT AND NOT Snappy_ROOT) + set(Snappy_ROOT + ${SNAPPY_ROOT} + CACHE PATH "Snappy base directory" + ) + unset(SNAPPY_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_SNAPPY QUIET snappy) find_path( - SNAPPY_INCLUDE_DIR snappy.h - HINTS ${SNAPPY_ROOT} + Snappy_INCLUDE_DIR snappy.h + HINTS ${Snappy_ROOT} ENV SNAPPY_ROOT - ${PC_SNAPPY_MINIMAL_INCLUDEDIR} - ${PC_SNAPPY_MINIMAL_INCLUDE_DIRS} - ${PC_SNAPPY_INCLUDEDIR} - ${PC_SNAPPY_INCLUDE_DIRS} + ${PC_Snappy_MINIMAL_INCLUDEDIR} + ${PC_Snappy_MINIMAL_INCLUDE_DIRS} + ${PC_Snappy_INCLUDEDIR} + ${PC_Snappy_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - SNAPPY_LIBRARY + Snappy_LIBRARY NAMES snappy libsnappy - HINTS ${SNAPPY_ROOT} + HINTS ${Snappy_ROOT} ENV SNAPPY_ROOT - ${PC_SNAPPY_MINIMAL_LIBDIR} - ${PC_SNAPPY_MINIMAL_LIBRARY_DIRS} - ${PC_SNAPPY_LIBDIR} - ${PC_SNAPPY_LIBRARY_DIRS} + ${PC_Snappy_MINIMAL_LIBDIR} + ${PC_Snappy_MINIMAL_LIBRARY_DIRS} + ${PC_Snappy_LIBDIR} + ${PC_Snappy_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -set(SNAPPY_LIBRARIES ${SNAPPY_LIBRARY}) -set(SNAPPY_INCLUDE_DIRS ${SNAPPY_INCLUDE_DIR}) +set(Snappy_LIBRARIES ${Snappy_LIBRARY}) +set(Snappy_INCLUDE_DIRS ${Snappy_INCLUDE_DIR}) find_package_handle_standard_args( - Snappy DEFAULT_MSG SNAPPY_LIBRARY SNAPPY_INCLUDE_DIR + Snappy DEFAULT_MSG Snappy_LIBRARY Snappy_INCLUDE_DIR ) get_property( _type - CACHE SNAPPY_ROOT + CACHE Snappy_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE SNAPPY_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Snappy_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE SNAPPY_ROOT PROPERTY TYPE PATH) + set_property(CACHE Snappy_ROOT PROPERTY TYPE PATH) endif() endif() -mark_as_advanced(SNAPPY_ROOT SNAPPY_LIBRARY SNAPPY_INCLUDE_DIR) +mark_as_advanced(Snappy_ROOT Snappy_LIBRARY Snappy_INCLUDE_DIR) diff --git a/cmake/FindSphinx.cmake b/cmake/FindSphinx.cmake index 9c5f66180e97..9b9e58ec6edd 100644 --- a/cmake/FindSphinx.cmake +++ b/cmake/FindSphinx.cmake @@ -4,14 +4,23 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(SPHINX_ROOT AND NOT Sphinx_ROOT) + set(Sphinx_ROOT + ${SPHINX_ROOT} + CACHE PATH "Sphinx base directory" + ) + unset(SPHINX_ROOT CACHE) +endif() + find_program( - SPHINX_EXECUTABLE + Sphinx_EXECUTABLE NAMES sphinx-build sphinx-build2 - PATHS ${SPHINX_ROOT} ENV SPHINX_ROOT + PATHS ${Sphinx_ROOT} ENV SPHINX_ROOT DOC "Path to sphinx-build executable" ) -if(SPHINX_EXECUTABLE) +if(Sphinx_EXECUTABLE) include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Sphinx DEFAULT_MESSAGE SPHINX_EXECUTABLE) + find_package_handle_standard_args(Sphinx DEFAULT_MESSAGE Sphinx_EXECUTABLE) endif() diff --git a/cmake/FindTBB.cmake b/cmake/FindTBB.cmake index 648e9587f43b..821117c52aba 100644 --- a/cmake/FindTBB.cmake +++ b/cmake/FindTBB.cmake @@ -5,38 +5,47 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(TBB_ROOT AND NOT Tbb_ROOT) + set(Tbb_ROOT + ${TBB_ROOT} + CACHE PATH "TBB base directory" + ) + unset(TBB_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) -pkg_check_modules(PC_TBB QUIET libtbb) +pkg_check_modules(PC_Tbb QUIET libtbb) find_path( - TBB_INCLUDE_DIR tbb/tbb.h - HINTS ${TBB_ROOT} ENV TBB_ROOT ${PC_TBB_INCLUDEDIR} ${PC_TBB_INCLUDE_DIRS} + Tbb_INCLUDE_DIR tbb/tbb.h + HINTS ${Tbb_ROOT} ENV TBB_ROOT ${PC_Tbb_INCLUDEDIR} ${PC_Tbb_INCLUDE_DIRS} PATH_SUFFIXES include ) -set(TBB_PATH_SUFFIX "lib/intel64" "lib/intel64/gcc4.4") -if(TBB_PLATFORM STREQUAL "mic") - set(TBB_PATH_SUFFIX "lib/mic") +set(Tbb_PATH_SUFFIX "lib/intel64" "lib/intel64/gcc4.4") +if(Tbb_PLATFORM STREQUAL "mic") + set(Tbb_PATH_SUFFIX "lib/mic") endif() -if(TBB_PLATFORM STREQUAL "mic-knl") - set(TBB_PATH_SUFFIX "lib/intel64_lin_mic") +if(Tbb_PLATFORM STREQUAL "mic-knl") + set(Tbb_PATH_SUFFIX "lib/intel64_lin_mic") endif() find_library( - TBB_PROXY_LIBRARY + Tbb_PROXY_LIBRARY NAMES tbb libtbb - HINTS ${TBB_ROOT} ENV TBB_ROOT ${PC_TBB_LIBDIR} ${PC_TBB_LIBRARY_DIRS} - PATH_SUFFIXES ${TBB_PATH_SUFFIX} lib lib64 + HINTS ${Tbb_ROOT} ENV TBB_ROOT ${PC_Tbb_LIBDIR} ${PC_Tbb_LIBRARY_DIRS} + PATH_SUFFIXES ${Tbb_PATH_SUFFIX} lib lib64 ) -set(TBB_LIBRARIES ${TBB_LIBRARY} ${TBB_PROXY_LIBRARY}) -set(TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR}) +set(Tbb_LIBRARIES ${Tbb_LIBRARY} ${Tbb_PROXY_LIBRARY}) +set(Tbb_INCLUDE_DIRS ${Tbb_INCLUDE_DIR}) find_package_handle_standard_args( - TBBmalloc DEFAULT_MSG TBB_LIBRARY TBB_PROXY_LIBRARY TBB_INCLUDE_DIR + TBBmalloc DEFAULT_MSG Tbb_LIBRARY Tbb_PROXY_LIBRARY Tbb_INCLUDE_DIR ) -foreach(v TBB_ROOT TBB_PLATFORM) +foreach(v Tbb_ROOT Tbb_PLATFORM) get_property( _type CACHE ${v} @@ -50,4 +59,4 @@ foreach(v TBB_ROOT TBB_PLATFORM) endif() endforeach() -mark_as_advanced(TBB_ROOT TBB_LIBRARY TBB_PROXY_LIBRARY TBB_INCLUDE_DIR) +mark_as_advanced(Tbb_ROOT Tbb_LIBRARY Tbb_PROXY_LIBRARY Tbb_INCLUDE_DIR) diff --git a/cmake/FindTBBmalloc.cmake b/cmake/FindTBBmalloc.cmake index 1329597fba40..9b029ac51068 100644 --- a/cmake/FindTBBmalloc.cmake +++ b/cmake/FindTBBmalloc.cmake @@ -7,58 +7,67 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(TBBMALLOC_ROOT AND NOT Tbbmalloc_ROOT) + set(Tbbmalloc_ROOT + ${TBBMALLOC_ROOT} + CACHE PATH "TBBMalloc base directory" + ) + unset(TBBMALLOC_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) -pkg_check_modules(PC_TBBMALLOC QUIET libtbbmalloc) +pkg_check_modules(PC_Tbbmalloc QUIET libtbbmalloc) find_path( - TBBMALLOC_INCLUDE_DIR tbb/scalable_allocator.h - HINTS ${TBBMALLOC_ROOT} ENV TBBMALLOC_ROOT ${HPX_TBBMALLOC_ROOT} - ${PC_TBBMALLOC_INCLUDEDIR} ${PC_TBBMALLOC_INCLUDE_DIRS} + Tbbmalloc_INCLUDE_DIR tbb/scalable_allocator.h + HINTS ${Tbbmalloc_ROOT} ENV TBBMALLOC_ROOT ${HPX_TBBMALLOC_ROOT} + ${PC_Tbbmalloc_INCLUDEDIR} ${PC_Tbbmalloc_INCLUDE_DIRS} PATH_SUFFIXES include ) -set(TBBMALLOC_PATH_SUFFIX "lib/intel64" "lib/intel64/gcc4.4") -if(TBBMALLOC_PLATFORM STREQUAL "mic") - set(TBBMALLOC_PATH_SUFFIX "lib/mic") +set(Tbbmalloc_PATH_SUFFIX "lib/intel64" "lib/intel64/gcc4.4") +if(Tbbmalloc_PLATFORM STREQUAL "mic") + set(Tbbmalloc_PATH_SUFFIX "lib/mic") endif() -if(TBBMALLOC_PLATFORM STREQUAL "mic-knl") - set(TBBMALLOC_PATH_SUFFIX "lib/intel64_lin_mic") +if(Tbbmalloc_PLATFORM STREQUAL "mic-knl") + set(Tbbmalloc_PATH_SUFFIX "lib/intel64_lin_mic") endif() -message("${TBBMALLOC_ROOT} ${TBBMALLOC_PATH_SUFFIX} ${TBBMALLOC_PLATFORM}") +message("${Tbbmalloc_ROOT} ${Tbbmalloc_PATH_SUFFIX} ${Tbbmalloc_PLATFORM}") find_library( - TBBMALLOC_LIBRARY + Tbbmalloc_LIBRARY NAMES tbbmalloc libtbbmalloc - HINTS ${TBBMALLOC_ROOT} ENV TBBMALLOC_ROOT ${HPX_TBBMALLOC_ROOT} - ${PC_TBBMALLOC_LIBDIR} ${PC_TBBMALLOC_LIBRARY_DIRS} - PATH_SUFFIXES ${TBBMALLOC_PATH_SUFFIX} lib lib64 + HINTS ${Tbbmalloc_ROOT} ENV TBBMALLOC_ROOT ${HPX_TBBMALLOC_ROOT} + ${PC_Tbbmalloc_LIBDIR} ${PC_Tbbmalloc_LIBRARY_DIRS} + PATH_SUFFIXES ${Tbbmalloc_PATH_SUFFIX} lib lib64 ) find_library( - TBBMALLOC_PROXY_LIBRARY + Tbbmalloc_PROXY_LIBRARY NAMES tbbmalloc_proxy libtbbmalloc_proxy - HINTS ${TBBMALLOC_ROOT} ENV TBBMALLOC_ROOT ${HPX_TBBMALLOC_ROOT} - ${PC_TBBMALLOC_LIBDIR} ${PC_TBBMALLOC_LIBRARY_DIRS} - PATH_SUFFIXES ${TBBMALLOC_PATH_SUFFIX} lib lib64 + HINTS ${Tbbmalloc_ROOT} ENV TBBMALLOC_ROOT ${HPX_TBBMALLOC_ROOT} + ${PC_Tbbmalloc_LIBDIR} ${PC_Tbbmalloc_LIBRARY_DIRS} + PATH_SUFFIXES ${Tbbmalloc_PATH_SUFFIX} lib lib64 ) -# Set TBBMALLOC_ROOT in case the other hints are used -if(NOT TBBMALLOC_ROOT AND "$ENV{TBBMALLOC_ROOT}") - set(TBBMALLOC_ROOT $ENV{TBBMALLOC_ROOT}) -elseif(NOT TBBMALLOC_ROOT) - string(REPLACE "/include" "" TBBMALLOC_ROOT "${TBBMALLOC_INCLUDE_DIR}") +# Set Tbbmalloc_ROOT in case the other hints are used +if(NOT Tbbmalloc_ROOT AND "$ENV{TBBMALLOC_ROOT}") + set(Tbbmalloc_ROOT $ENV{TBBMALLOC_ROOT}) +elseif(NOT Tbbmalloc_ROOT) + string(REPLACE "/include" "" Tbbmalloc_ROOT "${Tbbmalloc_INCLUDE_DIR}") endif() -set(TBBMALLOC_LIBRARIES ${TBBMALLOC_LIBRARY} ${TBBMALLOC_PROXY_LIBRARY}) -set(TBBMALLOC_INCLUDE_DIRS ${TBBMALLOC_INCLUDE_DIR}) +set(Tbbmalloc_LIBRARIES ${Tbbmalloc_LIBRARY} ${Tbbmalloc_PROXY_LIBRARY}) +set(Tbbmalloc_INCLUDE_DIRS ${Tbbmalloc_INCLUDE_DIR}) find_package_handle_standard_args( - TBBmalloc DEFAULT_MSG TBBMALLOC_LIBRARY TBBMALLOC_PROXY_LIBRARY - TBBMALLOC_INCLUDE_DIR + TBBmalloc DEFAULT_MSG Tbbmalloc_LIBRARY Tbbmalloc_PROXY_LIBRARY + Tbbmalloc_INCLUDE_DIR ) -foreach(v TBBMALLOC_ROOT TBBMALLOC_PLATFORM) +foreach(v Tbbmalloc_ROOT Tbbmalloc_PLATFORM) get_property( _type CACHE ${v} @@ -73,6 +82,6 @@ foreach(v TBBMALLOC_ROOT TBBMALLOC_PLATFORM) endforeach() mark_as_advanced( - TBBMALLOC_ROOT TBBMALLOC_LIBRARY TBBMALLOC_PROXY_LIBRARY - TBBMALLOC_INCLUDE_DIR + Tbbmalloc_ROOT Tbbmalloc_LIBRARY Tbbmalloc_PROXY_LIBRARY + Tbbmalloc_INCLUDE_DIR ) diff --git a/cmake/FindTCMalloc.cmake b/cmake/FindTCMalloc.cmake index 9d33a5559150..8503410b8d94 100644 --- a/cmake/FindTCMalloc.cmake +++ b/cmake/FindTCMalloc.cmake @@ -7,61 +7,70 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(TCMALLOC_ROOT AND NOT Tcmalloc_ROOT) + set(Tcmalloc_ROOT + ${TCMALLOC_ROOT} + CACHE PATH "TcMalloc base directory" + ) + unset(TCMALLOC_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) -pkg_check_modules(PC_TCMALLOC_MINIMAL QUIET libtcmalloc_minimal) -pkg_check_modules(PC_TCMALLOC QUIET libtcmalloc) +pkg_check_modules(PC_Tcmalloc_MINIMAL QUIET libtcmalloc_minimal) +pkg_check_modules(PC_Tcmalloc QUIET libtcmalloc) find_path( - TCMALLOC_INCLUDE_DIR google/tcmalloc.h - HINTS ${TCMALLOC_ROOT} + Tcmalloc_INCLUDE_DIR google/tcmalloc.h + HINTS ${Tcmalloc_ROOT} ENV TCMALLOC_ROOT ${HPX_TCMALLOC_ROOT} - ${PC_TCMALLOC_MINIMAL_INCLUDEDIR} - ${PC_TCMALLOC_MINIMAL_INCLUDE_DIRS} - ${PC_TCMALLOC_INCLUDEDIR} - ${PC_TCMALLOC_INCLUDE_DIRS} + ${PC_Tcmalloc_MINIMAL_INCLUDEDIR} + ${PC_Tcmalloc_MINIMAL_INCLUDE_DIRS} + ${PC_Tcmalloc_INCLUDEDIR} + ${PC_Tcmalloc_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - TCMALLOC_LIBRARY + Tcmalloc_LIBRARY NAMES tcmalloc_minimal libtcmalloc_minimal tcmalloc libtcmalloc - HINTS ${TCMALLOC_ROOT} + HINTS ${Tcmalloc_ROOT} ENV TCMALLOC_ROOT ${HPX_TCMALLOC_ROOT} - ${PC_TCMALLOC_MINIMAL_LIBDIR} - ${PC_TCMALLOC_MINIMAL_LIBRARY_DIRS} - ${PC_TCMALLOC_LIBDIR} - ${PC_TCMALLOC_LIBRARY_DIRS} + ${PC_Tcmalloc_MINIMAL_LIBDIR} + ${PC_Tcmalloc_MINIMAL_LIBRARY_DIRS} + ${PC_Tcmalloc_LIBDIR} + ${PC_Tcmalloc_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set TCMALLOC_ROOT in case the other hints are used -if(NOT TCMALLOC_ROOT AND "$ENV{TCMALLOC_ROOT}") - set(TCMALLOC_ROOT $ENV{TCMALLOC_ROOT}) -elseif(NOT TCMALLOC_ROOT) - string(REPLACE "/include" "" TCMALLOC_ROOT "${TCMALLOC_INCLUDE_DIR}") +# Set Tcmalloc_ROOT in case the other hints are used +if(NOT Tcmalloc_ROOT AND "$ENV{TCMALLOC_ROOT}") + set(Tcmalloc_ROOT $ENV{TCMALLOC_ROOT}) +elseif(NOT Tcmalloc_ROOT) + string(REPLACE "/include" "" Tcmalloc_ROOT "${Tcmalloc_INCLUDE_DIR}") endif() -set(TCMALLOC_LIBRARIES ${TCMALLOC_LIBRARY}) -set(TCMALLOC_INCLUDE_DIRS ${TCMALLOC_INCLUDE_DIR}) +set(Tcmalloc_LIBRARIES ${Tcmalloc_LIBRARY}) +set(Tcmalloc_INCLUDE_DIRS ${Tcmalloc_INCLUDE_DIR}) find_package_handle_standard_args( - TCMalloc DEFAULT_MSG TCMALLOC_LIBRARY TCMALLOC_INCLUDE_DIR + TCMalloc DEFAULT_MSG Tcmalloc_LIBRARY Tcmalloc_INCLUDE_DIR ) get_property( _type - CACHE TCMALLOC_ROOT + CACHE Tcmalloc_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE TCMALLOC_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Tcmalloc_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE TCMALLOC_ROOT PROPERTY TYPE PATH) + set_property(CACHE Tcmalloc_ROOT PROPERTY TYPE PATH) endif() endif() -mark_as_advanced(TCMALLOC_ROOT TCMALLOC_LIBRARY TCMALLOC_INCLUDE_DIR) +mark_as_advanced(Tcmalloc_ROOT Tcmalloc_LIBRARY Tcmalloc_INCLUDE_DIR) diff --git a/cmake/FindValgrind.cmake b/cmake/FindValgrind.cmake index 862901014333..f473ac321bc9 100644 --- a/cmake/FindValgrind.cmake +++ b/cmake/FindValgrind.cmake @@ -7,30 +7,39 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(VALGRIND_ROOT AND NOT Valgrind_ROOT) + set(Valgrind_ROOT + ${VALGRIND_ROOT} + CACHE PATH "Valgrind base directory" + ) + unset(VALGRIND_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) pkg_check_modules(PC_VALGRIND QUIET valgrind) find_path( - VALGRIND_INCLUDE_DIR valgrind/valgrind.h - HINTS ${VALGRIND_ROOT} ENV VALGRIND_ROOT ${PC_VALGRIND_INCLUDEDIR} - ${PC_VALGRIND_INCLUDE_DIRS} + Valgrind_INCLUDE_DIR valgrind/valgrind.h + HINTS ${Valgrind_ROOT} ENV VALGRIND_ROOT ${PC_Valgrind_INCLUDEDIR} + ${PC_Valgrind_INCLUDE_DIRS} PATH_SUFFIXES include ) -set(VALGRIND_INCLUDE_DIRS ${VALGRIND_INCLUDE_DIR}) +set(Valgrind_INCLUDE_DIRS ${Valgrind_INCLUDE_DIR}) -find_package_handle_standard_args(Valgrind DEFAULT_MSG VALGRIND_INCLUDE_DIR) +find_package_handle_standard_args(Valgrind DEFAULT_MSG Valgrind_INCLUDE_DIR) get_property( _type - CACHE VALGRIND_ROOT + CACHE Valgrind_ROOT PROPERTY TYPE ) if(_type) - set_property(CACHE VALGRIND_ROOT PROPERTY ADVANCED 1) + set_property(CACHE Valgrind_ROOT PROPERTY ADVANCED 1) if("x${_type}" STREQUAL "xUNINITIALIZED") - set_property(CACHE VALGRIND_ROOT PROPERTY TYPE PATH) + set_property(CACHE Valgrind_ROOT PROPERTY TYPE PATH) endif() endif() -mark_as_advanced(VALGRIND_ROOT VALGRIND_INCLUDE_DIR) +mark_as_advanced(Valgrind_ROOT Valgrind_INCLUDE_DIR) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 6dfb45989c2b..34793a49febd 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -82,9 +82,13 @@ function(add_hpx_test category name) set(_script_location ${PROJECT_BINARY_DIR}) endif() + if(PYTHON_EXECUTABLE AND NOT Python_EXECUTABLE) + set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) + endif() + # cmake-format: off set(cmd - "${PYTHON_EXECUTABLE}" + "${Python_EXECUTABLE}" "${_script_location}/bin/hpxrun.py" ${CMAKE_CROSSCOMPILING_EMULATOR} ${_exe} diff --git a/cmake/HPX_Documentation.cmake b/cmake/HPX_Documentation.cmake index fe9b38887fcb..c6530c04207b 100644 --- a/cmake/HPX_Documentation.cmake +++ b/cmake/HPX_Documentation.cmake @@ -11,14 +11,14 @@ if(HPX_WITH_DOCUMENTATION) find_package(Sphinx) find_package(Breathe) - if(NOT SPHINX_FOUND) + if(NOT Sphinx_FOUND) hpx_error( - "Sphinx is unavailable, sphinx documentation generation disabled. Set SPHINX_ROOT to your sphinx-build installation directory." + "Sphinx is unavailable, sphinx documentation generation disabled. Set Sphinx_ROOT to your sphinx-build installation directory." ) set(HPX_WITH_DOCUMENTATION OFF) - elseif(NOT BREATHE_FOUND) + elseif(NOT Breathe_FOUND) hpx_error( - "Breathe is unavailable, sphinx documentation generation disabled. Set BREATHE_APIDOC_ROOT to your breathe-apidoc installation directory." + "Breathe is unavailable, sphinx documentation generation disabled. Set Breathe_APIDOC_ROOT to your breathe-apidoc installation directory." ) set(HPX_WITH_DOCUMENTATION OFF) elseif(NOT DOXYGEN_FOUND) diff --git a/cmake/HPX_SetupAllocator.cmake b/cmake/HPX_SetupAllocator.cmake index 2e9d760a7781..4770573eef5a 100644 --- a/cmake/HPX_SetupAllocator.cmake +++ b/cmake/HPX_SetupAllocator.cmake @@ -43,12 +43,12 @@ if(NOT TARGET hpx_dependencies_allocator) # TCMALLOC if("${HPX_WITH_MALLOC_UPPER}" STREQUAL "TCMALLOC") find_package(TCMalloc) - if(NOT TCMALLOC_LIBRARIES) + if(NOT Tcmalloc_LIBRARIES) hpx_error(${allocator_error}) endif() target_link_libraries( - hpx_dependencies_allocator INTERFACE ${TCMALLOC_LIBRARIES} + hpx_dependencies_allocator INTERFACE ${Tcmalloc_LIBRARIES} ) if(MSVC) @@ -63,15 +63,15 @@ if(NOT TARGET hpx_dependencies_allocator) # JEMALLOC if("${HPX_WITH_MALLOC_UPPER}" STREQUAL "JEMALLOC") find_package(Jemalloc) - if(NOT JEMALLOC_LIBRARIES) + if(NOT Jemalloc_LIBRARIES) hpx_error(${allocator_error}) endif() target_include_directories( - hpx_dependencies_allocator INTERFACE ${JEMALLOC_INCLUDE_DIR} - ${JEMALLOC_ADDITIONAL_INCLUDE_DIR} + hpx_dependencies_allocator INTERFACE ${Jemalloc_INCLUDE_DIR} + ${Jemalloc_ADDITIONAL_INCLUDE_DIR} ) target_link_libraries( - hpx_dependencies_allocator INTERFACE ${JEMALLOC_LIBRARIES} + hpx_dependencies_allocator INTERFACE ${Jemalloc_LIBRARIES} ) endif() @@ -96,7 +96,7 @@ if(NOT TARGET hpx_dependencies_allocator) # TBBMALLOC if("${HPX_WITH_MALLOC_UPPER}" STREQUAL "TBBMALLOC") find_package(TBBmalloc) - if(NOT TBBMALLOC_LIBRARY AND NOT TBBMALLOC_PROXY_LIBRARY) + if(NOT Tbbmalloc_LIBRARY AND NOT Tbbmalloc_PROXY_LIBRARY) hpx_error(${allocator_error}) endif() if(MSVC) @@ -105,8 +105,8 @@ if(NOT TARGET hpx_dependencies_allocator) ) endif() target_link_libraries( - hpx_dependencies_allocator INTERFACE ${TBBMALLOC_LIBRARY} - ${TBBMALLOC_PROXY_LIBRARY} + hpx_dependencies_allocator INTERFACE ${Tbbmalloc_LIBRARY} + ${Tbbmalloc_PROXY_LIBRARY} ) endif() @@ -135,9 +135,9 @@ if(NOT TARGET hpx_dependencies_allocator) if((NOT HPX_WITH_APEX) AND HPX_WITH_ITTNOTIFY) find_package(Amplifier) - if(NOT AMPLIFIER_FOUND) + if(NOT Amplifier_FOUND) hpx_error( - "Intel Amplifier could not be found and HPX_WITH_ITTNOTIFY=On, please specify AMPLIFIER_ROOT to point to the root of your Amplifier installation" + "Intel Amplifier could not be found and HPX_WITH_ITTNOTIFY=On, please specify Amplifier_ROOT to point to the root of your Amplifier installation" ) endif() diff --git a/cmake/HPX_SetupApex.cmake b/cmake/HPX_SetupApex.cmake index 39cb573f3ad4..01d9b96a844c 100644 --- a/cmake/HPX_SetupApex.cmake +++ b/cmake/HPX_SetupApex.cmake @@ -15,20 +15,43 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(HPX_WITH_APEX AND NOT TARGET APEX::apex) + # compatibility with older CMake versions + if(APEX_ROOT AND NOT Apex_ROOT) + set(Apex_ROOT + ${APEX_ROOT} + CACHE PATH "Apex base directory" + ) + unset(APEX_ROOT CACHE) + endif() + if(MSR_ROOT AND NOT Msr_ROOT) + set(Msr_ROOT + ${MSR_ROOT} + CACHE PATH "MSR base directory" + ) + unset(MSR_ROOT CACHE) + endif() + if(OTF2_ROOT AND NOT Otf2_ROOT) + set(Otf2_ROOT + ${OTF2_ROOT} + CACHE PATH "OTF2 base directory" + ) + unset(OTF2_ROOT CACHE) + endif() + if(NOT HPX_FIND_PACKAGE) - if(NOT "${APEX_ROOT}" AND "$ENV{APEX_ROOT}") - set(APEX_ROOT "$ENV{APEX_ROOT}") + if(NOT "${Apex_ROOT}" AND "$ENV{APEX_ROOT}") + set(Apex_ROOT "$ENV{APEX_ROOT}") endif() # We want to track parent dependencies hpx_add_config_define(HPX_HAVE_THREAD_PARENT_REFERENCE) - if(APEX_ROOT) + if(Apex_ROOT) # Use given (external) APEX - set(HPX_APEX_ROOT ${APEX_ROOT}) + set(HPX_APEX_ROOT ${Apex_ROOT}) else() - # If APEX_ROOT not specified, local clone into hpx source dir + # If Apex_ROOT not specified, local clone into hpx source dir include(FetchContent) fetchcontent_declare( apex @@ -50,15 +73,15 @@ if(HPX_WITH_APEX AND NOT TARGET APEX::apex) if(NOT apex_POPULATED) hpx_error("APEX could not be populated with HPX_WITH_APEX=On") endif() - set(APEX_ROOT ${apex_SOURCE_DIR}) + set(Apex_ROOT ${apex_SOURCE_DIR}) - hpx_info("APEX_ROOT is not set. Cloning APEX into ${apex_SOURCE_DIR}.") + hpx_info("Apex_ROOT is not set. Cloning APEX into ${apex_SOURCE_DIR}.") endif() - list(APPEND CMAKE_MODULE_PATH "${APEX_ROOT}/cmake/Modules") - add_subdirectory(${APEX_ROOT}/src/apex ${CMAKE_BINARY_DIR}/apex/src/apex) - if(AMPLIFIER_FOUND) - hpx_error("AMPLIFIER_FOUND has been set. Please disable the use of the \ + list(APPEND CMAKE_MODULE_PATH "${Apex_ROOT}/cmake/Modules") + add_subdirectory(${Apex_ROOT}/src/apex ${CMAKE_BINARY_DIR}/apex/src/apex) + if(Amplifier_FOUND) + hpx_error("Amplifier_FOUND has been set. Please disable the use of the \ Intel Amplifier (WITH_AMPLIFIER=Off) in order to use APEX" ) endif() @@ -86,14 +109,14 @@ if(HPX_WITH_APEX AND NOT TARGET APEX::apex) # handle optional ITTNotify library (private dependency, skip when called in # find_package(HPX)) if(HPX_WITH_ITTNOTIFY AND NOT HPX_FIND_PACKAGE) - add_subdirectory(${APEX_ROOT}/src/ITTNotify) - if(NOT ITTNOTIFY_FOUND) + add_subdirectory(${Apex_ROOT}/src/ITTNotify) + if(NOT Ittnotify_FOUND) hpx_error("ITTNotify could not be found and HPX_WITH_ITTNOTIFY=On") endif() add_library(ITTNotify::ittnotify INTERFACE IMPORTED) target_include_directories( - ITTNotify::ittnotify SYSTEM INTERFACE ${ITTNOTIFY_SOURCE_DIR} + ITTNotify::ittnotify SYSTEM INTERFACE ${Ittnotify_SOURCE_DIR} ) target_link_libraries(APEX::apex INTERFACE ITTNotify::ittnotify) hpx_add_config_define(HPX_HAVE_ITTNOTIFY 1) diff --git a/cmake/HPX_SetupAsio.cmake b/cmake/HPX_SetupAsio.cmake index 1dfbdd9416d6..86a33a635f08 100644 --- a/cmake/HPX_SetupAsio.cmake +++ b/cmake/HPX_SetupAsio.cmake @@ -4,6 +4,15 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(ASIO_ROOT AND NOT Asio_ROOT) + set(Asio_ROOT + ${ASIO_ROOT} + CACHE PATH "ASIO base directory" + ) + unset(ASIO_ROOT CACHE) +endif() + if(NOT HPX_WITH_FETCH_ASIO) find_package(Asio 1.12.0 REQUIRED) elseif(NOT TARGET Asio::asio AND NOT HPX_FIND_PACKAGE) @@ -27,11 +36,11 @@ elseif(NOT TARGET Asio::asio AND NOT HPX_FIND_PACKAGE) if(NOT asio_POPULATED) fetchcontent_populate(asio) endif() - set(ASIO_ROOT ${asio_SOURCE_DIR}) + set(Asio_ROOT ${asio_SOURCE_DIR}) add_library(asio INTERFACE) target_include_directories( - asio SYSTEM INTERFACE $ + asio SYSTEM INTERFACE $ $ ) @@ -42,7 +51,7 @@ elseif(NOT TARGET Asio::asio AND NOT HPX_FIND_PACKAGE) ) install( - DIRECTORY ${ASIO_ROOT}/asio/include/ + DIRECTORY ${Asio_ROOT}/asio/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core FILES_MATCHING @@ -72,17 +81,22 @@ if(NOT HPX_FIND_PACKAGE) if(NOT HPX_WITH_CXX17_ALIGNED_NEW) hpx_add_config_cond_define(ASIO_DISABLE_STD_ALIGNED_ALLOC) endif() + # Asio does not detect that invoke_result is available, but we assume it # always is since we require C++17. hpx_add_config_cond_define(ASIO_HAS_STD_INVOKE_RESULT 1) + # Asio should not use Boost exceptions hpx_add_config_cond_define(ASIO_HAS_BOOST_THROW_EXCEPTION 0) + # Disable concepts support in Asio as a workaround to # https://github.com/boostorg/asio/issues/312 hpx_add_config_cond_define(ASIO_DISABLE_CONCEPTS) + # Disable experimental std::string_view support as a workaround to # https://github.com/chriskohlhoff/asio/issues/597 hpx_add_config_cond_define(ASIO_DISABLE_STD_EXPERIMENTAL_STRING_VIEW) + # Disable Asio's definition of NOMINMAX hpx_add_config_cond_define(ASIO_NO_NOMINMAX) endif() diff --git a/cmake/HPX_SetupBoost.cmake b/cmake/HPX_SetupBoost.cmake index 39b754606a05..873d7724a714 100644 --- a/cmake/HPX_SetupBoost.cmake +++ b/cmake/HPX_SetupBoost.cmake @@ -58,22 +58,31 @@ if(NOT TARGET hpx_dependencies_boost) list(REMOVE_DUPLICATES __boost_libraries) + # compatibility with older CMake versions + if(BOOST_ROOT AND NOT Boost_ROOT) + set(Boost_ROOT + ${BOOST_ROOT} + CACHE PATH "Boost base directory" + ) + unset(BOOST_ROOT CACHE) + endif() + find_package( - Boost ${Boost_MINIMUM_VERSION} MODULE REQUIRED + Boost ${Boost_MINIMUM_VERSION} NO_POLICY_SCOPE MODULE REQUIRED COMPONENTS ${__boost_libraries} ) if(NOT Boost_FOUND) hpx_error( - "Could not find Boost. Please set BOOST_ROOT to point to your Boost installation." + "Could not find Boost. Please set Boost_ROOT to point to your Boost installation." ) endif() # We are assuming that there is only one Boost Root - if(NOT BOOST_ROOT AND "$ENV{BOOST_ROOT}") - set(BOOST_ROOT $ENV{BOOST_ROOT}) - elseif(NOT BOOST_ROOT) - string(REPLACE "/include" "" BOOST_ROOT "${Boost_INCLUDE_DIRS}") + if(NOT Boost_ROOT AND "$ENV{BOOST_ROOT}") + set(Boost_ROOT $ENV{BOOST_ROOT}) + elseif(NOT Boost_ROOT) + string(REPLACE "/include" "" Boost_ROOT "${Boost_INCLUDE_DIRS}") endif() add_library(hpx_dependencies_boost INTERFACE IMPORTED) diff --git a/cmake/HPX_SetupBoostFilesystem.cmake b/cmake/HPX_SetupBoostFilesystem.cmake index 661099bbd949..44cc5f4531a3 100644 --- a/cmake/HPX_SetupBoostFilesystem.cmake +++ b/cmake/HPX_SetupBoostFilesystem.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2019 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -7,7 +7,10 @@ if(HPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY) # In case find_package(HPX) is called multiple times if(NOT TARGET Boost::filesystem) - find_package(Boost ${Boost_MINIMUM_VERSION} MODULE COMPONENTS filesystem) + find_package( + Boost ${Boost_MINIMUM_VERSION} NO_POLICY_SCOPE MODULE + COMPONENTS filesystem + ) if(NOT Boost_FILESYSTEM_FOUND) hpx_error( diff --git a/cmake/HPX_SetupBoostIostreams.cmake b/cmake/HPX_SetupBoostIostreams.cmake index 033c4f15e807..77784cd073c0 100644 --- a/cmake/HPX_SetupBoostIostreams.cmake +++ b/cmake/HPX_SetupBoostIostreams.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2019 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -9,7 +9,9 @@ if((HPX_WITH_COMPRESSION_BZIP2 OR HPX_WITH_COMPRESSION_ZLIB) AND NOT TARGET Boost::iostreams ) - find_package(Boost ${Boost_MINIMUM_VERSION} MODULE COMPONENTS iostreams) + find_package( + Boost ${Boost_MINIMUM_VERSION} NO_POLICY_SCOPE MODULE COMPONENTS iostreams + ) if(Boost_IOSTREAMS_FOUND) hpx_info(" iostreams") diff --git a/cmake/HPX_SetupBoostRegex.cmake b/cmake/HPX_SetupBoostRegex.cmake index 903810bc57e9..5ea12927da37 100644 --- a/cmake/HPX_SetupBoostRegex.cmake +++ b/cmake/HPX_SetupBoostRegex.cmake @@ -1,11 +1,13 @@ -# Copyright (c) 2019 The STE||AR-Group +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(NOT TARGET Boost::regex) - find_package(Boost ${Boost_MINIMUM_VERSION} MODULE COMPONENTS regex) + find_package( + Boost ${Boost_MINIMUM_VERSION} NO_POLICY_SCOPE MODULE COMPONENTS regex + ) if(Boost_REGEX_FOUND) hpx_info(" regex") diff --git a/cmake/HPX_SetupEve.cmake b/cmake/HPX_SetupEve.cmake index 9dff0f454a4d..884ef9b6793e 100644 --- a/cmake/HPX_SetupEve.cmake +++ b/cmake/HPX_SetupEve.cmake @@ -5,6 +5,16 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") AND NOT TARGET eve::eve) + + # compatibility with older CMake versions + if(EVE_ROOT AND NOT Eve_ROOT) + set(Eve_ROOT + ${EVE_ROOT} + CACHE PATH "Eve base directory" + ) + unset(EVE_ROOT CACHE) + endif() + if(HPX_WITH_FETCH_EVE) if(FETCHCONTENT_SOURCE_DIR_EVE) hpx_info( @@ -12,7 +22,7 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") AND NOT TARGET eve::eve) ) else() hpx_info( - "HPX_WITH_FETCH_EVE=${HPX_WITH_FETCH_EVE}, EVE will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_EVE_TAG=${HPX_WITH_EVE_TAG})" + "HPX_WITH_FETCH_EVE=${HPX_WITH_FETCH_EVE}, EVE will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Eve_TAG=${HPX_WITH_Eve_TAG})" ) endif() include(FetchContent) @@ -26,11 +36,11 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") AND NOT TARGET eve::eve) if(NOT eve_POPULATED) fetchcontent_populate(eve) endif() - set(EVE_ROOT ${eve_SOURCE_DIR}) + set(Eve_ROOT ${eve_SOURCE_DIR}) add_library(eve INTERFACE) target_include_directories( - eve SYSTEM INTERFACE $ + eve SYSTEM INTERFACE $ $ ) @@ -41,7 +51,7 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") AND NOT TARGET eve::eve) ) install( - DIRECTORY ${EVE_ROOT}/include/ + DIRECTORY ${Eve_ROOT}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core FILES_MATCHING @@ -66,10 +76,10 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") AND NOT TARGET eve::eve) add_library(eve::eve ALIAS eve) else() - if(EVE_ROOT) - find_package(eve REQUIRED PATHS ${EVE_ROOT}) + if(Eve_ROOT) + find_package(eve REQUIRED PATHS ${Eve_ROOT}) else() - hpx_error("EVE_ROOT not set") + hpx_error("Eve_ROOT not set") endif() endif() endif() diff --git a/cmake/HPX_SetupHIP.cmake b/cmake/HPX_SetupHIP.cmake index e19b78c7e271..2b2a42ae346f 100644 --- a/cmake/HPX_SetupHIP.cmake +++ b/cmake/HPX_SetupHIP.cmake @@ -23,7 +23,7 @@ if(HPX_WITH_HIP AND NOT TARGET roc::hipblas) if(NOT hipblas_FOUND) hpx_warn( "Hipblas could not be found, the blas parts will therefore be disabled.\n\ - You can reconfigure specifying HIPBLAS_ROOT to enable hipblas" + You can reconfigure specifying HIPBLAS_ROOT environment variable to enable hipblas" ) set(HPX_WITH_GPUBLAS OFF) else() diff --git a/cmake/HPX_SetupHwloc.cmake b/cmake/HPX_SetupHwloc.cmake index edbf001a8cfc..12ce52247a36 100644 --- a/cmake/HPX_SetupHwloc.cmake +++ b/cmake/HPX_SetupHwloc.cmake @@ -14,8 +14,8 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) find_package(Hwloc) -if(NOT HWLOC_FOUND) +if(NOT Hwloc_FOUND) hpx_error( - "Hwloc could not be found, please specify HWLOC_ROOT to point to the correct location" + "Hwloc could not be found, please specify Hwloc_ROOT to point to the correct location" ) endif() diff --git a/cmake/HPX_SetupLCI.cmake b/cmake/HPX_SetupLCI.cmake index 98a01ea68205..26b1ecd18076 100644 --- a/cmake/HPX_SetupLCI.cmake +++ b/cmake/HPX_SetupLCI.cmake @@ -9,13 +9,23 @@ macro(hpx_setup_lci) if(NOT TARGET LCI::LCI) + + # compatibility with older CMake versions + if(LCI_ROOT AND NOT Lci_ROOT) + set(Lci_ROOT + ${LCI_ROOT} + CACHE PATH "LCI base directory" + ) + unset(LCI_ROOT CACHE) + endif() + if(NOT HPX_WITH_FETCH_LCI) find_package( LCI CONFIG REQUIRED HINTS - ${LCI_ROOT} + ${Lci_ROOT} $ENV{LCI_ROOT} PATH_SUFFIXES lib/cmake @@ -28,14 +38,14 @@ macro(hpx_setup_lci) ) else() hpx_info( - "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_LCI_TAG=${HPX_WITH_LCI_TAG})" + "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Lci_TAG=${HPX_WITH_Lci_TAG})" ) endif() include(FetchContent) fetchcontent_declare( lci GIT_REPOSITORY https://github.com/uiuc-hpc/LC.git - GIT_TAG ${HPX_WITH_LCI_TAG} + GIT_TAG ${HPX_WITH_Lci_TAG} ) fetchcontent_getproperties(lci) diff --git a/cmake/HPX_SetupPapi.cmake b/cmake/HPX_SetupPapi.cmake index 300c41507d65..a091be1cb110 100644 --- a/cmake/HPX_SetupPapi.cmake +++ b/cmake/HPX_SetupPapi.cmake @@ -14,10 +14,10 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(HPX_WITH_PAPI) - find_package(PAPI) - if(NOT PAPI_FOUND) + find_package(Papi) + if(NOT Papi_FOUND) hpx_error("PAPI could not be found and HPX_WITH_PAPI=On, please specify \ - PAPI_ROOT to point to the root of your PAPI installation" + Papi_ROOT to point to the root of your PAPI installation" ) endif() endif() diff --git a/cmake/HPX_SetupSVE.cmake b/cmake/HPX_SetupSVE.cmake index c85b69c51104..467a6b048f6e 100644 --- a/cmake/HPX_SetupSVE.cmake +++ b/cmake/HPX_SetupSVE.cmake @@ -5,6 +5,16 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") AND NOT TARGET SVE::sve) + + # compatibility with older CMake versions + if(SVE_ROOT AND NOT Sve_ROOT) + set(Sve_ROOT + ${SVE_ROOT} + CACHE PATH "SVE base directory" + ) + unset(SVE_ROOT CACHE) + endif() + if(HPX_WITH_FETCH_SVE) if(FETCHCONTENT_SOURCE_DIR_SVE) hpx_info( @@ -12,7 +22,7 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") AND NOT TARGET SVE::sve) ) else() hpx_info( - "HPX_WITH_FETCH_SVE=${HPX_WITH_FETCH_SVE}, SVE will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_SVE_TAG=${HPX_WITH_SVE_TAG})" + "HPX_WITH_FETCH_SVE=${HPX_WITH_FETCH_SVE}, SVE will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Sve_TAG=${HPX_WITH_Sve_TAG})" ) endif() @@ -22,7 +32,7 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") AND NOT TARGET SVE::sve) ) endif() - set(SVE_LENGTH "${HPX_WITH_SVE_LENGTH}") + set(Sve_LENGTH "${HPX_WITH_SVE_LENGTH}") include(FetchContent) fetchcontent_declare( @@ -33,7 +43,7 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") AND NOT TARGET SVE::sve) fetchcontent_makeavailable(sve) - set(SVE_ROOT ${sve_SOURCE_DIR}) + set(Sve_ROOT ${sve_SOURCE_DIR}) install( TARGETS sve @@ -42,7 +52,7 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") AND NOT TARGET SVE::sve) ) install( - DIRECTORY ${SVE_ROOT}/include/ + DIRECTORY ${Sve_ROOT}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core FILES_MATCHING @@ -65,10 +75,10 @@ if(("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") AND NOT TARGET SVE::sve) ) else() - if(SVE_ROOT) - find_package(SVE REQUIRED PATHS ${SVE_ROOT}) + if(Sve_ROOT) + find_package(SVE REQUIRED PATHS ${Sve_ROOT}) else() - hpx_error("SVE_ROOT not set") + hpx_error("Sve_ROOT not set") endif() endif() endif() diff --git a/cmake/HPX_SetupValgrind.cmake b/cmake/HPX_SetupValgrind.cmake index f858a9186350..6c5295082610 100644 --- a/cmake/HPX_SetupValgrind.cmake +++ b/cmake/HPX_SetupValgrind.cmake @@ -16,9 +16,9 @@ if(HPX_WITH_VALGRIND AND NOT TARGET Valgrind::valgrind) find_package(Valgrind) - if(NOT VALGRIND_FOUND) + if(NOT Valgrind_FOUND) hpx_error("Valgrind could not be found and HPX_WITH_VALGRIND=On, please \ - specify VALGRIND_ROOT to point to the root of your Valgrind installation" + specify Valgrind_ROOT to point to the root of your Valgrind installation" ) endif() diff --git a/cmake/installed_hpx.cmake b/cmake/installed_hpx.cmake index 506fe121f0b8..5f079d802114 100644 --- a/cmake/installed_hpx.cmake +++ b/cmake/installed_hpx.cmake @@ -127,11 +127,20 @@ if(HPX_WITH_TESTS) include(CTest) # find Python interpreter (needed to run tests) - find_package(PythonInterp) - if(NOT PYTHONINTERP_FOUND) - hpx_warn( - "A python interpreter could not be found. The test suite can not be run automatically." - ) + if(POLICY CMP0148) + find_package(Python COMPONENTS Interpreter) + if(NOT Python_FOUND) + hpx_warn( + "A python interpreter could not be found. The test suite can not be run automatically." + ) + endif() + else() + find_package(PythonInterp) + if(NOT PYTHONINTERP_FOUND) + hpx_warn( + "A python interpreter could not be found. The test suite can not be run automatically." + ) + endif() endif() # add actual tests, first iterate through all modules diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 28cc6b872304..9d0f1d14fba2 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -23,7 +23,7 @@ include("${CMAKE_CURRENT_LIST_DIR}/HPXMacros.cmake") if(HPX_WITH_FETCH_ASIO) include("${CMAKE_CURRENT_LIST_DIR}/HPXAsioTarget.cmake") else() - set(HPX_ASIO_ROOT "@ASIO_ROOT@") + set(HPX_ASIO_ROOT "@Asio_ROOT@") include(HPX_SetupAsio) endif() @@ -34,7 +34,7 @@ if(HPX_COMMAND_LINE_HANDLING_WITH_JSON_CONFIGURATION_FILES) if(HPX_WITH_FETCH_JSON) include("${CMAKE_CURRENT_LIST_DIR}/HPXJSONTarget.cmake") else() - set(HPX_JSON_ROOT "@JSON_ROOT@") + set(HPX_JSON_ROOT "@Json_ROOT@") include(HPX_SetupJSON) endif() endif() @@ -46,7 +46,7 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) find_dependency(@LCI_FABRIC@) include("${CMAKE_CURRENT_LIST_DIR}/HPXLCITarget.cmake") else() - set(LCI_ROOT "@LCI_ROOT@") + set(Lci_ROOT "@Lci_ROOT@") include(HPX_SetupLCI) hpx_setup_lci() endif() @@ -63,7 +63,7 @@ if("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "EVE") if(HPX_WITH_FETCH_EVE) include("${CMAKE_CURRENT_LIST_DIR}/HPXEveTarget.cmake") else() - set(EVE_ROOT "@EVE_ROOT@") + set(Eve_ROOT "@Eve_ROOT@") include(HPX_SetupEve) endif() endif() @@ -72,7 +72,7 @@ if("${HPX_WITH_DATAPAR_BACKEND}" STREQUAL "SVE") if(HPX_WITH_FETCH_SVE) include("${CMAKE_CURRENT_LIST_DIR}/HPXSVETarget.cmake") else() - set(SVE_ROOT "@SVE_ROOT@") + set(Sve_ROOT "@Sve_ROOT@") include(HPX_SetupSVE) endif() endif() @@ -123,31 +123,31 @@ set(HPX_CXX_COMPILER_VERSION # Propagate Amplifier settings, if needed if(HPX_WITH_ITTNOTIFY) - set(AMPLIFIER_INCLUDE_DIR "@AMPLIFIER_INCLUDE_DIR@") - set(AMPLIFIER_LIBRARY "@AMPLIFIER_LIBRARY@") - set(AMPLIFIER_ROOT "@AMPLIFIER_ROOT@") + set(Amplifier_INCLUDE_DIR "@Amplifier_INCLUDE_DIR@") + set(Amplifier_LIBRARY "@Amplifier_LIBRARY@") + set(Amplifier_ROOT "@Amplifier_ROOT@") endif() # Allocator -set(HPX_JEMALLOC_ROOT "@JEMALLOC_ROOT@") -set(HPX_TCMALLOC_ROOT "@TCMALLOC_ROOT@") -set(HPX_TBBMALLOC_ROOT "@TBBMALLOC_ROOT@") +set(HPX_JEMALLOC_ROOT "@Jemalloc_ROOT@") +set(HPX_TCMALLOC_ROOT "@Tcmalloc_ROOT@") +set(HPX_TBBMALLOC_ROOT "@Tbbmalloc_ROOT@") # Special handle for mimalloc cause we can't specify HPX_MIMALLOC_ROOT as a HINT # to find_package -set(HPX_MIMALLOC_ROOT "@MIMALLOC_ROOT@") -if(NOT MIMALLOC_ROOT AND NOT "$ENV{MIMALLOC_ROOT}") - set(MIMALLOC_ROOT ${HPX_MIMALLOC_ROOT}) +set(HPX_MIMALLOC_ROOT "@Mimalloc_ROOT@") +if(NOT Mimalloc_ROOT AND NOT "$ENV{MIMALLOC_ROOT}") + set(Mimalloc_ROOT ${HPX_MIMALLOC_ROOT}) endif() include(HPX_SetupAllocator) include(HPX_SetupThreads) # Boost Separate boost targets to be unarily linked to some modules -set(HPX_BOOST_ROOT "@BOOST_ROOT@") -# By default BOOST_ROOT is set to HPX_BOOST_ROOT (not necessary for PAPI or +set(HPX_BOOST_ROOT "@Boost_ROOT@") +# By default Boost_ROOT is set to HPX_BOOST_ROOT (not necessary for PAPI or # HWLOC cause we are specifying HPX__ROOT as an HINT to find_package) -if(NOT BOOST_ROOT AND NOT "$ENV{BOOST_ROOT}") - set(BOOST_ROOT ${HPX_BOOST_ROOT}) +if(NOT Boost_ROOT AND NOT "$ENV{BOOST_ROOT}") + set(Boost_ROOT ${HPX_BOOST_ROOT}) endif() include(HPX_SetupBoost) include(HPX_SetupBoostFilesystem) @@ -157,11 +157,11 @@ include(HPX_SetupBoostIostreams) include(HPX_SetupHIP) # Hwloc -set(HPX_HWLOC_ROOT "@HWLOC_ROOT@") +set(HPX_HWLOC_ROOT "@Hwloc_ROOT@") include(HPX_SetupHwloc) # Papi -set(HPX_PAPI_ROOT "@PAPI_ROOT@") +set(HPX_PAPI_ROOT "@Papi_ROOT@") include(HPX_SetupPapi) # CUDA @@ -174,11 +174,11 @@ endif() # APEX set(APEX_WITH_MSR "@APEX_WITH_MSR@") -set(MSR_ROOT "@MSR_ROOT@") +set(Msr_ROOT "@Msr_ROOT@") set(APEX_WITH_ACTIVEHARMONY "@APEX_WITH_ACTIVEHARMONY@") -set(ACTIVEHARMONY_ROOT "@ACTIVEHARMONY_ROOT@") +set(Activeharmony_ROOT "@Activeharmony_ROOT@") set(APEX_WITH_OTF2 "@APEX_WITH_OTF2@") -set(OTF2_ROOT "@OTF2_ROOT@") +set(Otf2_ROOT "@Otf2_ROOT@") include(HPX_SetupApex) # ############################################################################## diff --git a/cmake/templates/HPXMacros.cmake.in b/cmake/templates/HPXMacros.cmake.in index 3315902d6e58..bd2f1dd6bb82 100644 --- a/cmake/templates/HPXMacros.cmake.in +++ b/cmake/templates/HPXMacros.cmake.in @@ -68,18 +68,18 @@ function(hpx_check_boost_compatibility) if(HPX_IGNORE_BOOST_COMPATIBILITY) return() endif() - if(NOT DEFINED BOOST_ROOT) + if(NOT DEFINED Boost_ROOT) return() endif() # make sure paths are tested even if not string identical get_filename_component(PATH1 "${HPX_BOOST_ROOT}" ABSOLUTE) - get_filename_component(PATH2 "${BOOST_ROOT}" ABSOLUTE) + get_filename_component(PATH2 "${Boost_ROOT}" ABSOLUTE) if(NOT PATH1 STREQUAL PATH2) - hpx_error("The specified BOOST_ROOT differs from what has been used when" + hpx_error("The specified Boost_ROOT differs from what has been used when" " configuring and building HPX. Please use the same Boost " - "versions. HPX boost is ${HPX_BOOST_ROOT} and users is ${BOOST_ROOT}. " + "versions. HPX boost is ${HPX_BOOST_ROOT} and users is ${Boost_ROOT}. " "To disable this message set HPX_IGNORE_BOOST_COMPATIBILITY On." ) endif() diff --git a/cmake/templates/cmake_variables.rst.in b/cmake/templates/cmake_variables.rst.in index 931d0de07d10..2a6752e668e9 100644 --- a/cmake/templates/cmake_variables.rst.in +++ b/cmake/templates/cmake_variables.rst.in @@ -42,9 +42,9 @@ Each of the tools or libraries listed here will be automatically detected if they are installed in some standard location. If a tool or library is installed in a different location, you can specify its base directory by appending ``_ROOT`` to the variable name as listed below. For instance, to configure a -custom directory for ``BOOST``, specify ``BOOST_ROOT=/custom/boost/root``. +custom directory for ``Boost``, specify ``Boost_ROOT=/custom/boost/root``. -.. option:: BOOST_ROOT:PATH +.. option:: Boost_ROOT:PATH Specifies where to look for the Boost installation to be used for compiling |hpx|. Set this if CMake is not able to locate a suitable version of @@ -52,7 +52,7 @@ custom directory for ``BOOST``, specify ``BOOST_ROOT=/custom/boost/root``. Boost distribution or the directory where you unpacked and built Boost without installing it (with staged libraries). -.. option:: HWLOC_ROOT:PATH +.. option:: Hwloc_ROOT:PATH Specifies where to look for the hwloc library. Set this if CMake is not able to locate a suitable version of hwloc. Hwloc provides platform- @@ -60,14 +60,14 @@ custom directory for ``BOOST``, specify ``BOOST_ROOT=/custom/boost/root``. architecture (number of cores, number of NUMA domains, hyperthreading, etc.). |hpx| utilizes this information if available. -.. option:: PAPI_ROOT:PATH +.. option:: Papi_ROOT:PATH Specifies where to look for the PAPI library. The PAPI library is needed to compile a special component exposing PAPI hardware events and counters as |hpx| performance counters. This is not available on the Windows platform. -.. option:: AMPLIFIER_ROOT:PATH +.. option:: Amplifier_ROOT:PATH Specifies where to look for one of the tools of the Intel Parallel Studio product, either Intel Amplifier or Intel Inspector. This should be @@ -78,6 +78,6 @@ custom directory for ``BOOST``, specify ``BOOST_ROOT=/custom/boost/root``. In addition, some of the examples may need the following variables: -.. option:: HDF5_ROOT:PATH +.. option:: Hdf5_ROOT:PATH Specifies where to look for the |hdf5| include files and libraries. diff --git a/cmake/templates/hpxcxx.in b/cmake/templates/hpxcxx.in index 8e2385265784..a6d091c8fe01 100755 --- a/cmake/templates/hpxcxx.in +++ b/cmake/templates/hpxcxx.in @@ -1,4 +1,4 @@ -#! @PYTHON_EXECUTABLE@ +#! @Python_EXECUTABLE@ # # Copyright (c) 2014 Steven R. Brandt # diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 76eed0195555..21a5badbd411 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -1,4 +1,4 @@ -#! @PYTHON_EXECUTABLE@ +#! @Python_EXECUTABLE@ # # Copyright (c) 2014 Thomas Heller # diff --git a/cmake/toolchains/BGION-gcc.cmake b/cmake/toolchains/BGION-gcc.cmake index fb100952552b..b0447328a573 100644 --- a/cmake/toolchains/BGION-gcc.cmake +++ b/cmake/toolchains/BGION-gcc.cmake @@ -85,8 +85,8 @@ set(HPX_HIDDEN_VISIBILITY # # Convenience setup for jb @ bbpbg2.cscs.ch # -set(BOOST_ROOT "/gpfs/bbp.cscs.ch/home/biddisco/apps/gcc-4.8.2/boost_1_56_0") -set(HWLOC_ROOT "/gpfs/bbp.cscs.ch/home/biddisco/apps/gcc-4.8.2/hwloc-1.8.1") +set(Boost_ROOT "/gpfs/bbp.cscs.ch/home/biddisco/apps/gcc-4.8.2/boost_1_56_0") +set(Hwloc_ROOT "/gpfs/bbp.cscs.ch/home/biddisco/apps/gcc-4.8.2/hwloc-1.8.1") set(CMAKE_BUILD_TYPE "Debug" diff --git a/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt b/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt index 6550e27e3ca1..4fcdaccbf0de 100644 --- a/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt +++ b/components/parcel_plugins/binary_filter/snappy/CMakeLists.txt @@ -11,14 +11,14 @@ endif() include(HPX_AddLibrary) find_package(Snappy) -if(NOT SNAPPY_FOUND) +if(NOT Snappy_FOUND) hpx_error("Snappy could not be found and HPX_WITH_COMPRESSION_SNAPPY=ON, \ please specify SNAPPY_ROOT to point to the correct location or set \ HPX_WITH_COMPRESSION_SNAPPY to OFF" ) endif() -hpx_debug("add_snappy_module" "SNAPPY_FOUND: ${SNAPPY_FOUND}") +hpx_debug("add_snappy_module" "SNAPPY_FOUND: ${Snappy_FOUND}") add_hpx_library( compression_snappy INTERNAL_FLAGS PLUGIN @@ -30,13 +30,14 @@ add_hpx_library( "hpx/binary_filter/snappy_serialization_filter.hpp" "hpx/binary_filter/snappy_serialization_filter_registration.hpp" PREPEND_HEADER_ROOT INSTALL_HEADERS - FOLDER "Core/Plugins/Compression" ${HPX_WITH_UNITY_BUILD_OPTION} + FOLDER "Core/Plugins/Compression" + DEPENDENCIES ${Snappy_LIBRARY} ${HPX_WITH_UNITY_BUILD_OPTION} ) target_include_directories( - compression_snappy SYSTEM PRIVATE ${SNAPPY_INCLUDE_DIR} + compression_snappy SYSTEM PRIVATE ${Snappy_INCLUDE_DIR} ) -target_link_directories(compression_snappy PRIVATE ${SNAPPY_LIBRARY_DIR}) +target_link_directories(compression_snappy PRIVATE ${Snappy_LIBRARY_DIR}) target_link_libraries( compression_snappy PUBLIC Boost::iostreams ${SNAPPY_LIBRARY} diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 437aa6671b00..bc2c88e473a7 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -221,14 +221,14 @@ foreach(output_format ${HPX_WITH_DOCUMENTATION_OUTPUT_FORMATS}) # the moment not worth the effort. if(${output_format} STREQUAL "latexpdf") set(SPHINX_DOCS_BUILD_COMMAND - ${SPHINX_EXECUTABLE} -M ${output_format} + ${Sphinx_EXECUTABLE} -M ${output_format} "${CMAKE_CURRENT_BINARY_DIR}/sphinx" "${SPHINX_DOCS_OUTPUT_DIR}" || ( exit 0) ) else() set(SPHINX_DOCS_BUILD_COMMAND - ${SPHINX_EXECUTABLE} -b ${output_format} -n -d + ${Sphinx_EXECUTABLE} -b ${output_format} -n -d "${CMAKE_CURRENT_BINARY_DIR}/doctree" "${CMAKE_CURRENT_BINARY_DIR}/sphinx" "${SPHINX_DOCS_OUTPUT_DIR}" ) diff --git a/docs/sphinx/contributing/documentation.rst b/docs/sphinx/contributing/documentation.rst index 99b577c696eb..79edaf15b5dc 100644 --- a/docs/sphinx/contributing/documentation.rst +++ b/docs/sphinx/contributing/documentation.rst @@ -41,15 +41,15 @@ manager, you can install them using the Python package manager ``pip``: You may need to set the following CMake variables to make sure CMake can find the required dependencies. -.. option:: DOXYGEN_ROOT:PATH +.. option:: Doxygen_ROOT:PATH Specifies where to look for the installation of the |doxygen|_ tool. -.. option:: SPHINX_ROOT:PATH +.. option:: Sphinx_ROOT:PATH Specifies where to look for the installation of the |sphinx|_ tool. -.. option:: BREATHE_APIDOC_ROOT:PATH +.. option:: Breathe_APIDOC_ROOT:PATH Specifies where to look for the installation of the |breathe|_ tool. diff --git a/docs/sphinx/manual/building_hpx.rst b/docs/sphinx/manual/building_hpx.rst index cc5343eb854d..3e974517ddfa 100644 --- a/docs/sphinx/manual/building_hpx.rst +++ b/docs/sphinx/manual/building_hpx.rst @@ -196,10 +196,10 @@ known to |cmake|, the following gets you started: .. code-block:: shell-session - $ cmake -DBOOST_ROOT=/path/to/boost - -DHWLOC_ROOT=/path/to/hwloc - -DTCMALLOC_ROOT=/path/to/tcmalloc - -DJEMALLOC_ROOT=/path/to/jemalloc + $ cmake -DBoost_ROOT=/path/to/boost + -DHwloc_ROOT=/path/to/hwloc + -DTcmalloc_ROOT=/path/to/tcmalloc + -DJemalloc_ROOT=/path/to/jemalloc [other CMake variable definitions] /path/to/source/tree @@ -207,7 +207,7 @@ known to |cmake|, the following gets you started: .. code-block:: shell-session - $ cmake -DBOOST_ROOT=~/packages/boost -DHWLOC_ROOT=/packages/hwloc -DCMAKE_INSTALL_PREFIX=~/packages/hpx ~/downloads/hpx_1.5.1 + $ cmake -DBoost_ROOT=~/packages/boost -DHwloc_ROOT=/packages/hwloc -DCMAKE_INSTALL_PREFIX=~/packages/hpx ~/downloads/hpx_1.5.1 * If you want to try |hpx| without using a custom allocator pass ``-DHPX_WITH_MALLOC=system`` to |cmake|: @@ -290,14 +290,14 @@ To build |hpx| under Windows 10 x64 with Visual Studio 2015: in any way. Instead, it will generate Visual Studio Solution Files, which will build |hpx| packages out of the |hpx| source tree. -* Set three new environment variables (in CMake, not in Windows environment): - ``BOOST_ROOT``, ``HWLOC_ROOT``, ``ASIO_ROOT``, ``CMAKE_INSTALL_PREFIX``. The meaning of +* Set new configuration variables (in CMake, not in Windows environment): + ``Boost_ROOT``, ``Hwloc_ROOT``, ``Asio_ROOT``, ``CMAKE_INSTALL_PREFIX``. The meaning of these variables is as follows: - * ``BOOST_ROOT`` the |hpx| root directory of the unpacked Boost headers/cpp files. - * ``HWLOC_ROOT`` the |hpx| root directory of the unpacked Portable Hardware Locality + * ``Boost_ROOT`` the |hpx| root directory of the unpacked Boost headers/cpp files. + * ``Hwloc_ROOT`` the |hpx| root directory of the unpacked Portable Hardware Locality files. - * ``ASIO_ROOT`` the |hpx| root directory of the unpacked ASIO files. Alternatively use + * ``Asio_ROOT`` the |hpx| root directory of the unpacked ASIO files. Alternatively use ``HPX_WITH_FETCH_ASIO`` with value ``True``. * ``CMAKE_INSTALL_PREFIX`` the |hpx| root directory where the future builds of |hpx| should be installed. @@ -320,13 +320,13 @@ To build |hpx| under Windows 10 x64 with Visual Studio 2015: Example CMake adding entry. - Alternatively, users could provide ``BOOST_LIBRARYDIR`` instead of - ``BOOST_ROOT``; the difference is that ``BOOST_LIBRARYDIR`` should point to + Alternatively, users could provide ``Boost_LIBRARYDIR`` instead of + ``Boost_ROOT``; the difference is that ``Boost_LIBRARYDIR`` should point to the subdirectory inside Boost root where all the compiled DLLs/LIBs are. For - example, ``BOOST_LIBRARYDIR`` may point to the ``bin.v2`` subdirectory under + example, ``Boost_LIBRARYDIR`` may point to the ``bin.v2`` subdirectory under the Boost rootdir. It is important to keep the meanings of these two variables - separated from each other: ``BOOST_DIR`` points to the ROOT folder of the - Boost library. ``BOOST_LIBRARYDIR`` points to the subdir inside the Boost root + separated from each other: ``Boost_DIR`` points to the ROOT folder of the + Boost library. ``Boost_LIBRARYDIR`` points to the subdir inside the Boost root folder where the compiled binaries are. * Click the 'Configure' button of CMake-GUI. You will be immediately presented with a diff --git a/docs/sphinx/manual/creating_hpx_projects.rst b/docs/sphinx/manual/creating_hpx_projects.rst index e4b9fb06b1b7..102f118f879a 100644 --- a/docs/sphinx/manual/creating_hpx_projects.rst +++ b/docs/sphinx/manual/creating_hpx_projects.rst @@ -476,14 +476,14 @@ Add the following code: CXXFLAGS=-O3 -std=c++17 - BOOST_ROOT=/path/to/boost - HWLOC_ROOT=/path/to/hwloc - TCMALLOC_ROOT=/path/to/tcmalloc + Boost_ROOT=/path/to/boost + Hwloc_ROOT=/path/to/hwloc + Tcmalloc_ROOT=/path/to/tcmalloc HPX_ROOT=/path/to/hpx - INCLUDE_DIRECTIVES=$(HPX_ROOT)/include $(BOOST_ROOT)/include $(HWLOC_ROOT)/include + INCLUDE_DIRECTIVES=$(HPX_ROOT)/include $(Boost_ROOT)/include $(Hwloc_ROOT)/include - LIBRARY_DIRECTIVES=-L$(HPX_ROOT)/lib $(HPX_ROOT)/lib/libhpx_init.a $(HPX_ROOT)/lib/libhpx.so $(BOOST_ROOT)/lib/libboost_atomic-mt.so $(BOOST_ROOT)/lib/libboost_filesystem-mt.so $(BOOST_ROOT)/lib/libboost_program_options-mt.so $(BOOST_ROOT)/lib/libboost_regex-mt.so $(BOOST_ROOT)/lib/libboost_system-mt.so -lpthread $(TCMALLOC_ROOT)/libtcmalloc_minimal.so $(HWLOC_ROOT)/libhwloc.so -ldl -lrt + LIBRARY_DIRECTIVES=-L$(HPX_ROOT)/lib $(HPX_ROOT)/lib/libhpx_init.a $(HPX_ROOT)/lib/libhpx.so $(Boost_ROOT)/lib/libboost_atomic-mt.so $(Boost_ROOT)/lib/libboost_filesystem-mt.so $(Boost_ROOT)/lib/libboost_program_options-mt.so $(Boost_ROOT)/lib/libboost_regex-mt.so $(Boost_ROOT)/lib/libboost_system-mt.so -lpthread $(Tcmalloc_ROOT)/libtcmalloc_minimal.so $(Hwloc_ROOT)/libhwloc.so -ldl -lrt LINK_FLAGS=$(HPX_ROOT)/lib/libhpx_wrap.a -Wl,-wrap=main # should be left empty for HPX_WITH_HPX_MAIN=OFF @@ -548,14 +548,14 @@ Now, in the directory, create a Makefile. Add the following code: CXXFLAGS=-O3 -std=c++17 - BOOST_ROOT=/path/to/boost - HWLOC_ROOT=/path/to/hwloc - TCMALLOC_ROOT=/path/to/tcmalloc + Boost_ROOT=/path/to/boost + Hwloc_ROOT=/path/to/hwloc + Tcmalloc_ROOT=/path/to/tcmalloc HPX_ROOT=/path/to/hpx - INCLUDE_DIRECTIVES=$(HPX_ROOT)/include $(BOOST_ROOT)/include $(HWLOC_ROOT)/include + INCLUDE_DIRECTIVES=$(HPX_ROOT)/include $(Boost_ROOT)/include $(Hwloc_ROOT)/include - LIBRARY_DIRECTIVES=-L$(HPX_ROOT)/lib $(HPX_ROOT)/lib/libhpx_init.a $(HPX_ROOT)/lib/libhpx.so $(BOOST_ROOT)/lib/libboost_atomic-mt.so $(BOOST_ROOT)/lib/libboost_filesystem-mt.so $(BOOST_ROOT)/lib/libboost_program_options-mt.so $(BOOST_ROOT)/lib/libboost_regex-mt.so $(BOOST_ROOT)/lib/libboost_system-mt.so -lpthread $(TCMALLOC_ROOT)/libtcmalloc_minimal.so $(HWLOC_ROOT)/libhwloc.so -ldl -lrt + LIBRARY_DIRECTIVES=-L$(HPX_ROOT)/lib $(HPX_ROOT)/lib/libhpx_init.a $(HPX_ROOT)/lib/libhpx.so $(Boost_ROOT)/lib/libboost_atomic-mt.so $(Boost_ROOT)/lib/libboost_filesystem-mt.so $(Boost_ROOT)/lib/libboost_program_options-mt.so $(Boost_ROOT)/lib/libboost_regex-mt.so $(Boost_ROOT)/lib/libboost_system-mt.so -lpthread $(Tcmalloc_ROOT)/libtcmalloc_minimal.so $(Hwloc_ROOT)/libhwloc.so -ldl -lrt LINK_FLAGS=$(HPX_ROOT)/lib/libhpx_wrap.a -Wl,-wrap=main # should be left empty for HPX_WITH_HPX_MAIN=OFF diff --git a/docs/sphinx/manual/optimizing_hpx_applications.rst b/docs/sphinx/manual/optimizing_hpx_applications.rst index f3ec0d4c2a27..90d956441728 100644 --- a/docs/sphinx/manual/optimizing_hpx_applications.rst +++ b/docs/sphinx/manual/optimizing_hpx_applications.rst @@ -3407,7 +3407,7 @@ It can be added as a ``git`` submodule by turning on the option :option:`HPX_WIT during |cmake| configuration. |tau|_ is an optional dependency when using |apex|. To build |hpx| with |apex|, add :option:`HPX_WITH_APEX`\ ``=ON``, and, -optionally, ``TAU_ROOT=$PATH_TO_TAU`` to your |cmake| configuration. In +optionally, ``Tau_ROOT=$PATH_TO_TAU`` to your |cmake| configuration. In addition, you can override the tag used for |apex| with the :option:`HPX_WITH_APEX_TAG` option. Please see the |apex_hpx_doc|_ for detailed instructions on using |apex| with |hpx|. diff --git a/docs/sphinx/manual/troubleshooting.rst b/docs/sphinx/manual/troubleshooting.rst index 8a693d30c6af..800e35f9e67b 100644 --- a/docs/sphinx/manual/troubleshooting.rst +++ b/docs/sphinx/manual/troubleshooting.rst @@ -74,7 +74,7 @@ You may see an error message that looks a bit like this: .. code-block:: text - Could NOT find TCMalloc (missing: TCMALLOC_LIBRARY TCMALLOC_INCLUDE_DIR) + Could NOT find TCMalloc (missing: Tcmalloc_LIBRARY Tcmalloc_INCLUDE_DIR) ERROR: HPX_WITH_MALLOC was set to tcmalloc, but tcmalloc could not be found. Valid options for HPX_WITH_MALLOC are: system, tcmalloc, jemalloc, mimalloc, tbbmalloc, and custom @@ -117,7 +117,7 @@ following flag on your |hpx| application CMake configuration: |hpx|-application build type conformance ---------------------------------------- -Your application’s build type should align with the HPX build type. For example, if you specified +Your application's build type should align with the HPX build type. For example, if you specified ``-DCMAKE_BUILD_TYPE=Debug`` during the |hpx| compilation, then your application needs to be compiled with the same flag. We recommend keeping a separate build folder for different build types and just point accordingly to the type you want by using ``-DHPX_DIR=/lib/cmake/HPX``. diff --git a/examples/1d_stencil/README.APEX b/examples/1d_stencil/README.APEX index 52eb26a83fa5..a17aa4badf8f 100644 --- a/examples/1d_stencil/README.APEX +++ b/examples/1d_stencil/README.APEX @@ -7,7 +7,7 @@ configure HPX with APEX: cmake ... \ -DHPX_WITH_APEX=TRUE \ -DAPEX_WITH_ACTIVEHARMONY=TRUE \ --DACTIVEHARMONY_ROOT= \ +-DActiveharmony_ROOT= \ ... Then to execute the example: diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index b09723ad28dc..3f5e4546c3ec 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -32,7 +32,7 @@ endif() set(hpx_external_SOURCES) if("${HPX_PLATFORM_UC}" STREQUAL "ANDROID") list(APPEND hpx_external_SOURCES ${hpx_external_objects_SOURCES} - "${ANDROID_NDK_ROOT}/sources/android/cpufeatures/cpu-features.c" + "${Android_NDK_ROOT}/sources/android/cpufeatures/cpu-features.c" ) endif() diff --git a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake index f0e5bca5653c..119cb8eb07e3 100644 --- a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake +++ b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake @@ -4,6 +4,15 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(JSON_ROOT AND NOT Json_ROOT) + set(Json_ROOT + ${JSON_ROOT} + CACHE PATH "Json base directory" + ) + unset(JSON_ROOT CACHE) +endif() + if(NOT HPX_WITH_FETCH_JSON) find_package(nlohmann_json 3.2.0 REQUIRED) elseif(NOT TARGET JSON::json) @@ -17,7 +26,7 @@ elseif(NOT TARGET JSON::json) ) else() hpx_info( - "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_JSON_TAG=${HPX_WITH_JSON_TAG})" + "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Json_TAG=${HPX_WITH_Json_TAG})" ) endif() @@ -29,14 +38,14 @@ elseif(NOT TARGET JSON::json) ) fetchcontent_makeavailable(nlohmann_json) - set(JSON_ROOT ${nlohmann_json_SOURCE_DIR}) + set(Json_ROOT ${nlohmann_json_SOURCE_DIR}) add_library(json INTERFACE) target_include_directories( - json SYSTEM INTERFACE $ + json SYSTEM INTERFACE $ $ ) - target_compile_definitions(json INTERFACE JSON_HAS_CPP_17) + target_compile_definitions(json INTERFACE Json_HAS_CPP_17) install( TARGETS json @@ -45,7 +54,7 @@ elseif(NOT TARGET JSON::json) ) install( - DIRECTORY ${JSON_ROOT}/include/ + DIRECTORY ${Json_ROOT}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core FILES_MATCHING diff --git a/libs/full/parcelport_libfabric/cmake/FindPMI.cmake b/libs/full/parcelport_libfabric/cmake/FindPMI.cmake index 20426147c638..ef7073ab1262 100644 --- a/libs/full/parcelport_libfabric/cmake/FindPMI.cmake +++ b/libs/full/parcelport_libfabric/cmake/FindPMI.cmake @@ -4,62 +4,70 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# compatibility with older CMake versions +if(PMI_ROOT AND NOT Pmi_ROOT) + set(Pmi_ROOT + ${PMI_ROOT} + CACHE PATH "PMI base directory" + ) + unset(PMI_ROOT CACHE) +endif() + find_package(PkgConfig QUIET) + # look for cray pmi... -pkg_check_modules(PC_PMI_CRAY QUIET cray-pmi) +pkg_check_modules(PC_Pmi_CRAY QUIET cray-pmi) + # look for the rest if we couldn't find the cray package -if(NOT PC_PMI_CRAY_FOUND) - pkg_check_modules(PC_PMI QUIET pmi) +if(NOT PC_Pmi_CRAY_FOUND) + pkg_check_modules(PC_Pmi QUIET pmi) endif() find_path( - PMI_INCLUDE_DIR pmi2.h - HINTS ${PMI_ROOT} + Pmi_INCLUDE_DIR pmi2.h + HINTS ${Pmi_ROOT} ENV PMI_ROOT - ${PMI_DIR} + ${Pmi_DIR} ENV PMI_DIR - ${PC_PMI_CRAY_INCLUDEDIR} - ${PC_PMI_CRAY_INCLUDE_DIRS} - ${PC_PMI_INCLUDEDIR} - ${PC_PMI_INCLUDE_DIRS} + ${PC_Pmi_CRAY_INCLUDEDIR} + ${PC_Pmi_CRAY_INCLUDE_DIRS} + ${PC_Pmi_INCLUDEDIR} + ${PC_Pmi_INCLUDE_DIRS} PATH_SUFFIXES include ) find_library( - PMI_LIBRARY + Pmi_LIBRARY NAMES pmi - HINTS ${PMI_ROOT} + HINTS ${Pmi_ROOT} ENV PMI_ROOT - ${PC_PMI_CRAY_LIBDIR} - ${PC_PMI_CRAY_LIBRARY_DIRS} - ${PC_PMI_LIBDIR} - ${PC_PMI_LIBRARY_DIRS} + ${PC_Pmi_CRAY_LIBDIR} + ${PC_Pmi_CRAY_LIBRARY_DIRS} + ${PC_Pmi_LIBDIR} + ${PC_Pmi_LIBRARY_DIRS} PATH_SUFFIXES lib lib64 ) -# Set PMI_ROOT in case the other hints are used -if(PMI_ROOT) +# Set Pmi_ROOT in case the other hints are used +if(Pmi_ROOT) # The call to file is for compatibility with windows paths - file(TO_CMAKE_PATH ${PMI_ROOT} PMI_ROOT) + file(TO_CMAKE_PATH ${Pmi_ROOT} Pmi_ROOT) elseif("$ENV{PMI_ROOT}") - file(TO_CMAKE_PATH $ENV{PMI_ROOT} PMI_ROOT) + file(TO_CMAKE_PATH $ENV{PMI_ROOT} Pmi_ROOT) else() - file(TO_CMAKE_PATH "${PMI_INCLUDE_DIR}" PMI_INCLUDE_DIR) - string(REPLACE "/include" "" PMI_ROOT "${PMI_INCLUDE_DIR}") + file(TO_CMAKE_PATH "${Pmi_INCLUDE_DIR}" Pmi_INCLUDE_DIR) + string(REPLACE "/include" "" Pmi_ROOT "${Pmi_INCLUDE_DIR}") endif() -if(NOT PMI_LIBRARY OR NOT PMI_INCLUDE_DIR) - set(PMI_FOUND=OFF) +if(NOT Pmi_LIBRARY OR NOT Pmi_INCLUDE_DIR) + set(Pmi_FOUND=OFF) return() endif() -# hpx_error( "PMI_LIBRARY OR PMI_INCLUDE_DIR not found, please install PMI or -# set \ the right PMI_ROOT path" ) - include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(PMI DEFAULT_MSG PMI_LIBRARY PMI_INCLUDE_DIR) +find_package_handle_standard_args(PMI DEFAULT_MSG Pmi_LIBRARY Pmi_INCLUDE_DIR) -mark_as_advanced(PMI_ROOT PMI_LIBRARY PMI_INCLUDE_DIR) +mark_as_advanced(Pmi_ROOT Pmi_LIBRARY Pmi_INCLUDE_DIR) diff --git a/tests/performance/local/CMakeLists.txt b/tests/performance/local/CMakeLists.txt index 9566be850d4f..4d537abdf382 100644 --- a/tests/performance/local/CMakeLists.txt +++ b/tests/performance/local/CMakeLists.txt @@ -65,17 +65,17 @@ if(HPX_WITH_EXAMPLES_QTHREADS) ) set(qthreads_homogeneous_timed_task_spawn_FLAGS - NOLIBS DEPENDENCIES ${boost_library_dependencies} ${QTHREADS_LIBRARIES} + NOLIBS DEPENDENCIES ${boost_library_dependencies} ${Qthreads_LIBRARIES} hpx_core ) set(qthreads_heterogeneous_timed_task_spawn_FLAGS - NOLIBS DEPENDENCIES ${boost_library_dependencies} ${QTHREADS_LIBRARIES} + NOLIBS DEPENDENCIES ${boost_library_dependencies} ${Qthreads_LIBRARIES} hpx_core ) set(qthreads_heterogeneous_timed_task_spawn_INCLUDE_DIRECTORIES - ${QTHREADS_INCLUDE_DIR} + ${Qthreads_INCLUDE_DIR} ) endif() @@ -83,11 +83,11 @@ if(HPX_WITH_EXAMPLES_TBB) list(APPEND benchmarks tbb_homogeneous_timed_task_spawn) set(tbb_homogeneous_timed_task_spawn_FLAGS - NOLIBS DEPENDENCIES ${boost_library_dependencies} ${TBB_LIBRARIES} + NOLIBS DEPENDENCIES ${boost_library_dependencies} ${Tbb_LIBRARIES} hpx_core ) - set(tbb_homogeneous_timed_task_spawn_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIR}) + set(tbb_homogeneous_timed_task_spawn_INCLUDE_DIRECTORIES ${Tbb_INCLUDE_DIR}) endif() if(HPX_WITH_DISTRIBUTED_RUNTIME) diff --git a/tests/performance/local/htts_v2/CMakeLists.txt b/tests/performance/local/htts_v2/CMakeLists.txt index 05647b2fd811..86bf74b2a5ba 100644 --- a/tests/performance/local/htts_v2/CMakeLists.txt +++ b/tests/performance/local/htts_v2/CMakeLists.txt @@ -12,14 +12,14 @@ endif() if(HPX_WITH_EXAMPLES_QTHREADS) set(benchmarks ${benchmarks} htts2_qthreads) - set(htts2_qthreads_LIBRARIES ${QTHREADS_LIBRARY}) - set(htts2_qthreads_INCLUDE_DIRECTORIES ${QTHREADS_INCLUDE_DIR}) + set(htts2_qthreads_LIBRARIES ${Qthreads_LIBRARY}) + set(htts2_qthreads_INCLUDE_DIRECTORIES ${Qthreads_INCLUDE_DIR}) endif() if(HPX_WITH_EXAMPLES_TBB) set(benchmarks ${benchmarks} htts2_tbb) - set(htts2_tbb_LIBRARIES ${TBB_LIBRARY}) - set(htts2_tbb_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIR}) + set(htts2_tbb_LIBRARIES ${Tbb_LIBRARY}) + set(htts2_tbb_INCLUDE_DIRECTORIES ${Tbb_INCLUDE_DIR}) endif() foreach(benchmark ${benchmarks}) diff --git a/tools/build_boost.sh b/tools/build_boost.sh index 9573ebe0b58b..e821eaf24c5b 100755 --- a/tools/build_boost.sh +++ b/tools/build_boost.sh @@ -198,10 +198,10 @@ echo echo "Successfully built Boost ${DOT_VERSION}" echo echo "Debug root:" -echo " BOOST_ROOT=$DIRECTORY/debug" +echo " Boost_ROOT=$DIRECTORY/debug" echo echo "Release root:" -echo " BOOST_ROOT=$DIRECTORY/release" +echo " Boost_ROOT=$DIRECTORY/release" From 6bf30d4a9697e7be7f420d5933ee561377012307 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 19 Aug 2023 10:14:33 -0500 Subject: [PATCH 174/333] Fixing HIP builder --- .../include/hpx/parallel/util/loop.hpp | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index dfa67230e5d6..b3207bc980e7 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -130,7 +130,7 @@ namespace hpx::parallel::util { inline constexpr loop_t loop = loop_t{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop( ExPolicy&& policy, Begin begin, End end, F&& f) { return hpx::parallel::util::loop_t{}( @@ -139,7 +139,7 @@ namespace hpx::parallel::util { template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop( ExPolicy&& policy, Begin begin, End end, CancelToken& tok, F&& f) { return hpx::parallel::util::loop_t{}( @@ -190,7 +190,7 @@ namespace hpx::parallel::util { inline constexpr loop_pred_t loop_pred = loop_pred_t{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_pred( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_pred( Begin begin, End end, Pred&& pred) { return hpx::parallel::util::loop_pred_t{}( @@ -262,7 +262,7 @@ namespace hpx::parallel::util { inline constexpr loop_ind_t loop_ind = loop_ind_t{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_ind( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_ind( Begin begin, End end, F&& f) { return hpx::parallel::util::loop_ind_t{}( @@ -326,7 +326,7 @@ namespace hpx::parallel::util { #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr std::pair loop2( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop2( Begin1 begin1, End1 end1, Begin2 begin2, F&& f) { return hpx::parallel::util::loop2_t{}( @@ -454,7 +454,7 @@ namespace hpx::parallel::util { inline constexpr loop_n_t loop_n = loop_n_t{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Iter loop_n( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_n( Iter it, std::size_t count, F&& f) { return hpx::parallel::util::loop_n_t{}( @@ -463,7 +463,7 @@ namespace hpx::parallel::util { template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Iter loop_n( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_n( Iter it, std::size_t count, CancelToken& tok, F&& f) { return hpx::parallel::util::loop_n_t{}( @@ -593,7 +593,7 @@ namespace hpx::parallel::util { loop_n_ind_t{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Iter loop_n_ind( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_n_ind( Iter it, std::size_t count, F&& f) { return hpx::parallel::util::loop_n_ind_t{}( @@ -602,7 +602,7 @@ namespace hpx::parallel::util { template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Iter loop_n_ind( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_n_ind( Iter it, std::size_t count, CancelToken& tok, F&& f) { return hpx::parallel::util::loop_n_ind_t{}( @@ -745,7 +745,7 @@ namespace hpx::parallel::util { #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_with_cleanup( ExPolicy&& policy, Begin begin, Sent end, F&& f, Cleanup&& cleanup) { return hpx::parallel::util::detail::loop_with_cleanup{}( @@ -755,7 +755,7 @@ namespace hpx::parallel::util { template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_with_cleanup( ExPolicy&& policy, Begin begin, Sent end, Begin2 dest, F&& f, Cleanup&& cleanup) { @@ -1010,7 +1010,8 @@ namespace hpx::parallel::util { detail::loop_with_cleanup_n{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) + loop_with_cleanup_n( ExPolicy&& policy, Begin begin, std::size_t n, F&& f, Cleanup&& cleanup) { return hpx::parallel::util::detail::loop_with_cleanup_n{}( @@ -1020,9 +1021,9 @@ namespace hpx::parallel::util { template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_cleanup_n( - ExPolicy&& policy, Begin begin, std::size_t n, Begin2 dest, F&& f, - Cleanup&& cleanup) + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) + loop_with_cleanup_n(ExPolicy&& policy, Begin begin, std::size_t n, + Begin2 dest, F&& f, Cleanup&& cleanup) { return hpx::parallel::util::detail::loop_with_cleanup_n{}( HPX_FORWARD(ExPolicy, policy), begin, n, dest, HPX_FORWARD(F, f), @@ -1254,7 +1255,7 @@ namespace hpx::parallel::util { loop_idx_n_t{}; #else template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Iter loop_idx_n( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_idx_n( std::size_t base_idx, Iter it, std::size_t count, F&& f) { return hpx::parallel::util::loop_idx_n_t{}( @@ -1263,7 +1264,7 @@ namespace hpx::parallel::util { template - HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Iter loop_idx_n( + HPX_HOST_DEVICE HPX_FORCEINLINE constexpr decltype(auto) loop_idx_n( std::size_t base_idx, Iter it, std::size_t count, CancelToken& tok, F&& f) { From 2a1937e8a5ad8a9ac00bb7c1d8dc2b4d78f182f5 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 19 Aug 2023 10:16:43 -0500 Subject: [PATCH 175/333] Remove obsolete and broken Daint builders --- .jenkins/cscs/Jenkinsfile | 2 +- .jenkins/cscs/env-clang-12.sh | 41 ----------------- .jenkins/cscs/env-clang-cuda.sh | 46 ------------------- .jenkins/cscs/env-gcc-10.sh | 34 -------------- .jenkins/cscs/env-gcc-cuda.sh | 28 ----------- .jenkins/cscs/slurm-constraint-clang-12.sh | 7 --- .jenkins/cscs/slurm-constraint-clang-9.sh | 7 --- .jenkins/cscs/slurm-constraint-clang-cuda.sh | 7 --- .jenkins/cscs/slurm-constraint-gcc-10.sh | 7 --- .jenkins/cscs/slurm-constraint-gcc-8.sh | 7 --- .jenkins/cscs/slurm-constraint-gcc-cuda.sh | 7 --- .jenkins/cscs/slurm-constraint-icc.sh | 7 --- .../tests/regressions/barrier_3792.cpp | 15 +++--- 13 files changed, 10 insertions(+), 205 deletions(-) delete mode 100644 .jenkins/cscs/env-clang-12.sh delete mode 100644 .jenkins/cscs/env-clang-cuda.sh delete mode 100644 .jenkins/cscs/env-gcc-10.sh delete mode 100644 .jenkins/cscs/env-gcc-cuda.sh delete mode 100644 .jenkins/cscs/slurm-constraint-clang-12.sh delete mode 100644 .jenkins/cscs/slurm-constraint-clang-9.sh delete mode 100644 .jenkins/cscs/slurm-constraint-clang-cuda.sh delete mode 100644 .jenkins/cscs/slurm-constraint-gcc-10.sh delete mode 100644 .jenkins/cscs/slurm-constraint-gcc-8.sh delete mode 100644 .jenkins/cscs/slurm-constraint-gcc-cuda.sh delete mode 100644 .jenkins/cscs/slurm-constraint-icc.sh diff --git a/.jenkins/cscs/Jenkinsfile b/.jenkins/cscs/Jenkinsfile index 817f2610cfb1..32c63feb7a8c 100644 --- a/.jenkins/cscs/Jenkinsfile +++ b/.jenkins/cscs/Jenkinsfile @@ -42,7 +42,7 @@ pipeline { // number of node-hours consumed axis { name 'configuration_name' - values 'gcc-11', 'gcc-cuda', 'clang-10', 'clang-13', 'clang-cuda', 'clang-apex' + values 'gcc-11', 'clang-10', 'clang-13', 'clang-apex' } axis { name 'build_type' diff --git a/.jenkins/cscs/env-clang-12.sh b/.jenkins/cscs/env-clang-12.sh deleted file mode 100644 index 402bfed89ab0..000000000000 --- a/.jenkins/cscs/env-clang-12.sh +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -export CRAYPE_LINK_TYPE=dynamic -export APPS_ROOT="/apps/daint/SSL/HPX/packages" -export CLANG_VER="12.0.0" -export CXX_STD="20" -export BOOST_VER="1.77.0" -export HWLOC_VER="2.2.0" -export CLANG_ROOT="${APPS_ROOT}/llvm-${CLANG_VER}" -export BOOST_ROOT="${APPS_ROOT}/boost-${BOOST_VER}-clang-${CLANG_VER}-c++20-release" -export HWLOC_ROOT="${APPS_ROOT}/hwloc-${HWLOC_VER}-gcc-10.2.0" -export CXXFLAGS="-Wno-unused-command-line-argument -stdlib=libc++ -nostdinc++ -isystem${CLANG_ROOT}/include/c++/v1 -L${CLANG_ROOT}/lib -Wl,-rpath,${CLANG_ROOT}/lib,-lsupc++" -export LDCXXFLAGS="-stdlib=libc++ -L${CLANG_ROOT}/lib -Wl,-rpath,${CLANG_ROOT}/lib,-lsupc++" -export CXX="${CLANG_ROOT}/bin/clang++" -export CC="${CLANG_ROOT}/bin/clang" -export CPP="${CLANG_ROOT}/bin/clang -E" - -module load daint-mc -spack load cmake@3.18.6 -spack load ninja@1.10.0 - -configure_extra_options+=" -DHPX_WITH_MALLOC=system" -configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" -configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_DYNAMIC_HPX_MAIN=OFF" -configure_extra_options+=" -DHPX_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -configure_extra_options+=" -DHPX_WITH_UNITY_BUILD=ON" - -# enable extra counters to verify everything compiles -configure_extra_options+=" -DHPX_WITH_PARCELPORT_ACTION_COUNTERS=ON" -configure_extra_options+=" -DHPX_WITH_THREAD_IDLE_RATES=ON" -configure_extra_options+=" -DHPX_WITH_THREAD_CREATION_AND_CLEANUP_RATES=ON" -configure_extra_options+=" -DHPX_WITH_THREAD_CUMULATIVE_COUNTS=ON" -configure_extra_options+=" -DHPX_WITH_THREAD_QUEUE_WAITTIME=ON" -configure_extra_options+=" -DHPX_WITH_THREAD_STEALING_COUNTS=ON" diff --git a/.jenkins/cscs/env-clang-cuda.sh b/.jenkins/cscs/env-clang-cuda.sh deleted file mode 100644 index ce5150aa40d6..000000000000 --- a/.jenkins/cscs/env-clang-cuda.sh +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -export CRAYPE_LINK_TYPE=dynamic -export APPS_ROOT="/apps/daint/SSL/HPX/packages" -export CXX_STD="17" -export HWLOC_ROOT="${APPS_ROOT}/hwloc-2.0.3-gcc-8.3.0" -export BOOST_ROOT="${APPS_ROOT}/boost-1.75.0-gcc-10.1.0-c++17-debug/" - -module load daint-gpu -module load cudatoolkit/11.0.2_3.38-8.1__g5b73779 -spack load cmake@3.18.6 -spack load ninja@1.10.0 - -export CXX=`which CC` -export CC=`which cc` - -configure_extra_options+=" -DHPX_WITH_MAX_CPU_COUNT=64" -configure_extra_options+=" -DHPX_WITH_CUDA=ON" -configure_extra_options+=" -DHPX_WITH_MALLOC=system" -configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" -configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" -configure_extra_options+=" -DCMAKE_CUDA_COMPILER=$(which $CXX)" -configure_extra_options+=" -DCMAKE_CUDA_ARCHITECTURES=60" - -# The build unit test with HPX in Debug and the hello_world project in Debug -# mode hangs on this configuration Release-Debug, Debug-Release, and -# Release-Release do not hang. -configure_extra_options+=" -DHPX_WITH_TESTS_EXTERNAL_BUILD=OFF" - -# This is a workaround for a bug in the Cray clang compiler and/or Boost. When -# compiling in device mode, Boost detects that float128 is available, but -# compilation later fails with an error message saying float128 is not -# available for the target. -# -# This sets a custom Boost user configuration, which is a concatenation of the -# clang and nvcc compiler configurations, with the exception that -# BOOST_HAS_FLOAT128 is unconditionally disabled. -configure_extra_options+=" \"-DCMAKE_CXX_FLAGS=-I${src_dir}/.jenkins/cscs/ -DBOOST_USER_CONFIG=''\"" -configure_extra_options+=" \"-DCMAKE_CUDA_FLAGS=--cuda-gpu-arch=sm_60 -I${src_dir}/.jenkins/cscs/ -DBOOST_USER_CONFIG=''\"" diff --git a/.jenkins/cscs/env-gcc-10.sh b/.jenkins/cscs/env-gcc-10.sh deleted file mode 100644 index c1189ca53667..000000000000 --- a/.jenkins/cscs/env-gcc-10.sh +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -export CRAYPE_LINK_TYPE=dynamic -export APPS_ROOT="/apps/daint/SSL/HPX/packages" -export GCC_VER="10.1.0" -export CXX_STD="17" -export BOOST_VER="1.75.0" -export HWLOC_VER="2.2.0" -export GCC_ROOT="${APPS_ROOT}/gcc-${GCC_VER}" -export BOOST_ROOT="${APPS_ROOT}/boost-${BOOST_VER}-gcc-${GCC_VER}-c++${CXX_STD}-debug" -export HWLOC_ROOT="${APPS_ROOT}/hwloc-${HWLOC_VER}-gcc-${GCC_VER}" -export CXXFLAGS="-nostdinc++ -I${GCC_ROOT}/include/c++/${GCC_VER} -I${GCC_ROOT}/include/c++/${GCC_VER}/x86_64-unknown-linux-gnu -I${GCC_ROOT}/include/c++/${GCC_VER}/x86_64-pc-linux-gnu -L${GCC_ROOT}/lib64 -Wl,-rpath,${GCC_ROOT}/lib64" -export LDFLAGS="-L${GCC_ROOT}/lib64" -export CXX=${GCC_ROOT}/bin/g++ -export CC=${GCC_ROOT}/bin/gcc - -module load daint-mc -spack load cmake@3.18.6 -spack load ninja@1.10.0 - -configure_extra_options+=" -DHPX_WITH_MAX_CPU_COUNT=128" -configure_extra_options+=" -DHPX_WITH_MALLOC=system" -configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" -configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" -configure_extra_options+=" -DHPX_WITH_NETWORKING=OFF" -configure_extra_options+=" -DHPX_WITH_DISTRIBUTED_RUNTIME=OFF" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" - diff --git a/.jenkins/cscs/env-gcc-cuda.sh b/.jenkins/cscs/env-gcc-cuda.sh deleted file mode 100644 index dd0b0b11f6b0..000000000000 --- a/.jenkins/cscs/env-gcc-cuda.sh +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -export CRAYPE_LINK_TYPE=dynamic -export CXX_STD="17" - -module load daint-gpu -module switch PrgEnv-cray PrgEnv-gnu -module load cudatoolkit -module load Boost/1.78.0-CrayGNU-21.09 -module load hwloc/2.4.1 -spack load cmake@3.18.6 -spack load ninja@1.10.0 - -export CXX=`which CC` -export CC=`which cc` - -configure_extra_options+=" -DHPX_WITH_MALLOC=system" -configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" -configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" -configure_extra_options+=" -DHPX_WITH_CUDA=ON" -configure_extra_options+=" -DHPX_WITH_EXAMPLES_OPENMP=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHWLOC_ROOT=${EBROOTHWLOC}" diff --git a/.jenkins/cscs/slurm-constraint-clang-12.sh b/.jenkins/cscs/slurm-constraint-clang-12.sh deleted file mode 100644 index 1f9f300cc8fd..000000000000 --- a/.jenkins/cscs/slurm-constraint-clang-12.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="mc" diff --git a/.jenkins/cscs/slurm-constraint-clang-9.sh b/.jenkins/cscs/slurm-constraint-clang-9.sh deleted file mode 100644 index 1f9f300cc8fd..000000000000 --- a/.jenkins/cscs/slurm-constraint-clang-9.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="mc" diff --git a/.jenkins/cscs/slurm-constraint-clang-cuda.sh b/.jenkins/cscs/slurm-constraint-clang-cuda.sh deleted file mode 100644 index 87293c78712d..000000000000 --- a/.jenkins/cscs/slurm-constraint-clang-cuda.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="gpu" diff --git a/.jenkins/cscs/slurm-constraint-gcc-10.sh b/.jenkins/cscs/slurm-constraint-gcc-10.sh deleted file mode 100644 index 1f9f300cc8fd..000000000000 --- a/.jenkins/cscs/slurm-constraint-gcc-10.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="mc" diff --git a/.jenkins/cscs/slurm-constraint-gcc-8.sh b/.jenkins/cscs/slurm-constraint-gcc-8.sh deleted file mode 100644 index 1f9f300cc8fd..000000000000 --- a/.jenkins/cscs/slurm-constraint-gcc-8.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="mc" diff --git a/.jenkins/cscs/slurm-constraint-gcc-cuda.sh b/.jenkins/cscs/slurm-constraint-gcc-cuda.sh deleted file mode 100644 index 87293c78712d..000000000000 --- a/.jenkins/cscs/slurm-constraint-gcc-cuda.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="gpu" diff --git a/.jenkins/cscs/slurm-constraint-icc.sh b/.jenkins/cscs/slurm-constraint-icc.sh deleted file mode 100644 index 1f9f300cc8fd..000000000000 --- a/.jenkins/cscs/slurm-constraint-icc.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_constraint="mc" diff --git a/libs/full/collectives/tests/regressions/barrier_3792.cpp b/libs/full/collectives/tests/regressions/barrier_3792.cpp index a24dd0509e3d..519a3d081f99 100644 --- a/libs/full/collectives/tests/regressions/barrier_3792.cpp +++ b/libs/full/collectives/tests/regressions/barrier_3792.cpp @@ -20,7 +20,7 @@ #include /////////////////////////////////////////////////////////////////////////////// -void run_barrier_test1(std::vector locs) +void run_barrier_test1(std::vector const& locs) { auto loc_it = std::find(locs.begin(), locs.end(), hpx::get_locality_id()); if (loc_it == locs.end()) @@ -28,20 +28,20 @@ void run_barrier_test1(std::vector locs) std::size_t barrier_rank = std::distance(locs.begin(), loc_it); - std::string barrier_name = - "/loc_list/barrier" + std::to_string(locs[0]) + std::to_string(locs[1]); + std::string barrier_name = "/loc_list/barrier1" + std::to_string(locs[0]) + + std::to_string(locs[1]); hpx::distributed::barrier b(barrier_name, locs.size(), barrier_rank); b.wait(); } -void run_barrier_test2(std::vector locs) +void run_barrier_test2(std::vector const& locs) { auto loc_it = std::find(locs.begin(), locs.end(), hpx::get_locality_id()); if (loc_it == locs.end()) return; - std::string barrier_name = - "/loc_list/barrier" + std::to_string(locs[0]) + std::to_string(locs[1]); + std::string barrier_name = "/loc_list/barrier2" + std::to_string(locs[0]) + + std::to_string(locs[1]); hpx::distributed::barrier b(barrier_name, locs, hpx::get_locality_id()); b.wait(); } @@ -53,12 +53,15 @@ int hpx_main() std::vector locs_0{0, 1}; run_barrier_test1(locs_0); run_barrier_test2(locs_0); + std::vector locs_1{0, 2}; run_barrier_test1(locs_1); run_barrier_test2(locs_1); + std::vector locs_2{1, 2}; run_barrier_test1(locs_2); run_barrier_test2(locs_2); + return hpx::finalize(); } From 7e527d0f429d9ebaebaf2678e928a382a6ef1399 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 25 Jul 2023 15:15:44 -0500 Subject: [PATCH 176/333] Making sure changed number of counts is propagated to executor --- .../container_algorithms/is_sorted.hpp | 2 +- .../hpx/parallel/util/detail/chunk_size.hpp | 56 +++- .../hpx/parallel/util/foreach_partitioner.hpp | 10 +- .../include/hpx/parallel/util/partitioner.hpp | 21 +- .../tests/regressions/CMakeLists.txt | 1 + .../tests/regressions/num_cores.cpp | 41 +++ .../tests/unit/algorithms/rotate_sender.cpp | 8 +- .../is_sorted_until_range.cpp | 43 +-- .../executors/execution_parameters_fwd.hpp | 23 +- .../hpx/execution/executors/num_cores.hpp | 9 + .../tests/regressions/is_executor_1691.cpp | 1 + .../unit/executor_parameters_dispatching.cpp | 280 ++++++++---------- .../disable_thread_stealing_executor.cpp | 32 ++ .../examples/executor_with_thread_hooks.cpp | 6 + .../hpx/executors/annotating_executor.hpp | 10 +- .../executors/datapar/execution_policy.hpp | 114 ++++++- .../datapar/execution_policy_fwd.hpp | 8 +- .../hpx/executors/execution_policy.hpp | 241 ++++++++++++++- .../hpx/executors/execution_policy_fwd.hpp | 16 +- .../executors/execution_policy_mappings.hpp | 2 +- .../executors/explicit_scheduler_executor.hpp | 18 +- .../hpx/executors/parallel_executor.hpp | 62 +++- .../restricted_thread_pool_executor.hpp | 15 +- .../hpx/executors/scheduler_executor.hpp | 18 +- .../hpx/executors/sequenced_executor.hpp | 29 +- .../hpx/executors/thread_pool_scheduler.hpp | 53 +++- .../executors/thread_pool_scheduler_bulk.hpp | 8 +- .../tests/regressions/pu_count_6184.cpp | 2 +- .../hpx/resiliency/replay_executor.hpp | 56 +++- .../hpx/resiliency/replicate_executor.hpp | 63 +++- .../functional/detail/tag_fallback_invoke.hpp | 4 +- .../functional/detail/tag_priority_invoke.hpp | 6 +- .../include/hpx/functional/tag_invoke.hpp | 2 +- .../examples/1d_stencil_4_checkpoint.cpp | 88 +++--- 34 files changed, 984 insertions(+), 364 deletions(-) create mode 100644 libs/core/algorithms/tests/regressions/num_cores.cpp diff --git a/libs/core/algorithms/include/hpx/parallel/container_algorithms/is_sorted.hpp b/libs/core/algorithms/include/hpx/parallel/container_algorithms/is_sorted.hpp index beed8d6b620f..f72353ebd092 100644 --- a/libs/core/algorithms/include/hpx/parallel/container_algorithms/is_sorted.hpp +++ b/libs/core/algorithms/include/hpx/parallel/container_algorithms/is_sorted.hpp @@ -508,7 +508,7 @@ namespace hpx { namespace ranges { namespace hpx::ranges { - inline constexpr struct is_sorted_t final + inline constexpr struct is_sorted_t : hpx::detail::tag_parallel_algorithm { private: diff --git a/libs/core/algorithms/include/hpx/parallel/util/detail/chunk_size.hpp b/libs/core/algorithms/include/hpx/parallel/util/detail/chunk_size.hpp index 22836ce2de03..691c399c5744 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/detail/chunk_size.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/detail/chunk_size.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include @@ -132,7 +134,7 @@ namespace hpx::parallel::util::detail { template hpx::util::iterator_range> - get_bulk_iteration_shape(ExPolicy&& policy, IterOrR& it_or_r, + get_bulk_iteration_shape(ExPolicy& policy, IterOrR& it_or_r, std::size_t& count, Stride s = Stride(1)) { if (count == 0) @@ -166,6 +168,10 @@ namespace hpx::parallel::util::detail { // clang-format on } + // update executor with new values + policy = hpx::experimental::prefer( + execution::with_processing_units_count, policy, cores); + auto shape_begin = chunk_size_iterator(it_or_r, chunk_size, count); auto shape_end = chunk_size_iterator(last, chunk_size, count, count); @@ -175,7 +181,7 @@ namespace hpx::parallel::util::detail { template hpx::util::iterator_range> - get_bulk_iteration_shape(ExPolicy&& policy, std::vector& workitems, + get_bulk_iteration_shape(ExPolicy& policy, std::vector& workitems, F1&& f1, IterOrR& it_or_r, std::size_t& count, Stride s = Stride(1)) { if (count == 0) @@ -241,6 +247,10 @@ namespace hpx::parallel::util::detail { // clang-format on } + // update executor with new values + policy = hpx::experimental::prefer( + execution::with_processing_units_count, policy, cores); + auto shape_begin = chunk_size_iterator(it_or_r, chunk_size, count); auto shape_end = chunk_size_iterator(last, chunk_size, count, count); @@ -250,7 +260,7 @@ namespace hpx::parallel::util::detail { template std::vector> - get_bulk_iteration_shape_variable(ExPolicy&& policy, IterOrR& it_or_r, + get_bulk_iteration_shape_variable(ExPolicy& policy, IterOrR& it_or_r, std::size_t& count, Stride s = Stride(1)) { using tuple_type = hpx::tuple; @@ -308,27 +318,31 @@ namespace hpx::parallel::util::detail { } // clang-format on + // update executor with new values + policy = hpx::experimental::prefer( + execution::with_processing_units_count, policy, cores); + return shape; } template - decltype(auto) get_bulk_iteration_shape(std::false_type, ExPolicy&& policy, + decltype(auto) get_bulk_iteration_shape(std::false_type, ExPolicy& policy, std::vector& workitems, F1&& f1, FwdIter& begin, std::size_t& count, Stride s = Stride(1)) { - return get_bulk_iteration_shape(HPX_FORWARD(ExPolicy, policy), - workitems, HPX_FORWARD(F1, f1), begin, count, s); + return get_bulk_iteration_shape( + policy, workitems, HPX_FORWARD(F1, f1), begin, count, s); } template - decltype(auto) get_bulk_iteration_shape(std::true_type, ExPolicy&& policy, + decltype(auto) get_bulk_iteration_shape(std::true_type, ExPolicy& policy, std::vector& workitems, F1&& f1, FwdIter& begin, std::size_t& count, Stride s = Stride(1)) { - return get_bulk_iteration_shape_variable(HPX_FORWARD(ExPolicy, policy), - workitems, HPX_FORWARD(F1, f1), begin, count, s); + return get_bulk_iteration_shape_variable( + policy, workitems, HPX_FORWARD(F1, f1), begin, count, s); } /////////////////////////////////////////////////////////////////////////// @@ -360,7 +374,7 @@ namespace hpx::parallel::util::detail { typename Stride = std::size_t> hpx::util::iterator_range< parallel::util::detail::chunk_size_idx_iterator> - get_bulk_iteration_shape_idx(ExPolicy&& policy, FwdIter begin, + get_bulk_iteration_shape_idx(ExPolicy& policy, FwdIter begin, std::size_t count, Stride s = Stride(1)) { using iterator = @@ -397,6 +411,13 @@ namespace hpx::parallel::util::detail { // clang-format on } + // update executor with new values + policy = hpx::experimental::prefer( + execution::with_processing_units_count, policy, cores); + + using iterator = + parallel::util::detail::chunk_size_idx_iterator; + iterator shape_begin(begin, chunk_size, count, 0, 0); iterator shape_end(last, chunk_size, count, count, 0); @@ -407,7 +428,7 @@ namespace hpx::parallel::util::detail { typename Stride = std::size_t> hpx::util::iterator_range< parallel::util::detail::chunk_size_idx_iterator> - get_bulk_iteration_shape_idx(ExPolicy&& policy, + get_bulk_iteration_shape_idx(ExPolicy& policy, std::vector& workitems, F1&& f1, FwdIter begin, std::size_t count, Stride s = Stride(1)) { @@ -475,6 +496,13 @@ namespace hpx::parallel::util::detail { // clang-format on } + // update executor with new values + policy = hpx::experimental::prefer( + execution::with_processing_units_count, policy, cores); + + using iterator = + parallel::util::detail::chunk_size_idx_iterator; + iterator shape_begin(begin, chunk_size, count, 0, base_idx); iterator shape_end(last, chunk_size, count, count, base_idx); @@ -484,7 +512,7 @@ namespace hpx::parallel::util::detail { template std::vector> - get_bulk_iteration_shape_idx_variable(ExPolicy&& policy, FwdIter first, + get_bulk_iteration_shape_idx_variable(ExPolicy& policy, FwdIter first, std::size_t count, Stride s = Stride(1)) { using tuple_type = hpx::tuple; @@ -543,6 +571,10 @@ namespace hpx::parallel::util::detail { } // clang-format on + // update executor with new values + policy = hpx::experimental::prefer( + execution::with_processing_units_count, policy, cores); + return shape; } } // namespace hpx::parallel::util::detail diff --git a/libs/core/algorithms/include/hpx/parallel/util/foreach_partitioner.hpp b/libs/core/algorithms/include/hpx/parallel/util/foreach_partitioner.hpp index fccac47d871c..a47090890fc9 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/foreach_partitioner.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/foreach_partitioner.hpp @@ -36,7 +36,7 @@ namespace hpx::parallel::util::detail { template auto foreach_partition( - ExPolicy&& policy, FwdIter first, std::size_t count, F&& f) + ExPolicy policy, FwdIter first, std::size_t count, F&& f) { // estimate a chunk size based on number of cores used using parameters_type = @@ -53,7 +53,7 @@ namespace hpx::parallel::util::detail { "has_variable_chunk_size and invokes_testing_function"); auto&& shape = detail::get_bulk_iteration_shape_idx_variable( - HPX_FORWARD(ExPolicy, policy), first, count); + policy, first, count); return execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -61,8 +61,8 @@ namespace hpx::parallel::util::detail { } else if constexpr (!invokes_testing_function) { - auto&& shape = detail::get_bulk_iteration_shape_idx( - HPX_FORWARD(ExPolicy, policy), first, count); + auto&& shape = + detail::get_bulk_iteration_shape_idx(policy, first, count); return execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -72,7 +72,7 @@ namespace hpx::parallel::util::detail { { std::vector> inititems; auto&& shape = detail::get_bulk_iteration_shape_idx( - HPX_FORWARD(ExPolicy, policy), inititems, f, first, count); + policy, inititems, f, first, count); auto&& workitems = execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, diff --git a/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp b/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp index 2decc47e4f19..d458e422a6e5 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/partitioner.hpp @@ -40,7 +40,7 @@ namespace hpx::parallel::util::detail { template - auto partition(ExPolicy&& policy, IterOrR it_or_r, std::size_t count, F&& f) + auto partition(ExPolicy policy, IterOrR it_or_r, std::size_t count, F&& f) { // estimate a chunk size based on number of cores used using parameters_type = @@ -57,7 +57,7 @@ namespace hpx::parallel::util::detail { "has_variable_chunk_size and invokes_testing_function"); auto&& shape = detail::get_bulk_iteration_shape_variable( - HPX_FORWARD(ExPolicy, policy), it_or_r, count); + policy, it_or_r, count); return execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -65,8 +65,8 @@ namespace hpx::parallel::util::detail { } else if constexpr (!invokes_testing_function) { - auto&& shape = detail::get_bulk_iteration_shape( - HPX_FORWARD(ExPolicy, policy), it_or_r, count); + auto&& shape = + detail::get_bulk_iteration_shape(policy, it_or_r, count); return execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -76,7 +76,7 @@ namespace hpx::parallel::util::detail { { std::vector> inititems; auto&& shape = detail::get_bulk_iteration_shape( - HPX_FORWARD(ExPolicy, policy), inititems, f, it_or_r, count); + policy, inititems, f, it_or_r, count); auto&& workitems = execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -88,8 +88,8 @@ namespace hpx::parallel::util::detail { template - auto partition_with_index(ExPolicy&& policy, FwdIter first, - std::size_t count, Stride stride, F&& f) + auto partition_with_index( + ExPolicy policy, FwdIter first, std::size_t count, Stride stride, F&& f) { // estimate a chunk size based on number of cores used using parameters_type = @@ -106,7 +106,7 @@ namespace hpx::parallel::util::detail { "has_variable_chunk_size and invokes_testing_function"); auto&& shape = detail::get_bulk_iteration_shape_idx_variable( - HPX_FORWARD(ExPolicy, policy), first, count, stride); + policy, first, count, stride); return execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -115,7 +115,7 @@ namespace hpx::parallel::util::detail { else if constexpr (!invokes_testing_function) { auto&& shape = detail::get_bulk_iteration_shape_idx( - HPX_FORWARD(ExPolicy, policy), first, count, stride); + policy, first, count, stride); return execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, @@ -125,8 +125,7 @@ namespace hpx::parallel::util::detail { { std::vector> inititems; auto&& shape = detail::get_bulk_iteration_shape_idx( - HPX_FORWARD(ExPolicy, policy), inititems, f, first, count, - stride); + policy, inititems, f, first, count, stride); auto&& workitems = execution::bulk_async_execute(policy.executor(), partitioner_iteration{HPX_FORWARD(F, f)}, diff --git a/libs/core/algorithms/tests/regressions/CMakeLists.txt b/libs/core/algorithms/tests/regressions/CMakeLists.txt index e4a023dad590..9c132a490c22 100644 --- a/libs/core/algorithms/tests/regressions/CMakeLists.txt +++ b/libs/core/algorithms/tests/regressions/CMakeLists.txt @@ -12,6 +12,7 @@ set(tests for_loop_5735 for_loop_with_auto_chunk_size minimal_findend + num_cores reduce_3641 scan_different_inits scan_non_commutative diff --git a/libs/core/algorithms/tests/regressions/num_cores.cpp b/libs/core/algorithms/tests/regressions/num_cores.cpp new file mode 100644 index 000000000000..de0287bd02cb --- /dev/null +++ b/libs/core/algorithms/tests/regressions/num_cores.cpp @@ -0,0 +1,41 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include + +#include + +int hpx_main() +{ + hpx::execution::experimental::num_cores nc(2); + auto policy = hpx::execution::par.with(nc); + + HPX_TEST_EQ( + hpx::parallel::execution::processing_units_count(policy.parameters(), + policy.executor(), hpx::chrono::null_duration, 0), + static_cast(2)); + + auto policy2 = + hpx::parallel::execution::with_processing_units_count(policy, 2); + HPX_TEST_EQ(hpx::parallel::execution::processing_units_count( + hpx::execution::par.parameters(), policy2.executor(), + hpx::chrono::null_duration, 0), + static_cast(2)); + + return hpx::local::finalize(); +} + +int main(int argc, char* argv[]) +{ + HPX_TEST_EQ_MSG(hpx::local::init(hpx_main, argc, argv), 0, + "HPX main exited with non-zero status"); + + return hpx::util::report_errors(); +} diff --git a/libs/core/algorithms/tests/unit/algorithms/rotate_sender.cpp b/libs/core/algorithms/tests/unit/algorithms/rotate_sender.cpp index 3c52d08828a2..44dda8249d61 100644 --- a/libs/core/algorithms/tests/unit/algorithms/rotate_sender.cpp +++ b/libs/core/algorithms/tests/unit/algorithms/rotate_sender.cpp @@ -46,7 +46,7 @@ void test_rotate_direct(Policy l, ExPolicy&& policy, IteratorTag) std::iota(std::begin(c), std::end(c), std::rand()); std::copy(std::begin(c), std::end(c), std::back_inserter(d1)); - std::size_t mid_pos = std::rand() % c.size(); //-V104 + std::size_t const mid_pos = std::rand() % c.size(); //-V104 base_iterator mid = std::begin(c); std::advance(mid, mid_pos); @@ -88,7 +88,7 @@ void test_rotate(Policy l, ExPolicy&& policy, IteratorTag) std::iota(std::begin(c), std::end(c), std::rand()); std::copy(std::begin(c), std::end(c), std::back_inserter(d1)); - std::size_t mid_pos = std::rand() % c.size(); //-V104 + std::size_t const mid_pos = std::rand() % c.size(); //-V104 base_iterator mid = std::begin(c); std::advance(mid, mid_pos); @@ -127,7 +127,7 @@ void test_rotate_async_direct(Policy l, ExPolicy&& p, IteratorTag) std::iota(std::begin(c), std::end(c), std::rand()); std::copy(std::begin(c), std::end(c), std::back_inserter(d1)); - std::size_t mid_pos = std::rand() % c.size(); //-V104 + std::size_t const mid_pos = std::rand() % c.size(); //-V104 base_iterator mid = std::begin(c); std::advance(mid, mid_pos); @@ -191,7 +191,7 @@ void rotate_test() int hpx_main(hpx::program_options::variables_map& vm) { - unsigned int seed = (unsigned int) std::time(nullptr); + unsigned int seed = static_cast(std::time(nullptr)); if (vm.count("seed")) seed = vm["seed"].as(); diff --git a/libs/core/algorithms/tests/unit/container_algorithms/is_sorted_until_range.cpp b/libs/core/algorithms/tests/unit/container_algorithms/is_sorted_until_range.cpp index bb40a007a334..f60e99669ff0 100644 --- a/libs/core/algorithms/tests/unit/container_algorithms/is_sorted_until_range.cpp +++ b/libs/core/algorithms/tests/unit/container_algorithms/is_sorted_until_range.cpp @@ -5,6 +5,7 @@ // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include +#include #include #include @@ -1214,7 +1215,7 @@ void test_sorted_until_bad_alloc(ExPolicy policy, IteratorTag) decorated_iterator( std::end(c), []() { throw std::runtime_error("test"); })); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1232,7 +1233,7 @@ void test_sorted_until_bad_alloc(ExPolicy policy, IteratorTag) iterator(std::end(c)), [](int, int) -> bool { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1250,7 +1251,7 @@ void test_sorted_until_bad_alloc(ExPolicy policy, IteratorTag) iterator(std::end(c)), std::less(), [](int) -> int { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1285,7 +1286,7 @@ void test_sorted_until_async_bad_alloc(ExPolicy p, IteratorTag) HPX_TEST(false); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1306,7 +1307,7 @@ void test_sorted_until_async_bad_alloc(ExPolicy p, IteratorTag) HPX_TEST(false); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1327,7 +1328,7 @@ void test_sorted_until_async_bad_alloc(ExPolicy p, IteratorTag) HPX_TEST(false); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1361,7 +1362,7 @@ void test_sorted_until_seq_bad_alloc(IteratorTag) decorated_iterator( std::end(c), []() { throw std::runtime_error("test"); })); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1379,7 +1380,7 @@ void test_sorted_until_seq_bad_alloc(IteratorTag) iterator(std::end(c)), [](int, int) -> int { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1397,7 +1398,7 @@ void test_sorted_until_seq_bad_alloc(IteratorTag) iterator(std::end(c)), std::less(), [](int) -> int { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1426,7 +1427,7 @@ void test_sorted_until_bad_alloc(ExPolicy policy) hpx::ranges::is_sorted_until(policy, c, [](int, int) -> bool { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1443,7 +1444,7 @@ void test_sorted_until_bad_alloc(ExPolicy policy) hpx::ranges::is_sorted_until(policy, c, std::less(), [](int) -> int { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1470,7 +1471,7 @@ void test_sorted_until_async_bad_alloc(ExPolicy p) HPX_TEST(false); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1490,7 +1491,7 @@ void test_sorted_until_async_bad_alloc(ExPolicy p) HPX_TEST(false); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1515,7 +1516,7 @@ void test_sorted_until_seq_bad_alloc() hpx::ranges::is_sorted_until( c, [](int, int) -> int { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1532,7 +1533,7 @@ void test_sorted_until_seq_bad_alloc() hpx::ranges::is_sorted_until(c, std::less(), [](int) -> int { throw std::runtime_error("test"); }); } - catch (hpx::exception_list const& e) + catch (hpx::exception_list const&) { caught_bad_alloc = true; } @@ -1585,12 +1586,12 @@ int hpx_main() sorted_until_exception_test(); sorted_until_bad_alloc_test(); - std::vector c(100); - hpx::future f = - hpx::dataflow(hpx::ranges::is_sorted, hpx::execution::par, c); - f = hpx::dataflow( - hpx::unwrapping(hpx::ranges::is_sorted), hpx::execution::par, c, f); - f.get(); + // std::vector c(100); + // hpx::future f = + // hpx::dataflow(hpx::ranges::is_sorted, hpx::execution::par, c); + // f = hpx::dataflow( + // hpx::unwrapping(hpx::ranges::is_sorted), hpx::execution::par, c, f); + // f.get(); return hpx::local::finalize(); } diff --git a/libs/core/execution/include/hpx/execution/executors/execution_parameters_fwd.hpp b/libs/core/execution/include/hpx/execution/executors/execution_parameters_fwd.hpp index ba4b834d6d89..856e389abf6e 100644 --- a/libs/core/execution/include/hpx/execution/executors/execution_parameters_fwd.hpp +++ b/libs/core/execution/include/hpx/execution/executors/execution_parameters_fwd.hpp @@ -64,6 +64,16 @@ namespace hpx::parallel::execution { /////////////////////////////////////////////////////////////////////////// // define customization points + inline constexpr struct null_parameters_t + { + } null_parameters{}; + + /// \cond NOINTERNAL + template <> + struct is_executor_parameters : std::true_type + { + }; + /// \endcond /// Return the number of invocations of the given function \a f which should /// be combined into a single task @@ -287,7 +297,7 @@ namespace hpx::parallel::execution { // clang-format off template ::value + hpx::traits::is_executor_any_v )> // clang-format on friend HPX_FORCEINLINE decltype(auto) tag_fallback_invoke( @@ -295,22 +305,23 @@ namespace hpx::parallel::execution { hpx::chrono::steady_duration const& iteration_duration, std::size_t num_tasks) { - return detail::processing_units_count_fn_helper>::call(HPX_FORWARD(Executor, exec), - iteration_duration, num_tasks); + return detail::processing_units_count_fn_helper>::call(null_parameters, + HPX_FORWARD(Executor, exec), iteration_duration, num_tasks); } // clang-format off template ::value + hpx::traits::is_executor_any_v )> // clang-format on friend HPX_FORCEINLINE decltype(auto) tag_fallback_invoke( processing_units_count_t tag, Executor&& exec, std::size_t num_tasks = 0) { - return tag(HPX_FORWARD(Executor, exec), hpx::chrono::null_duration, + return tag(hpx::parallel::execution::null_parameters, + HPX_FORWARD(Executor, exec), hpx::chrono::null_duration, num_tasks); } } processing_units_count{}; diff --git a/libs/core/execution/include/hpx/execution/executors/num_cores.hpp b/libs/core/execution/include/hpx/execution/executors/num_cores.hpp index eb3a0e128d73..ab02e9b4f2c9 100644 --- a/libs/core/execution/include/hpx/execution/executors/num_cores.hpp +++ b/libs/core/execution/include/hpx/execution/executors/num_cores.hpp @@ -35,6 +35,15 @@ namespace hpx::execution::experimental { /// \cond NOINTERNAL // discover the number of cores to use for parallelization + template + friend std::size_t tag_invoke( + hpx::parallel::execution::processing_units_count_t, + num_cores params, Executor&&, hpx::chrono::steady_duration const&, + std::size_t) noexcept + { + return params.num_cores_; + } + template constexpr std::size_t processing_units_count(Executor&&, hpx::chrono::steady_duration const&, std::size_t) const noexcept diff --git a/libs/core/execution/tests/regressions/is_executor_1691.cpp b/libs/core/execution/tests/regressions/is_executor_1691.cpp index 3184f83b7721..33e4902cd9e3 100644 --- a/libs/core/execution/tests/regressions/is_executor_1691.cpp +++ b/libs/core/execution/tests/regressions/is_executor_1691.cpp @@ -17,6 +17,7 @@ struct my_executor : hpx::execution::parallel_executor }; namespace hpx::parallel::execution { + template <> struct is_one_way_executor : std::true_type { diff --git a/libs/core/execution/tests/unit/executor_parameters_dispatching.cpp b/libs/core/execution/tests/unit/executor_parameters_dispatching.cpp index 0947189d345f..83296b9bb67a 100644 --- a/libs/core/execution/tests/unit/executor_parameters_dispatching.cpp +++ b/libs/core/execution/tests/unit/executor_parameters_dispatching.cpp @@ -27,10 +27,7 @@ std::atomic exec_count(0); struct test_executor_get_chunk_size : hpx::execution::parallel_executor { - test_executor_get_chunk_size() - : hpx::execution::parallel_executor() - { - } + test_executor_get_chunk_size() = default; template static std::size_t get_chunk_size(Parameters&& /* params */, @@ -42,12 +39,11 @@ struct test_executor_get_chunk_size : hpx::execution::parallel_executor } }; -namespace hpx::parallel::execution { - template <> - struct is_two_way_executor : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_two_way_executor< + test_executor_get_chunk_size> : std::true_type +{ +}; struct test_chunk_size { @@ -61,14 +57,11 @@ struct test_chunk_size } }; -namespace hpx::parallel::execution { - /// \cond NOINTERNAL - template <> - struct is_executor_parameters : std::true_type - { - }; - /// \endcond -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_executor_parameters + : std::true_type +{ +}; /////////////////////////////////////////////////////////////////////////////// void test_get_chunk_size() @@ -80,8 +73,8 @@ void test_get_chunk_size() hpx::parallel::execution::get_chunk_size( test_chunk_size{}, hpx::execution::par.executor(), 1, 1); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -91,8 +84,8 @@ void test_get_chunk_size() hpx::parallel::execution::get_chunk_size(test_chunk_size{}, hpx::execution::par.executor(), hpx::chrono::null_duration, 1, 1); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -102,8 +95,8 @@ void test_get_chunk_size() hpx::parallel::execution::get_chunk_size( test_chunk_size{}, test_executor_get_chunk_size{}, 1, 1); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } { @@ -113,8 +106,8 @@ void test_get_chunk_size() hpx::parallel::execution::get_chunk_size(test_chunk_size{}, test_executor_get_chunk_size{}, hpx::chrono::null_duration, 1, 1); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } @@ -123,10 +116,7 @@ void test_get_chunk_size() struct test_executor_measure_iteration : hpx::execution::parallel_executor { - test_executor_measure_iteration() - : hpx::execution::parallel_executor() - { - } + test_executor_measure_iteration() = default; template static auto measure_iteration(Parameters&&, F&&, std::size_t) @@ -136,12 +126,11 @@ struct test_executor_measure_iteration : hpx::execution::parallel_executor } }; -namespace hpx::parallel::execution { - template <> - struct is_two_way_executor : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_two_way_executor< + test_executor_measure_iteration> : std::true_type +{ +}; struct test_measure_iteration { @@ -153,14 +142,11 @@ struct test_measure_iteration } }; -namespace hpx::parallel::execution { - /// \cond NOINTERNAL - template <> - struct is_executor_parameters : std::true_type - { - }; - /// \endcond -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_executor_parameters + : std::true_type +{ +}; /////////////////////////////////////////////////////////////////////////////// void test_get_measure_iteration() @@ -173,8 +159,8 @@ void test_get_measure_iteration() test_measure_iteration{}, hpx::execution::par.executor(), [](std::size_t) { return 0; }, 1); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -185,8 +171,8 @@ void test_get_measure_iteration() test_measure_iteration{}, test_executor_measure_iteration{}, [](std::size_t) { return 0; }, 1); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } @@ -196,10 +182,7 @@ void test_get_measure_iteration() struct test_executor_maximal_number_of_chunks : hpx::execution::parallel_executor { - test_executor_maximal_number_of_chunks() - : hpx::execution::parallel_executor() - { - } + test_executor_maximal_number_of_chunks() = default; template static std::size_t maximal_number_of_chunks( @@ -210,18 +193,16 @@ struct test_executor_maximal_number_of_chunks } }; -namespace hpx::parallel::execution { - template <> - struct is_two_way_executor - : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_two_way_executor< + test_executor_maximal_number_of_chunks> : std::true_type +{ +}; struct test_number_of_chunks { template - std::size_t maximal_number_of_chunks( + static std::size_t maximal_number_of_chunks( Executor&&, std::size_t, std::size_t num_tasks) { ++params_count; @@ -229,12 +210,11 @@ struct test_number_of_chunks } }; -namespace hpx::parallel::execution { - template <> - struct is_executor_parameters : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_executor_parameters + : std::true_type +{ +}; /////////////////////////////////////////////////////////////////////////////// void test_maximal_number_of_chunks() @@ -246,8 +226,8 @@ void test_maximal_number_of_chunks() hpx::parallel::execution::maximal_number_of_chunks( test_number_of_chunks{}, hpx::execution::par.executor(), 1, 1); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -258,8 +238,8 @@ void test_maximal_number_of_chunks() test_number_of_chunks{}, test_executor_maximal_number_of_chunks{}, 1, 1); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } @@ -269,10 +249,7 @@ void test_maximal_number_of_chunks() struct test_executor_reset_thread_distribution : hpx::execution::parallel_executor { - test_executor_reset_thread_distribution() - : hpx::execution::parallel_executor() - { - } + test_executor_reset_thread_distribution() = default; template static void reset_thread_distribution(Parameters&&) @@ -281,29 +258,26 @@ struct test_executor_reset_thread_distribution } }; -namespace hpx::parallel::execution { - template <> - struct is_two_way_executor - : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_two_way_executor< + test_executor_reset_thread_distribution> : std::true_type +{ +}; struct test_thread_distribution { template - void reset_thread_distribution(Executor&&) + static void reset_thread_distribution(Executor&&) { ++params_count; } }; -namespace hpx::parallel::execution { - template <> - struct is_executor_parameters : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_executor_parameters< + test_thread_distribution> : std::true_type +{ +}; /////////////////////////////////////////////////////////////////////////////// void test_reset_thread_distribution() @@ -315,8 +289,8 @@ void test_reset_thread_distribution() hpx::parallel::execution::reset_thread_distribution( test_thread_distribution{}, hpx::execution::par.executor()); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -327,8 +301,8 @@ void test_reset_thread_distribution() test_thread_distribution{}, test_executor_reset_thread_distribution{}); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } @@ -337,10 +311,7 @@ void test_reset_thread_distribution() struct test_executor_processing_units_count : hpx::execution::parallel_executor { - test_executor_processing_units_count() - : hpx::execution::parallel_executor() - { - } + test_executor_processing_units_count() = default; template static std::size_t processing_units_count( @@ -351,31 +322,30 @@ struct test_executor_processing_units_count : hpx::execution::parallel_executor } }; -namespace hpx::parallel::execution { - template <> - struct is_two_way_executor - : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_two_way_executor< + test_executor_processing_units_count> : std::true_type +{ +}; struct test_processing_units { template - static std::size_t processing_units_count( - Executor&&, hpx::chrono::steady_duration const&, std::size_t) + friend std::size_t tag_invoke( + hpx::parallel::execution::processing_units_count_t, + test_processing_units, Executor&&, hpx::chrono::steady_duration const&, + std::size_t) { ++params_count; return 1; } }; -namespace hpx::parallel::execution { - template <> - struct is_executor_parameters : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_executor_parameters + : std::true_type +{ +}; /////////////////////////////////////////////////////////////////////////////// void test_processing_units_count() @@ -386,7 +356,7 @@ void test_processing_units_count() hpx::parallel::execution::processing_units_count( test_processing_units{}, hpx::execution::parallel_executor()); - HPX_TEST_EQ(params_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(0)); } { @@ -397,8 +367,8 @@ void test_processing_units_count() test_processing_units{}, test_executor_processing_units_count{}, hpx::chrono::null_duration, 0); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -407,38 +377,35 @@ void test_processing_units_count() auto p = hpx::parallel::execution::with_processing_units_count( hpx::execution::par, 2); - std::size_t num_cores = + std::size_t const num_cores = hpx::parallel::execution::processing_units_count( test_processing_units{}, p.executor()); - HPX_TEST_EQ(num_cores, std::size_t(2)); - HPX_TEST_EQ(params_count, std::size_t(0)); + HPX_TEST_EQ(num_cores, static_cast(2)); + HPX_TEST_EQ(params_count, static_cast(0)); } { - params_count = 0; - hpx::execution::experimental::num_cores nc(2); auto p = hpx::parallel::execution::with_processing_units_count( hpx::execution::par, nc); - std::size_t num_cores = + std::size_t const num_cores = hpx::parallel::execution::processing_units_count( - test_processing_units{}, p.executor(), + hpx::parallel::execution::null_parameters, p.executor(), hpx::chrono::null_duration, 0); - HPX_TEST_EQ(num_cores, std::size_t(2)); - HPX_TEST_EQ(params_count, std::size_t(0)); + HPX_TEST_EQ(num_cores, static_cast(2)); } { auto p = hpx::parallel::execution::with_processing_units_count( hpx::execution::par, 2); - std::size_t num_cores = + std::size_t const num_cores = hpx::parallel::execution::processing_units_count(p); - HPX_TEST_EQ(num_cores, std::size_t(2)); + HPX_TEST_EQ(num_cores, static_cast(2)); } } @@ -447,64 +414,59 @@ void test_processing_units_count() struct test_executor_begin_end : hpx::execution::parallel_executor { - test_executor_begin_end() - : hpx::execution::parallel_executor() - { - } + test_executor_begin_end() = default; template - void mark_begin_execution(Parameters&&) + static void mark_begin_execution(Parameters&&) { ++exec_count; } template - void mark_end_of_scheduling(Parameters&&) + static void mark_end_of_scheduling(Parameters&&) { ++exec_count; } template - void mark_end_execution(Parameters&&) + static void mark_end_execution(Parameters&&) { ++exec_count; } }; -namespace hpx::parallel::execution { - template <> - struct is_two_way_executor : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_two_way_executor + : std::true_type +{ +}; struct test_begin_end { template - void mark_begin_execution(Executor&&) + static void mark_begin_execution(Executor&&) { ++params_count; } template - void mark_end_of_scheduling(Executor&&) + static void mark_end_of_scheduling(Executor&&) { ++params_count; } template - void mark_end_execution(Executor&&) + static void mark_end_execution(Executor&&) { ++params_count; } }; -namespace hpx::parallel::execution { - template <> - struct is_executor_parameters : std::true_type - { - }; -} // namespace hpx::parallel::execution +template <> +struct hpx::parallel::execution::is_executor_parameters + : std::true_type +{ +}; /////////////////////////////////////////////////////////////////////////////// void test_mark_begin_execution() @@ -516,8 +478,8 @@ void test_mark_begin_execution() hpx::parallel::execution::mark_begin_execution( test_begin_end{}, hpx::execution::par.executor()); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -527,8 +489,8 @@ void test_mark_begin_execution() hpx::parallel::execution::mark_begin_execution( test_begin_end{}, test_executor_begin_end{}); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } @@ -541,8 +503,8 @@ void test_mark_end_of_scheduling() hpx::parallel::execution::mark_end_of_scheduling( test_begin_end{}, hpx::execution::par.executor()); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -552,8 +514,8 @@ void test_mark_end_of_scheduling() hpx::parallel::execution::mark_end_of_scheduling( test_begin_end{}, test_executor_begin_end{}); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } @@ -566,8 +528,8 @@ void test_mark_end_execution() hpx::parallel::execution::mark_end_execution( test_begin_end{}, hpx::execution::par.executor()); - HPX_TEST_EQ(params_count, std::size_t(1)); - HPX_TEST_EQ(exec_count, std::size_t(0)); + HPX_TEST_EQ(params_count, static_cast(1)); + HPX_TEST_EQ(exec_count, static_cast(0)); } { @@ -577,8 +539,8 @@ void test_mark_end_execution() hpx::parallel::execution::mark_end_execution( test_begin_end{}, test_executor_begin_end{}); - HPX_TEST_EQ(params_count, std::size_t(0)); - HPX_TEST_EQ(exec_count, std::size_t(1)); + HPX_TEST_EQ(params_count, static_cast(0)); + HPX_TEST_EQ(exec_count, static_cast(1)); } } diff --git a/libs/core/executors/examples/disable_thread_stealing_executor.cpp b/libs/core/executors/examples/disable_thread_stealing_executor.cpp index 4a3e952a40df..721b1064ec64 100644 --- a/libs/core/executors/examples/disable_thread_stealing_executor.cpp +++ b/libs/core/executors/examples/disable_thread_stealing_executor.cpp @@ -66,6 +66,38 @@ namespace executor_example { } }; + // support all properties exposed by the wrapped executor + // clang-format off + template + )> + auto tag_invoke(Tag tag, + disable_thread_stealing_executor const& exec, + Property&& prop) + -> decltype(disable_thread_stealing_executor( + std::declval()( + std::declval(), std::declval()))) + // clang-format on + { + return disable_thread_stealing_executor( + tag(static_cast(exec), + HPX_FORWARD(Property, prop))); + } + + // clang-format off + template + )> + // clang-format on + auto tag_invoke( + Tag tag, disable_thread_stealing_executor const& exec) + -> decltype(std::declval()(std::declval())) + { + return tag(static_cast(exec)); + } + template auto make_disable_thread_stealing_executor(BaseExecutor&& exec) { diff --git a/libs/core/executors/examples/executor_with_thread_hooks.cpp b/libs/core/executors/examples/executor_with_thread_hooks.cpp index 19a5db048b82..5dc5bd7b113e 100644 --- a/libs/core/executors/examples/executor_with_thread_hooks.cpp +++ b/libs/core/executors/examples/executor_with_thread_hooks.cpp @@ -168,6 +168,12 @@ namespace executor_example { std::forward(predecessor), std::forward(ts)...); } + [[nodiscard]] constexpr std::decay_t const& get_executor() + const noexcept + { + return exec_; + } + private: using thread_hook = hpx::function; diff --git a/libs/core/executors/include/hpx/executors/annotating_executor.hpp b/libs/core/executors/include/hpx/executors/annotating_executor.hpp index 2f34179b140b..6f9a750fa6d2 100644 --- a/libs/core/executors/include/hpx/executors/annotating_executor.hpp +++ b/libs/core/executors/include/hpx/executors/annotating_executor.hpp @@ -203,7 +203,7 @@ namespace hpx::execution::experimental { } std::decay_t exec_; - char const* const annotation_ = nullptr; + char const* annotation_ = nullptr; /// \endcond }; @@ -212,7 +212,7 @@ namespace hpx::execution::experimental { template - )> + )> // clang-format on auto tag_invoke( Tag tag, annotating_executor const& exec, Property&& prop) @@ -227,7 +227,7 @@ namespace hpx::execution::experimental { template - )> + )> // clang-format on auto tag_invoke(Tag tag, annotating_executor const& exec) -> decltype(std::declval()(std::declval())) @@ -259,7 +259,7 @@ namespace hpx::execution::experimental { template - )> + )> // clang-format on constexpr auto tag_fallback_invoke( with_annotation_t, Executor&& exec, char const* annotation) @@ -272,7 +272,7 @@ namespace hpx::execution::experimental { template - )> + )> // clang-format on auto tag_fallback_invoke( with_annotation_t, Executor&& exec, std::string annotation) diff --git a/libs/core/executors/include/hpx/executors/datapar/execution_policy.hpp b/libs/core/executors/include/hpx/executors/datapar/execution_policy.hpp index dd46113d68ed..6ade940bab80 100644 --- a/libs/core/executors/include/hpx/executors/datapar/execution_policy.hpp +++ b/libs/core/executors/include/hpx/executors/datapar/execution_policy.hpp @@ -58,6 +58,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + simd_task_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr simd_task_policy_shim( + simd_task_policy_shim const& rhs) + : base_type( + simd_task_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + simd_task_policy_shim& operator=( + simd_task_policy_shim const& rhs) + { + base_type::operator=( + simd_task_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -70,7 +96,8 @@ namespace hpx::execution { /// /// The algorithm returns a future representing the result of the /// corresponding algorithm when invoked with the sequenced_policy. - using simd_task_policy = detail::simd_task_policy_shim; + using simd_task_policy = detail::simd_task_policy_shim>; namespace detail { @@ -96,6 +123,30 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + simd_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr simd_policy_shim( + simd_policy_shim const& rhs) + : base_type(simd_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + simd_policy_shim& operator=( + simd_policy_shim const& rhs) + { + base_type::operator=( + simd_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -104,7 +155,8 @@ namespace hpx::execution { /// The class simd_policy is an execution policy type used as a unique type /// to disambiguate parallel algorithm overloading and require that a /// parallel algorithm's execution may not be parallelized. - using simd_policy = detail::simd_policy_shim; + using simd_policy = detail::simd_policy_shim>; /// Default sequential execution policy object. inline constexpr simd_policy simd{}; @@ -135,6 +187,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + par_simd_task_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr par_simd_task_policy_shim( + par_simd_task_policy_shim const& rhs) + : base_type( + par_simd_task_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + par_simd_task_policy_shim& operator=( + par_simd_task_policy_shim const& rhs) + { + base_type::operator=(par_simd_task_policy_shim( + rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -147,7 +225,8 @@ namespace hpx::execution { /// The algorithm returns a future representing the result of the /// corresponding algorithm when invoked with the parallel_policy. using par_simd_task_policy = - detail::par_simd_task_policy_shim; + detail::par_simd_task_policy_shim>; namespace detail { @@ -174,6 +253,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + par_simd_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr par_simd_policy_shim( + par_simd_policy_shim const& rhs) + : base_type( + par_simd_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + par_simd_policy_shim& operator=( + par_simd_policy_shim const& rhs) + { + base_type::operator=( + par_simd_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -185,7 +290,8 @@ namespace hpx::execution { /// /// The algorithm returns a future representing the result of the /// corresponding algorithm when invoked with the parallel_policy. - using par_simd_policy = detail::par_simd_policy_shim; + using par_simd_policy = detail::par_simd_policy_shim>; /// Default data-parallel execution policy object. inline constexpr par_simd_policy par_simd{}; diff --git a/libs/core/executors/include/hpx/executors/datapar/execution_policy_fwd.hpp b/libs/core/executors/include/hpx/executors/datapar/execution_policy_fwd.hpp index e34c4c8a9289..e54f6731e76e 100644 --- a/libs/core/executors/include/hpx/executors/datapar/execution_policy_fwd.hpp +++ b/libs/core/executors/include/hpx/executors/datapar/execution_policy_fwd.hpp @@ -15,16 +15,16 @@ namespace hpx::execution::detail { /////////////////////////////////////////////////////////////////////////// - template + template struct simd_policy_shim; - template + template struct simd_task_policy_shim; - template + template struct par_simd_policy_shim; - template + template struct par_simd_task_policy_shim; } // namespace hpx::execution::detail diff --git a/libs/core/executors/include/hpx/executors/execution_policy.hpp b/libs/core/executors/include/hpx/executors/execution_policy.hpp index 1336d99d6241..e6ce801008fb 100644 --- a/libs/core/executors/include/hpx/executors/execution_policy.hpp +++ b/libs/core/executors/include/hpx/executors/execution_policy.hpp @@ -124,10 +124,7 @@ namespace hpx::execution { // The type of the associated executor parameters object which is // associated with this execution policy - using executor_parameters_type = - std::conditional_t, - hpx::traits::executor_parameters_type_t, - decayed_parameters_type>; + using executor_parameters_type = decayed_parameters_type; // The category of the execution agents created by this execution // policy. @@ -284,6 +281,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + sequenced_task_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr sequenced_task_policy_shim( + sequenced_task_policy_shim const& rhs) + : base_type(sequenced_task_policy_shim( + rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + sequenced_task_policy_shim& operator=( + sequenced_task_policy_shim const& rhs) + { + base_type::operator=(sequenced_task_policy_shim( + rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -297,7 +320,8 @@ namespace hpx::execution { /// The algorithm returns a future representing the result of the /// corresponding algorithm when invoked with the sequenced_policy. using sequenced_task_policy = - detail::sequenced_task_policy_shim; + detail::sequenced_task_policy_shim>; namespace detail { @@ -324,6 +348,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + sequenced_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr sequenced_policy_shim( + sequenced_policy_shim const& rhs) + : base_type( + sequenced_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + sequenced_policy_shim& operator=( + sequenced_policy_shim const& rhs) + { + base_type::operator=( + sequenced_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -332,7 +382,8 @@ namespace hpx::execution { /// The class sequenced_policy is an execution policy type used as a unique /// type to disambiguate parallel algorithm overloading and require that a /// parallel algorithm's execution may not be parallelized. - using sequenced_policy = detail::sequenced_policy_shim; + using sequenced_policy = detail::sequenced_policy_shim>; /// Default sequential execution policy object. inline constexpr sequenced_policy seq{}; @@ -364,6 +415,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + parallel_task_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr parallel_task_policy_shim( + parallel_task_policy_shim const& rhs) + : base_type( + parallel_task_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + parallel_task_policy_shim& operator=( + parallel_task_policy_shim const& rhs) + { + base_type::operator=(parallel_task_policy_shim( + rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -376,7 +453,8 @@ namespace hpx::execution { /// The algorithm returns a future representing the result of the /// corresponding algorithm when invoked with the parallel_policy. using parallel_task_policy = - detail::parallel_task_policy_shim; + detail::parallel_task_policy_shim>; namespace detail { @@ -402,6 +480,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + parallel_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr parallel_policy_shim( + parallel_policy_shim const& rhs) + : base_type( + parallel_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + parallel_policy_shim& operator=( + parallel_policy_shim const& rhs) + { + base_type::operator=( + parallel_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -410,7 +514,8 @@ namespace hpx::execution { /// The class parallel_policy is an execution policy type used as a unique /// type to disambiguate parallel algorithm overloading and indicate that a /// parallel algorithm's execution may be parallelized. - using parallel_policy = detail::parallel_policy_shim; + using parallel_policy = detail::parallel_policy_shim>; /// Default parallel execution policy object. inline constexpr parallel_policy par{}; @@ -443,6 +548,34 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + parallel_unsequenced_task_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr parallel_unsequenced_task_policy_shim( + parallel_unsequenced_task_policy_shim const& rhs) + : base_type(parallel_unsequenced_task_policy_shim( + rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + parallel_unsequenced_task_policy_shim& operator=( + parallel_unsequenced_task_policy_shim const& rhs) + { + base_type::operator=(parallel_unsequenced_task_policy_shim( + rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -453,7 +586,8 @@ namespace hpx::execution { /// and indicate that a parallel algorithm's execution may be parallelized /// and vectorized. using parallel_unsequenced_task_policy = - detail::parallel_unsequenced_task_policy_shim; + detail::parallel_unsequenced_task_policy_shim>; namespace detail { @@ -480,6 +614,34 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + parallel_unsequenced_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr parallel_unsequenced_policy_shim( + parallel_unsequenced_policy_shim const& + rhs) + : base_type(parallel_unsequenced_policy_shim( + rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + parallel_unsequenced_policy_shim& operator=( + parallel_unsequenced_policy_shim const& + rhs) + { + base_type::operator=( + parallel_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -490,7 +652,8 @@ namespace hpx::execution { /// indicate that a parallel algorithm's execution may be parallelized and /// vectorized. using parallel_unsequenced_policy = - detail::parallel_unsequenced_policy_shim; + detail::parallel_unsequenced_policy_shim>; /// Default vector execution policy object. inline constexpr parallel_unsequenced_policy par_unseq{}; @@ -524,6 +687,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + unsequenced_task_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr unsequenced_task_policy_shim( + unsequenced_task_policy_shim const& rhs) + : base_type(unsequenced_task_policy_shim( + rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + unsequenced_task_policy_shim& operator=( + unsequenced_task_policy_shim const& rhs) + { + base_type::operator=(unsequenced_task_policy_shim( + rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -533,7 +722,8 @@ namespace hpx::execution { /// unique type to disambiguate parallel algorithm overloading and indicate /// that a parallel algorithm's execution may be vectorized. using unsequenced_task_policy = - detail::unsequenced_task_policy_shim; + detail::unsequenced_task_policy_shim>; namespace detail { @@ -559,6 +749,32 @@ namespace hpx::execution { HPX_FORWARD(Parameters_, params)) { } + + template , + unsequenced_policy_shim> && + std::is_convertible_v && + std::is_convertible_v>> + explicit constexpr unsequenced_policy_shim( + unsequenced_policy_shim const& rhs) + : base_type( + unsequenced_policy_shim(rhs.executor(), rhs.parameters())) + { + } + + template && + std::is_convertible_v>> + unsequenced_policy_shim& operator=( + unsequenced_policy_shim const& rhs) + { + base_type::operator=( + unsequenced_policy_shim(rhs.executor(), rhs.parameters())); + return *this; + } /// \endcond }; } // namespace detail @@ -568,7 +784,8 @@ namespace hpx::execution { /// unique type to disambiguate parallel algorithm overloading and indicate /// that a parallel algorithm's execution may be vectorized. using unsequenced_policy = - detail::unsequenced_policy_shim; + detail::unsequenced_policy_shim>; /// Default vector execution policy object. inline constexpr unsequenced_policy unseq{}; diff --git a/libs/core/executors/include/hpx/executors/execution_policy_fwd.hpp b/libs/core/executors/include/hpx/executors/execution_policy_fwd.hpp index 557b0f6f5350..a07f8707c3ee 100644 --- a/libs/core/executors/include/hpx/executors/execution_policy_fwd.hpp +++ b/libs/core/executors/include/hpx/executors/execution_policy_fwd.hpp @@ -11,27 +11,27 @@ namespace hpx::execution::detail { // forward declarations, see execution_policy.hpp - template + template struct sequenced_policy_shim; - template + template struct sequenced_task_policy_shim; - template + template struct parallel_policy_shim; - template + template struct parallel_task_policy_shim; - template + template struct unsequenced_task_policy_shim; - template + template struct unsequenced_policy_shim; - template + template struct parallel_unsequenced_task_policy_shim; - template + template struct parallel_unsequenced_policy_shim; } // namespace hpx::execution::detail diff --git a/libs/core/executors/include/hpx/executors/execution_policy_mappings.hpp b/libs/core/executors/include/hpx/executors/execution_policy_mappings.hpp index 83149ed60f32..2994f699d701 100644 --- a/libs/core/executors/include/hpx/executors/execution_policy_mappings.hpp +++ b/libs/core/executors/include/hpx/executors/execution_policy_mappings.hpp @@ -135,7 +135,7 @@ namespace hpx::execution::experimental { }; /////////////////////////////////////////////////////////////////////////// - // Return the matching non-unsequences execution policy + // Return the matching non-unsequenced execution policy inline constexpr struct to_non_unseq_t final : hpx::functional::detail::tag_fallback { diff --git a/libs/core/executors/include/hpx/executors/explicit_scheduler_executor.hpp b/libs/core/executors/include/hpx/executors/explicit_scheduler_executor.hpp index 233bd98849a3..e6c64a0da9c2 100644 --- a/libs/core/executors/include/hpx/executors/explicit_scheduler_executor.hpp +++ b/libs/core/executors/include/hpx/executors/explicit_scheduler_executor.hpp @@ -98,17 +98,23 @@ namespace hpx::execution::experimental { return sched_; } + // clang-format off + template + )> + // clang-format on friend auto tag_invoke( hpx::parallel::execution::processing_units_count_t tag, - explicit_scheduler_executor const& exec, + Parameters&& params, explicit_scheduler_executor const& exec, hpx::chrono::steady_duration const& = hpx::chrono::null_duration, std::size_t = 0) -> decltype(std::declval< hpx::parallel::execution::processing_units_count_t>()( - std::declval(), + std::declval(), std::declval(), std::declval(), 0)) { - return tag(exec.sched_); + return tag(HPX_FORWARD(Parameters, params), exec.sched_); } // Associate the parallel_execution_tag executor tag type as a default @@ -250,16 +256,16 @@ namespace hpx::execution::experimental { HPX_CONCEPT_REQUIRES_( hpx::execution::experimental::is_scheduling_property_v )> - // clang-format on auto tag_invoke(Tag tag, explicit_scheduler_executor const& exec, Property&& prop) - -> decltype( - explicit_scheduler_executor(std::declval()( + -> decltype(explicit_scheduler_executor( + std::declval()( std::declval(), std::declval()))) { return explicit_scheduler_executor( tag(exec.sched(), HPX_FORWARD(Property, prop))); } + // clang-format on // clang-format off template ::call()) + constexpr explicit parallel_policy_executor(Policy l) : pool_(nullptr) , policy_(l) { } + constexpr parallel_policy_executor() + : pool_(nullptr) + , policy_( + parallel::execution::detail::get_default_policy::call()) + { + } + constexpr explicit parallel_policy_executor( threads::thread_pool_base* pool, Policy l, std::size_t hierarchical_threshold = @@ -175,18 +180,29 @@ namespace hpx::execution { // property implementations #if defined(HPX_HAVE_THREAD_DESCRIPTION) - friend constexpr parallel_policy_executor tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::execution::experimental::with_annotation_t, - parallel_policy_executor const& exec, char const* annotation) + Executor_ const& exec, char const* annotation) { auto exec_with_annotation = exec; exec_with_annotation.annotation_ = annotation; return exec_with_annotation; } - friend parallel_policy_executor tag_invoke( - hpx::execution::experimental::with_annotation_t, - parallel_policy_executor const& exec, std::string annotation) + // clang-format off + template + )> + // clang-format on + friend auto tag_invoke(hpx::execution::experimental::with_annotation_t, + Executor_ const& exec, std::string annotation) { auto exec_with_annotation = exec; exec_with_annotation.annotation_ = @@ -202,18 +218,29 @@ namespace hpx::execution { } #endif - friend constexpr parallel_policy_executor tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::parallel::execution::with_processing_units_count_t, - parallel_policy_executor const& exec, - std::size_t num_cores) noexcept + Executor_ const& exec, std::size_t num_cores) noexcept { auto exec_with_num_cores = exec; exec_with_num_cores.num_cores_ = num_cores; return exec_with_num_cores; } + // clang-format off + template + )> + // clang-format on friend constexpr std::size_t tag_invoke( - hpx::parallel::execution::processing_units_count_t, + hpx::parallel::execution::processing_units_count_t, Parameters&&, parallel_policy_executor const& exec, hpx::chrono::steady_duration const& = hpx::chrono::null_duration, std::size_t = 0) @@ -221,10 +248,15 @@ namespace hpx::execution { return exec.get_num_cores(); } - friend constexpr parallel_policy_executor tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::execution::experimental::with_first_core_t, - parallel_policy_executor const& exec, - std::size_t first_core) noexcept + Executor_ const& exec, std::size_t first_core) noexcept { auto exec_with_first_core = exec; exec_with_first_core.first_core_ = first_core; diff --git a/libs/core/executors/include/hpx/executors/restricted_thread_pool_executor.hpp b/libs/core/executors/include/hpx/executors/restricted_thread_pool_executor.hpp index 6404f60243df..e4929a69f55c 100644 --- a/libs/core/executors/include/hpx/executors/restricted_thread_pool_executor.hpp +++ b/libs/core/executors/include/hpx/executors/restricted_thread_pool_executor.hpp @@ -148,18 +148,23 @@ namespace hpx::parallel::execution { friend decltype(auto) tag_invoke( Tag tag, restricted_policy_executor const& exec) { - return hpx::functional::tag_invoke( - tag, exec.generate_executor(exec.get_current_thread_num())); + return tag(exec.generate_executor(exec.get_current_thread_num())); } + // clang-format off + template + )> + // clang-format on friend constexpr std::size_t tag_invoke( hpx::parallel::execution::processing_units_count_t tag, - restricted_policy_executor const& exec, + Parameters&& params, restricted_policy_executor const& exec, hpx::chrono::steady_duration const& duration = hpx::chrono::null_duration, std::size_t num_tasks = 0) { - return hpx::functional::tag_invoke(tag, + return tag(HPX_FORWARD(Parameters, params), exec.generate_executor(exec.get_current_thread_num()), duration, num_tasks); } @@ -242,7 +247,7 @@ namespace hpx::parallel::execution { /// \endcond private: - std::uint16_t const first_thread_; + std::uint16_t first_thread_; mutable std::atomic os_thread_; embedded_executor exec_; diff --git a/libs/core/executors/include/hpx/executors/scheduler_executor.hpp b/libs/core/executors/include/hpx/executors/scheduler_executor.hpp index 439d1d569447..a152d6202c1f 100644 --- a/libs/core/executors/include/hpx/executors/scheduler_executor.hpp +++ b/libs/core/executors/include/hpx/executors/scheduler_executor.hpp @@ -130,17 +130,25 @@ namespace hpx::execution::experimental { using future_type = hpx::future; private: + // clang-format off + template + )> + // clang-format on friend auto tag_invoke( hpx::parallel::execution::processing_units_count_t tag, - scheduler_executor const& exec, - hpx::chrono::steady_duration const& = hpx::chrono::null_duration, - std::size_t = 0) + Parameters&& params, scheduler_executor const& exec, + hpx::chrono::steady_duration const& duration = + hpx::chrono::null_duration, + std::size_t num_cores = 0) -> decltype(std::declval< hpx::parallel::execution::processing_units_count_t>()( - std::declval(), + std::declval(), std::declval(), std::declval(), 0)) { - return tag(exec.sched_); + return tag(HPX_FORWARD(Parameters, params), exec.sched_, duration, + num_cores); } // NonBlockingOneWayExecutor interface diff --git a/libs/core/executors/include/hpx/executors/sequenced_executor.hpp b/libs/core/executors/include/hpx/executors/sequenced_executor.hpp index 24b774308386..53162b7966ba 100644 --- a/libs/core/executors/include/hpx/executors/sequenced_executor.hpp +++ b/libs/core/executors/include/hpx/executors/sequenced_executor.hpp @@ -185,18 +185,29 @@ namespace hpx::execution { } #if defined(HPX_HAVE_THREAD_DESCRIPTION) - friend constexpr sequenced_executor tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::execution::experimental::with_annotation_t, - sequenced_executor const& exec, char const* annotation) + Executor_ const& exec, char const* annotation) { auto exec_with_annotation = exec; exec_with_annotation.annotation_ = annotation; return exec_with_annotation; } - friend sequenced_executor tag_invoke( - hpx::execution::experimental::with_annotation_t, - sequenced_executor const& exec, std::string annotation) + // clang-format off + template + )> + // clang-format on + friend auto tag_invoke(hpx::execution::experimental::with_annotation_t, + Executor_ const& exec, std::string annotation) { auto exec_with_annotation = exec; exec_with_annotation.annotation_ = @@ -212,8 +223,14 @@ namespace hpx::execution { } #endif + // clang-format off + template + )> + // clang-format on friend constexpr std::size_t tag_invoke( - hpx::parallel::execution::processing_units_count_t, + hpx::parallel::execution::processing_units_count_t, Parameters&&, sequenced_executor const&, hpx::chrono::steady_duration const& = hpx::chrono::null_duration, std::size_t = 0) diff --git a/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp b/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp index 888e9abe4d4c..e9d2c7d25ddf 100644 --- a/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp +++ b/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp @@ -99,18 +99,29 @@ namespace hpx::execution::experimental { return pool_; } - friend constexpr thread_pool_policy_scheduler tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::parallel::execution::with_processing_units_count_t, - thread_pool_policy_scheduler const& scheduler, - std::size_t num_cores) noexcept + Executor_ const& scheduler, std::size_t num_cores) noexcept { auto scheduler_with_num_cores = scheduler; scheduler_with_num_cores.num_cores_ = num_cores; return scheduler_with_num_cores; } + // clang-format off + template + )> + // clang-format on friend constexpr std::size_t tag_invoke( - hpx::parallel::execution::processing_units_count_t, + hpx::parallel::execution::processing_units_count_t, Parameters&&, thread_pool_policy_scheduler const& scheduler, hpx::chrono::steady_duration const& = hpx::chrono::null_duration, std::size_t = 0) @@ -118,10 +129,15 @@ namespace hpx::execution::experimental { return scheduler.get_num_cores(); } - friend constexpr thread_pool_policy_scheduler tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::execution::experimental::with_first_core_t, - thread_pool_policy_scheduler const& exec, - std::size_t first_core) noexcept + Executor_ const& exec, std::size_t first_core) noexcept { auto exec_with_first_core = exec; exec_with_first_core.first_core_ = first_core; @@ -137,20 +153,29 @@ namespace hpx::execution::experimental { #if defined(HPX_HAVE_THREAD_DESCRIPTION) // support with_annotation property - friend constexpr thread_pool_policy_scheduler tag_invoke( + // clang-format off + template + )> + // clang-format on + friend constexpr auto tag_invoke( hpx::execution::experimental::with_annotation_t, - thread_pool_policy_scheduler const& scheduler, - char const* annotation) + Executor_ const& scheduler, char const* annotation) { auto sched_with_annotation = scheduler; sched_with_annotation.annotation_ = annotation; return sched_with_annotation; } - friend thread_pool_policy_scheduler tag_invoke( - hpx::execution::experimental::with_annotation_t, - thread_pool_policy_scheduler const& scheduler, - std::string annotation) + // clang-format off + template + )> + // clang-format on + friend auto tag_invoke(hpx::execution::experimental::with_annotation_t, + Executor_ const& scheduler, std::string annotation) { auto sched_with_annotation = scheduler; sched_with_annotation.annotation_ = diff --git a/libs/core/executors/include/hpx/executors/thread_pool_scheduler_bulk.hpp b/libs/core/executors/include/hpx/executors/thread_pool_scheduler_bulk.hpp index 6f5eb98943d5..fb826e25e93f 100644 --- a/libs/core/executors/include/hpx/executors/thread_pool_scheduler_bulk.hpp +++ b/libs/core/executors/include/hpx/executors/thread_pool_scheduler_bulk.hpp @@ -643,7 +643,9 @@ namespace hpx::execution::experimental::detail { , f(HPX_FORWARD(F_, f)) , pu_mask(detail::full_mask( hpx::execution::experimental::get_first_core(scheduler), - hpx::parallel::execution::processing_units_count(scheduler))) + hpx::parallel::execution::processing_units_count( + hpx::parallel::execution::null_parameters, scheduler, + hpx::chrono::null_duration, 0))) { } @@ -740,7 +742,9 @@ namespace hpx::execution::experimental::detail { , first_thread( hpx::execution::experimental::get_first_core(scheduler)) , num_worker_threads( - hpx::parallel::execution::processing_units_count(scheduler)) + hpx::parallel::execution::processing_units_count( + hpx::parallel::execution::null_parameters, scheduler, + hpx::chrono::null_duration, 0)) , pu_mask(HPX_MOVE(pumask)) , queues(num_worker_threads) , shape(HPX_FORWARD(Shape_, shape)) diff --git a/libs/core/executors/tests/regressions/pu_count_6184.cpp b/libs/core/executors/tests/regressions/pu_count_6184.cpp index 825e4a65b959..59a2dfb4fa85 100644 --- a/libs/core/executors/tests/regressions/pu_count_6184.cpp +++ b/libs/core/executors/tests/regressions/pu_count_6184.cpp @@ -16,7 +16,7 @@ int hpx_main() { hpx::parallel::execution::restricted_thread_pool_executor executor{0, 3}; HPX_TEST_EQ(hpx::parallel::execution::processing_units_count(executor), - std::size_t(3)); + static_cast(3)); return hpx::local::finalize(); } diff --git a/libs/core/resiliency/include/hpx/resiliency/replay_executor.hpp b/libs/core/resiliency/include/hpx/resiliency/replay_executor.hpp index 23cecc84c582..8610afa54b58 100644 --- a/libs/core/resiliency/include/hpx/resiliency/replay_executor.hpp +++ b/libs/core/resiliency/include/hpx/resiliency/replay_executor.hpp @@ -43,9 +43,9 @@ namespace hpx::resiliency::experimental { using future_type = hpx::traits::executor_future_t; - template - explicit replay_executor(BaseExecutor& exec, std::size_t n, F&& f) - : exec_(exec) + template + explicit replay_executor(BaseExecutor_&& exec, std::size_t n, F&& f) + : exec_(HPX_FORWARD(BaseExecutor_, exec)) , replay_count_(n) , validator_(HPX_FORWARD(F, f)) { @@ -154,12 +154,60 @@ namespace hpx::resiliency::experimental { } /// \endcond + public: + BaseExecutor const& get_executor() const + { + return exec_; + } + std::size_t get_replay_count() const + { + return replay_count_; + } + Validate const& get_validator() const + { + return validator_; + } + private: - BaseExecutor& exec_; + BaseExecutor exec_; std::size_t replay_count_; Validate validator_; }; + /////////////////////////////////////////////////////////////////////////// + // support all properties exposed by the wrapped executor + // clang-format off + template + )> + // clang-format on + auto tag_invoke(Tag tag, + replay_executor const& exec, Property&& prop) + -> decltype(replay_executor( + std::declval()( + std::declval(), std::declval()), + std::declval(), std::declval())) + { + return replay_executor( + tag(exec.get_executor(), HPX_FORWARD(Property, prop)), + exec.get_replay_count(), exec.get_validator()); + } + + // clang-format off + template + )> + // clang-format on + auto tag_invoke( + Tag tag, replay_executor const& exec) + -> decltype(std::declval()(std::declval())) + { + return tag(exec.get_executor()); + } + /////////////////////////////////////////////////////////////////////////// template replay_executor> make_replay_executor( diff --git a/libs/core/resiliency/include/hpx/resiliency/replicate_executor.hpp b/libs/core/resiliency/include/hpx/resiliency/replicate_executor.hpp index 81f499fd4b28..2fee3fe6eb2e 100644 --- a/libs/core/resiliency/include/hpx/resiliency/replicate_executor.hpp +++ b/libs/core/resiliency/include/hpx/resiliency/replicate_executor.hpp @@ -43,10 +43,10 @@ namespace hpx::resiliency::experimental { using future_type = hpx::traits::executor_future_t; - template + template explicit replicate_executor( - BaseExecutor& exec, std::size_t n, V&& v, F&& f) - : exec_(exec) + BaseExecutor_&& exec, std::size_t n, V&& v, F&& f) + : exec_(HPX_FORWARD(BaseExecutor_, exec)) , replicate_count_(n) , voter_(HPX_FORWARD(V, v)) , validator_(HPX_FORWARD(F, f)) @@ -157,13 +157,68 @@ namespace hpx::resiliency::experimental { } /// \endcond + public: + BaseExecutor const& get_executor() const + { + return exec_; + } + std::size_t get_replicate_count() const + { + return replicate_count_; + } + Vote const& get_voter() const + { + return voter_; + } + Validate const& get_validator() const + { + return validator_; + } + private: - BaseExecutor& exec_; + BaseExecutor exec_; std::size_t replicate_count_; Vote voter_; Validate validator_; }; + /////////////////////////////////////////////////////////////////////////// + // support all properties exposed by the wrapped executor + // clang-format off + template + )> + // clang-format on + auto tag_invoke(Tag tag, + replicate_executor const& exec, + Property&& prop) + -> decltype(replicate_executor( + std::declval()( + std::declval(), std::declval()), + std::declval(), std::declval(), + std::declval())) + { + return replicate_executor( + tag(exec.get_executor(), HPX_FORWARD(Property, prop)), + exec.get_replicate_count(), exec.get_voter(), exec.get_validator()); + } + + // clang-format off + template + )> + // clang-format on + auto tag_invoke( + Tag tag, replicate_executor const& exec) + -> decltype(std::declval()(std::declval())) + { + return tag(exec.get_executor()); + } + /////////////////////////////////////////////////////////////////////////// template replicate_executor, diff --git a/libs/core/tag_invoke/include/hpx/functional/detail/tag_fallback_invoke.hpp b/libs/core/tag_invoke/include/hpx/functional/detail/tag_fallback_invoke.hpp index 3a8c959b934b..801569cef949 100644 --- a/libs/core/tag_invoke/include/hpx/functional/detail/tag_fallback_invoke.hpp +++ b/libs/core/tag_invoke/include/hpx/functional/detail/tag_fallback_invoke.hpp @@ -274,7 +274,7 @@ namespace hpx::functional::detail { Args&&...>>>> HPX_HOST_DEVICE HPX_FORCEINLINE constexpr auto operator()( Args&&... args) const - noexcept(is_nothrow_tag_invocable_v) + noexcept(is_nothrow_tag_invocable_v) -> tag_invoke_result_t { return tag_invoke( @@ -289,7 +289,7 @@ namespace hpx::functional::detail { enable_tag_fallback_invoke_t, Args&&...>>>> HPX_HOST_DEVICE HPX_FORCEINLINE constexpr auto operator()( Args&&... args) const - noexcept(is_nothrow_tag_fallback_invocable_v) + noexcept(is_nothrow_tag_fallback_invocable_v) -> tag_fallback_invoke_result_t { return tag_fallback_invoke( diff --git a/libs/core/tag_invoke/include/hpx/functional/detail/tag_priority_invoke.hpp b/libs/core/tag_invoke/include/hpx/functional/detail/tag_priority_invoke.hpp index d759841102d9..593a93034604 100644 --- a/libs/core/tag_invoke/include/hpx/functional/detail/tag_priority_invoke.hpp +++ b/libs/core/tag_invoke/include/hpx/functional/detail/tag_priority_invoke.hpp @@ -258,7 +258,7 @@ namespace hpx::functional::detail { enable_tag_override_invoke_t, Args&&...>>>> HPX_HOST_DEVICE HPX_FORCEINLINE constexpr auto operator()( Args&&... args) const - noexcept(is_nothrow_tag_override_invocable_v) + noexcept(is_nothrow_tag_override_invocable_v) -> tag_override_invoke_result_t { return tag_override_invoke( @@ -274,7 +274,7 @@ namespace hpx::functional::detail { meta::invoke>>> HPX_HOST_DEVICE HPX_FORCEINLINE constexpr auto operator()( Args&&... args) const - noexcept(is_nothrow_tag_invocable_v) + noexcept(is_nothrow_tag_invocable_v) -> tag_invoke_result_t { return tag_invoke( @@ -292,7 +292,7 @@ namespace hpx::functional::detail { enable_tag_fallback_invoke_t, Args&&...>>>> HPX_HOST_DEVICE HPX_FORCEINLINE constexpr auto operator()( Args&&... args) const - noexcept(is_nothrow_tag_fallback_invocable_v) + noexcept(is_nothrow_tag_fallback_invocable_v) -> tag_fallback_invoke_result_t { return tag_fallback_invoke( diff --git a/libs/core/tag_invoke/include/hpx/functional/tag_invoke.hpp b/libs/core/tag_invoke/include/hpx/functional/tag_invoke.hpp index 08b9cbafef1e..00e8ba48f7a6 100644 --- a/libs/core/tag_invoke/include/hpx/functional/tag_invoke.hpp +++ b/libs/core/tag_invoke/include/hpx/functional/tag_invoke.hpp @@ -237,7 +237,7 @@ namespace hpx::functional { meta::invoke>>> HPX_HOST_DEVICE HPX_FORCEINLINE constexpr auto operator()( Args&&... args) const - noexcept(is_nothrow_tag_invocable_v) + noexcept(is_nothrow_tag_invocable_v) -> tag_invoke_result_t { return tag_invoke( diff --git a/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp b/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp index 7e52a55f9632..9a8fcf5f933d 100644 --- a/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp +++ b/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp @@ -6,25 +6,26 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// This is the fourth in a series of examples demonstrating the development of -// a fully distributed solver for a simple 1D heat distribution problem. +// This is the fourth in a series of examples demonstrating the development of a +// fully distributed solver for a simple 1D heat distribution problem. // // This example builds on example three. It futurizes the code from that // example. Compared to example two this code runs much more efficiently. It // allows for changing the amount of work executed in one HPX thread which -// enables tuning the performance for the optimal grain size of the -// computation. This example is still fully local but demonstrates nice -// scalability on SMP machines. +// enables tuning the performance for the optimal grain size of the computation. +// This example is still fully local but demonstrates nice scalability on SMP +// machines. // // In this variation of stencil we use the save_checkpoint and -// revive_checkpint functions to back up the state of the application -// every n time steps. +// restore_checkpoint functions to back up the state of the application every n +// time steps. // #include #include #include +#include #include #include #include @@ -63,16 +64,12 @@ inline std::size_t idx(std::size_t i, int dir, std::size_t size) struct partition_data { private: - typedef hpx::serialization::serialize_buffer buffer_type; + using buffer_type = hpx::serialization::serialize_buffer; public: - partition_data() - : data_() - , size_(0) - { - } + partition_data() = default; - partition_data(std::size_t size) + explicit partition_data(std::size_t size) : data_(std::allocator().allocate(size), size, buffer_type::take) , size_(size) { @@ -82,12 +79,12 @@ struct partition_data : data_(std::allocator().allocate(size), size, buffer_type::take) , size_(size) { - double base_value = double(initial_value * size); + double const base_value = static_cast(initial_value * size); for (std::size_t i = 0; i != size; ++i) - data_[i] = base_value + double(i); + data_[i] = base_value + static_cast(i); } - partition_data(const partition_data& old_part) + partition_data(partition_data const& old_part) : data_(std::allocator().allocate(old_part.size()), old_part.size(), buffer_type::take) , size_(old_part.size()) @@ -114,12 +111,12 @@ struct partition_data private: buffer_type data_; - std::size_t size_; + std::size_t size_ = 0; // Serialization Definitions friend class hpx::serialization::access; template - void serialize(Volume& vol, const unsigned int) + void serialize(Volume& vol, unsigned int const) { // clang-format off vol & data_ & size_; @@ -153,12 +150,12 @@ struct backup , file_name_(file_name) { } - backup(backup&& old) + backup(backup&& old) noexcept : bin(std::move(old.bin)) , file_name_(std::move(old.file_name_)) { } - ~backup() {} + ~backup() = default; void save(partition_data const& status, std::size_t index) { @@ -167,9 +164,10 @@ struct backup void write() { - hpx::util::checkpoint archive_data = + hpx::util::checkpoint const archive_data = hpx::util::save_checkpoint(hpx::launch::sync, bin); - // Make sure file stream is bianary for Windows/Mac machines + + // Make sure file stream is binary for Windows/Mac machines std::ofstream file_archive( file_name_, std::ios::binary | std::ios::out); if (file_archive.is_open()) @@ -180,20 +178,19 @@ struct backup { std::cout << "Error opening file!" << std::endl; } - file_archive.close(); } void revive(std::vector>>& U, std::size_t nx) { hpx::util::checkpoint temp_archive; - // Make sure file stream is bianary for Windows/Mac machines + // Make sure file stream is binary for Windows/Mac machines std::ifstream ist(file_name_, std::ios::binary | std::ios::in); ist >> temp_archive; hpx::util::restore_checkpoint(temp_archive, bin); for (std::size_t i = 0; i < U[0].size(); i++) { - partition_data temp(nx, double(i)); + partition_data temp(nx, static_cast(i)); hpx::util::restore_checkpoint(bin[i], temp); //Check for (std::size_t e = 0; e < temp.size(); e++) @@ -206,7 +203,8 @@ struct backup } }; -void print(std::vector>> U) +void print( + std::vector>> const& U) { for (std::size_t out = 0; out < U[0].size(); out++) { @@ -220,7 +218,7 @@ void print(std::vector>> U) } std::cout << std::endl; } -void print_space(std::vector> next) +void print_space(std::vector> const& next) { for (std::size_t out = 0; out < next.size(); out++) { @@ -239,8 +237,8 @@ void print_space(std::vector> next) struct stepper { // Our data for one time step - typedef hpx::shared_future partition; - typedef std::vector space; + using partition = hpx::shared_future; + using space = std::vector; // Our operator static double heat(double left, double middle, double right) @@ -253,7 +251,7 @@ struct stepper static partition_data heat_part(partition_data const& left, partition_data const& middle, partition_data const& right) { - std::size_t size = middle.size(); + std::size_t const size = middle.size(); partition_data next(size); next[0] = heat(left[size - 1], middle[0], middle[1]); @@ -270,8 +268,9 @@ struct stepper // do all the work on 'np' partitions, 'nx' data points each, for 'nt' // time steps, limit depth of dependency tree to 'nd' - hpx::future do_work(std::size_t np, std::size_t nx, std::size_t nt, - std::uint64_t nd, std::uint64_t cp, std::string rsf, std::string fn) + static hpx::future do_work(std::size_t np, std::size_t nx, + std::size_t nt, std::uint64_t nd, std::uint64_t cp, std::string rsf, + std::string fn) { using hpx::dataflow; using hpx::unwrapping; @@ -302,7 +301,8 @@ struct stepper auto range = hpx::util::counting_shape(np); using hpx::execution::par; hpx::ranges::for_each(par, range, [&U, nx](std::size_t i) { - U[0][i] = hpx::make_ready_future(partition_data(nx, double(i))); + U[0][i] = hpx::make_ready_future( + partition_data(nx, static_cast(i))); }); //Initialize from backup @@ -404,16 +404,18 @@ struct stepper /////////////////////////////////////////////////////////////////////////////// int hpx_main(hpx::program_options::variables_map& vm) { - std::uint64_t np = vm["np"].as(); // Number of partitions. - std::uint64_t nx = + std::uint64_t const np = + vm["np"].as(); // Number of partitions. + std::uint64_t const nx = vm["nx"].as(); // Number of grid points. - std::uint64_t nt = vm["nt"].as(); // Number of steps. - std::uint64_t nd = + std::uint64_t const nt = + vm["nt"].as(); // Number of steps. + std::uint64_t const nd = vm["nd"].as(); // Max depth of dep tree. - std::uint64_t cp = + std::uint64_t const cp = vm["cp"].as(); // Num. steps to checkpoint - std::string rsf = vm["restart-file"].as(); - std::string fn = vm["output-file"].as(); + std::string const rsf = vm["restart-file"].as(); + std::string const fn = vm["output-file"].as(); if (vm.count("no-header")) header = false; @@ -422,7 +424,7 @@ int hpx_main(hpx::program_options::variables_map& vm) stepper step; // Measure execution time. - std::uint64_t t = hpx::chrono::high_resolution_clock::now(); + std::uint64_t const t = hpx::chrono::high_resolution_clock::now(); // Execute nt time steps on nx grid points and print the final solution. hpx::future result = @@ -431,7 +433,7 @@ int hpx_main(hpx::program_options::variables_map& vm) stepper::space solution = result.get(); hpx::wait_all(solution); - std::uint64_t elapsed = hpx::chrono::high_resolution_clock::now() - t; + std::uint64_t const elapsed = hpx::chrono::high_resolution_clock::now() - t; // Print the final solution if (vm.count("results")) From 25f2bed753906f88ed16f34c05df5c74e3a61f7d Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 20 Aug 2023 12:03:02 -0500 Subject: [PATCH 177/333] Fixing test scheduler_priority_check --- .../tests/unit/scheduler_priority_check.cpp | 101 +++++++++++------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/libs/core/resource_partitioner/tests/unit/scheduler_priority_check.cpp b/libs/core/resource_partitioner/tests/unit/scheduler_priority_check.cpp index 7be77c7ebeda..f38853cf81fb 100644 --- a/libs/core/resource_partitioner/tests/unit/scheduler_priority_check.cpp +++ b/libs/core/resource_partitioner/tests/unit/scheduler_priority_check.cpp @@ -32,7 +32,7 @@ void dummy_task(std::size_t n) { // no other work can take place on this thread whilst it sleeps bool sleep = true; - auto start = std::chrono::steady_clock::now(); + auto const start = std::chrono::steady_clock::now(); do { std::this_thread::sleep_for(std::chrono::microseconds(n) / 25); @@ -45,7 +45,7 @@ void dummy_task(std::size_t n) inline std::size_t st_rand() { - return std::size_t(std::rand()); + return static_cast(std::rand()); } int hpx_main(variables_map& vm) @@ -86,25 +86,31 @@ int hpx_main(variables_map& vm) // randomly create normal priority tasks // and then a set of HP tasks in periodic bursts // Use task plotting tools to validate that scheduling is correct - const int np_loop = vm["nnp"].as(); - const int hp_loop = vm["nhp"].as(); - const int np_m = vm["mnp"].as(); - const int hp_m = vm["mhp"].as(); - const int cycles = vm["cycles"].as(); - - const int np_total = np_loop * cycles; - // + int const np_loop = vm["nnp"].as(); + int const hp_loop = vm["nhp"].as(); + int const np_m = vm["mnp"].as(); + int const hp_m = vm["mhp"].as(); + int const cycles = vm["cycles"].as(); + + int const np_total = np_loop * cycles; + struct dec_counter { explicit dec_counter(std::atomic& counter) : counter_(counter) { } + + dec_counter(dec_counter const&) = delete; + dec_counter(dec_counter&&) = delete; + dec_counter& operator=(dec_counter const&) = delete; + dec_counter& operator=(dec_counter&&) = delete; + ~dec_counter() { --counter_; } - // + std::atomic& counter_; }; @@ -113,52 +119,63 @@ int hpx_main(variables_map& vm) std::atomic hp_task_count(0); std::atomic hp_launch_count(0); std::atomic launch_count(0); - // + std::atomic count_down((np_loop + hp_loop) * cycles); std::atomic counter(0); auto f3 = hpx::async(NP_executor, hpx::annotated_function( - [&]() { + [&]() -> hpx::future { ++launch_count; + + std::vector> v; + v.reserve(np_total); for (int i = 0; i < np_total; ++i) { // normal priority - auto f3 = hpx::async(NP_executor, + auto f = hpx::async(NP_executor, hpx::annotated_function( [&, np_m]() { - np_task_count++; + ++np_task_count; dec_counter dec(count_down); - dummy_task(std::size_t(np_m)); + dummy_task(static_cast(np_m)); }, "NP task")); // continuation runs as a sync task - f3.then(hpx::launch::sync, [&](hpx::future&&) { - // on every Nth task, spawn new HP tasks, otherwise quit - if ((++counter) % np_loop != 0) - return; - - // Launch HP tasks using an HP task to do it - hpx::async(HP_executor, - hpx::annotated_function( - [&]() { - ++hp_launch_count; - for (int j = 0; j < hp_loop; ++j) - { - hpx::async(HP_executor, - hpx::annotated_function( - [&]() { - ++hp_task_count; - dec_counter dec(count_down); - dummy_task( - std::size_t(hp_m)); - }, - "HP task")); - } - }, - "Launch HP")); - }); + v.push_back(f.then(hpx::launch::sync, + [&](hpx::future&&) -> hpx::future { + // on every Nth task, spawn new HP tasks, otherwise quit + if ((++counter) % np_loop != 0) + return hpx::make_ready_future(); + + // Launch HP tasks using an HP task to do it + return hpx::async(HP_executor, + hpx::annotated_function( + [&]() -> hpx::future { + ++hp_launch_count; + + std::vector> v1; + v1.reserve(hp_loop); + for (int j = 0; j < hp_loop; ++j) + { + v1.push_back(hpx::async(HP_executor, + hpx::annotated_function( + [&]() { + ++hp_task_count; + dec_counter dec( + count_down); + dummy_task(static_cast< + std::size_t>(hp_m)); + }, + "HP task"))); + } + return hpx::when_all(std::move(v1)); + }, + "Launch HP")); + })); } + + return hpx::when_all(std::move(v)); }, "Launch")); @@ -168,6 +185,8 @@ int hpx_main(variables_map& vm) hpx::this_thread::yield(); } while (count_down > 0); + f3.get(); + std::cout << "Tasks NP : " << np_task_count << "\n" << "Tasks HP : " << hp_task_count << "\n" << "Launch : " << launch_count << "\n" From a8cc2af4d8a0d2c4fa009ce62f8ee96116bceaa1 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 20 Aug 2023 09:52:20 -0500 Subject: [PATCH 178/333] Cleanup previous patch adapting to CMake V3.27 --- cmake/templates/HPXConfig.cmake.in | 2 +- .../CMakeLists.txt | 2 +- .../cmake/HPX_SetupJSON.cmake | 20 +++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 9d0f1d14fba2..81d11112f5e7 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -32,7 +32,7 @@ endif() # find_package. if(HPX_COMMAND_LINE_HANDLING_WITH_JSON_CONFIGURATION_FILES) if(HPX_WITH_FETCH_JSON) - include("${CMAKE_CURRENT_LIST_DIR}/HPXJSONTarget.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/HPXJsonTarget.cmake") else() set(HPX_JSON_ROOT "@Json_ROOT@") include(HPX_SetupJSON) diff --git a/libs/core/command_line_handling_local/CMakeLists.txt b/libs/core/command_line_handling_local/CMakeLists.txt index b46c96205e01..51d744771110 100644 --- a/libs/core/command_line_handling_local/CMakeLists.txt +++ b/libs/core/command_line_handling_local/CMakeLists.txt @@ -39,7 +39,7 @@ set(command_line_handling_local_sources if(HPX_COMMAND_LINE_HANDLING_WITH_JSON_CONFIGURATION_FILES) include(HPX_SetupJSON) - set(command_line_handling_local_dependencies JSON::json) + set(command_line_handling_local_dependencies Json::json) set(command_line_handling_local_headers ${command_line_handling_local_headers} diff --git a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake index 119cb8eb07e3..faf92256426e 100644 --- a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake +++ b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake @@ -15,7 +15,7 @@ endif() if(NOT HPX_WITH_FETCH_JSON) find_package(nlohmann_json 3.2.0 REQUIRED) -elseif(NOT TARGET JSON::json) +elseif(NOT TARGET Json::json) if(NOT HPX_WITH_JSON_TAG) set(HPX_WITH_JSON_TAG "v3.11.2") endif() @@ -26,7 +26,7 @@ elseif(NOT TARGET JSON::json) ) else() hpx_info( - "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Json_TAG=${HPX_WITH_Json_TAG})" + "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_JSON_TAG=${HPX_WITH_JSON_TAG})" ) endif() @@ -45,11 +45,11 @@ elseif(NOT TARGET JSON::json) json SYSTEM INTERFACE $ $ ) - target_compile_definitions(json INTERFACE Json_HAS_CPP_17) + target_compile_definitions(json INTERFACE JSON_HAS_CPP_17) install( TARGETS json - EXPORT HPXJSONTarget + EXPORT HPXJsonTarget COMPONENT core ) @@ -63,17 +63,17 @@ elseif(NOT TARGET JSON::json) export( TARGETS json - NAMESPACE JSON:: - FILE "${CMAKE_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXJSONTarget.cmake" + NAMESPACE Json:: + FILE "${CMAKE_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXJsonTarget.cmake" ) install( - EXPORT HPXJSONTarget - NAMESPACE JSON:: - FILE HPXJSONTarget.cmake + EXPORT HPXJsonTarget + NAMESPACE Json:: + FILE HPXJsonTarget.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${HPX_PACKAGE_NAME} COMPONENT cmake ) - add_library(JSON::json ALIAS json) + add_library(Json::json ALIAS json) endif() From a36d655294d06d4eac9f55bee035963aae5d5ff5 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 24 Aug 2023 11:45:15 -0500 Subject: [PATCH 179/333] More fixes for CMake V3.27 --- cmake/FindJemalloc.cmake | 12 ++++++------ cmake/HPX_SetupLCI.cmake | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmake/FindJemalloc.cmake b/cmake/FindJemalloc.cmake index e828a39b5170..c6e111ede3a2 100644 --- a/cmake/FindJemalloc.cmake +++ b/cmake/FindJemalloc.cmake @@ -23,8 +23,8 @@ find_path( Jemalloc_INCLUDE_DIR jemalloc/jemalloc.h HINTS ${Jemalloc_ROOT} ENV - Jemalloc_ROOT - ${HPX_Jemalloc_ROOT} + JEMALLOC_ROOT + ${HPX_JEMALLOC_ROOT} ${PC_Jemalloc_MINIMAL_INCLUDEDIR} ${PC_Jemalloc_MINIMAL_INCLUDE_DIRS} ${PC_Jemalloc_INCLUDEDIR} @@ -39,8 +39,8 @@ if(MSVC) Jemalloc_ADDITIONAL_INCLUDE_DIR msvc_compat/strings.h HINTS ${Jemalloc_ROOT} ENV - Jemalloc_ROOT - ${HPX_Jemalloc_ROOT} + JEMALLOC_ROOT + ${HPX_JEMALLOC_ROOT} ${PC_Jemalloc_MINIMAL_INCLUDEDIR} ${PC_Jemalloc_MINIMAL_INCLUDE_DIRS} ${PC_Jemalloc_INCLUDEDIR} @@ -65,8 +65,8 @@ find_library( NAMES jemalloc libjemalloc HINTS ${Jemalloc_ROOT} ENV - Jemalloc_ROOT - ${HPX_Jemalloc_ROOT} + JEMALLOC_ROOT + ${HPX_JEMALLOC_ROOT} ${PC_Jemalloc_MINIMAL_LIBDIR} ${PC_Jemalloc_MINIMAL_LIBRARY_DIRS} ${PC_Jemalloc_LIBDIR} diff --git a/cmake/HPX_SetupLCI.cmake b/cmake/HPX_SetupLCI.cmake index 26b1ecd18076..d34cc6e08ec0 100644 --- a/cmake/HPX_SetupLCI.cmake +++ b/cmake/HPX_SetupLCI.cmake @@ -38,14 +38,14 @@ macro(hpx_setup_lci) ) else() hpx_info( - "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Lci_TAG=${HPX_WITH_Lci_TAG})" + "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_LCI_TAG=${HPX_WITH_LCI_TAG})" ) endif() include(FetchContent) fetchcontent_declare( lci GIT_REPOSITORY https://github.com/uiuc-hpc/LC.git - GIT_TAG ${HPX_WITH_Lci_TAG} + GIT_TAG ${HPX_WITH_LCI_TAG} ) fetchcontent_getproperties(lci) From f0ccebb9ee89dd44a6c6ae80f80c0e2095cce51f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 14 May 2023 15:49:21 -0500 Subject: [PATCH 180/333] Modernize modules in level 17 and 18 --- .../debugging/include/hpx/debugging/print.hpp | 4 +- .../detail/async_launch_policy_dispatch.hpp | 2 +- .../execution/detail/post_policy_dispatch.hpp | 2 +- .../hpx/executors/parallel_executor.hpp | 2 +- .../hpx/executors/thread_pool_scheduler.hpp | 1 + .../include/hpx/futures/futures_factory.hpp | 1 + .../hpx/io_service/io_service_pool.hpp | 5 +- .../hpx/io_service/io_service_thread_pool.hpp | 2 - libs/core/io_service/src/io_service_pool.cpp | 6 +- .../io_service/src/io_service_thread_pool.cpp | 2 - .../hpx/runtime_local/run_as_hpx_thread.hpp | 3 +- .../hpx/schedulers/background_scheduler.hpp | 19 +-- .../local_priority_queue_scheduler.hpp | 8 +- .../hpx/schedulers/local_queue_scheduler.hpp | 95 ++++++------ .../schedulers/lockfree_queue_backends.hpp | 20 +-- .../schedulers/maintain_queue_wait_times.hpp | 4 +- .../include/hpx/schedulers/queue_helpers.hpp | 8 +- .../hpx/schedulers/queue_holder_numa.hpp | 46 +++--- .../hpx/schedulers/queue_holder_thread.hpp | 56 +++---- .../shared_priority_queue_scheduler.hpp | 139 ++++++++++-------- .../static_priority_queue_scheduler.hpp | 4 +- .../hpx/schedulers/static_queue_scheduler.hpp | 10 +- .../include/hpx/schedulers/thread_queue.hpp | 7 +- libs/core/threading_base/CMakeLists.txt | 1 + .../hpx/threading_base/annotated_function.hpp | 43 +++--- .../detail/get_default_pool.hpp | 15 +- .../threading_base/detail/switch_status.hpp | 9 +- .../hpx/threading_base/execution_agent.hpp | 2 +- .../hpx/threading_base/register_thread.hpp | 86 +++-------- .../hpx/threading_base/scheduler_base.hpp | 7 +- .../hpx/threading_base/scoped_annotation.hpp | 3 +- .../hpx/threading_base/thread_data.hpp | 86 +---------- .../hpx/threading_base/thread_helpers.hpp | 3 + .../hpx/threading_base/thread_pool_base.hpp | 3 +- .../hpx/threading_base/threading_base_fwd.hpp | 87 ++++++++++- .../threading_base/src/annotated_function.cpp | 2 +- .../core/threading_base/src/create_thread.cpp | 4 +- libs/core/threading_base/src/create_work.cpp | 2 +- .../threading_base/src/execution_agent.cpp | 17 +-- .../threading_base/src/get_default_pool.cpp | 5 +- libs/core/threading_base/src/print.cpp | 3 +- .../threading_base/src/register_thread.cpp | 103 +++++++++++++ .../threading_base/src/thread_description.cpp | 3 +- .../command_line_handling.hpp | 1 - .../src/command_line_handling.cpp | 103 +++++++------ .../src/late_command_line_handling.cpp | 7 +- .../src/parse_command_line.cpp | 32 ++-- 47 files changed, 558 insertions(+), 515 deletions(-) create mode 100644 libs/core/threading_base/src/register_thread.cpp diff --git a/libs/core/debugging/include/hpx/debugging/print.hpp b/libs/core/debugging/include/hpx/debugging/print.hpp index bf914ec72b31..9a5fd6ae29e6 100644 --- a/libs/core/debugging/include/hpx/debugging/print.hpp +++ b/libs/core/debugging/include/hpx/debugging/print.hpp @@ -370,7 +370,7 @@ namespace hpx::debug { std::string buffered_msg; // - scoped_var(char const* p, Args const&... args) + explicit scoped_var(char const* p, Args const&... args) : prefix_(p) , message_(args...) { @@ -395,7 +395,7 @@ namespace hpx::debug { double const delay_; std::tuple const message_; // - timed_var(double delay, Args const&... args) + explicit timed_var(double delay, Args const&... args) : time_start_(std::chrono::steady_clock::now()) , delay_(delay) , message_(args...) diff --git a/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp index a314139d54a0..dd3af81985f9 100644 --- a/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/async_launch_policy_dispatch.hpp @@ -17,10 +17,10 @@ #include #include #include +#include #include #include #include -#include #include #include diff --git a/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp b/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp index 2d3365ccebdb..0a68afb21dcd 100644 --- a/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp +++ b/libs/core/execution/include/hpx/execution/detail/post_policy_dispatch.hpp @@ -13,10 +13,10 @@ #include #include #include +#include #include #include #include -#include #include #include diff --git a/libs/core/executors/include/hpx/executors/parallel_executor.hpp b/libs/core/executors/include/hpx/executors/parallel_executor.hpp index 7e85b9959175..0543c15a9e03 100644 --- a/libs/core/executors/include/hpx/executors/parallel_executor.hpp +++ b/libs/core/executors/include/hpx/executors/parallel_executor.hpp @@ -30,8 +30,8 @@ #include #include #include +#include #include -#include #include #include diff --git a/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp b/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp index e9d2c7d25ddf..456a1576d7b4 100644 --- a/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp +++ b/libs/core/executors/include/hpx/executors/thread_pool_scheduler.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include diff --git a/libs/core/futures/include/hpx/futures/futures_factory.hpp b/libs/core/futures/include/hpx/futures/futures_factory.hpp index 853969c0979c..685261a8a97b 100644 --- a/libs/core/futures/include/hpx/futures/futures_factory.hpp +++ b/libs/core/futures/include/hpx/futures/futures_factory.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/libs/core/io_service/include/hpx/io_service/io_service_pool.hpp b/libs/core/io_service/include/hpx/io_service/io_service_pool.hpp index 2c499b613eb3..39f163bfd081 100644 --- a/libs/core/io_service/include/hpx/io_service/io_service_pool.hpp +++ b/libs/core/io_service/include/hpx/io_service/io_service_pool.hpp @@ -100,7 +100,7 @@ namespace hpx::util { } /// \brief Activate the thread \a index for this thread pool - void thread_run(std::size_t index, barrier* startup = nullptr); + void thread_run(std::size_t index, barrier* startup = nullptr) const; /// \brief Return name of this pool constexpr char const* get_name() const noexcept @@ -122,7 +122,8 @@ namespace hpx::util { using io_service_ptr = std::unique_ptr; using work_type = std::unique_ptr; - HPX_FORCEINLINE work_type initialize_work(asio::io_context& io_service) + HPX_FORCEINLINE work_type initialize_work( + asio::io_context& io_service) const { return work_type( std::make_unique(io_service)); diff --git a/libs/core/io_service/include/hpx/io_service/io_service_thread_pool.hpp b/libs/core/io_service/include/hpx/io_service/io_service_thread_pool.hpp index ea58e4650cd0..cb64fa1fcbbe 100644 --- a/libs/core/io_service/include/hpx/io_service/io_service_thread_pool.hpp +++ b/libs/core/io_service/include/hpx/io_service/io_service_thread_pool.hpp @@ -8,8 +8,6 @@ #include #include -#include -#include #include #include diff --git a/libs/core/io_service/src/io_service_pool.cpp b/libs/core/io_service/src/io_service_pool.cpp index a5ff061d2e63..041d87109c33 100644 --- a/libs/core/io_service/src/io_service_pool.cpp +++ b/libs/core/io_service/src/io_service_pool.cpp @@ -37,8 +37,6 @@ namespace hpx::util { , pool_name_(pool_name) , pool_name_postfix_(name_postfix) , waiting_(false) - , wait_barrier_() - , continue_barrier_() { LPROGRESS_ << pool_name; init(pool_size); @@ -52,7 +50,6 @@ namespace hpx::util { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "io_service_pool::io_service_pool", "io_service_pool size is 0"); - return; } wait_barrier_.reset(new barrier(pool_size + 1)); @@ -93,7 +90,8 @@ namespace hpx::util { clear_locked(); } - void io_service_pool::thread_run(std::size_t index, util::barrier* startup) + void io_service_pool::thread_run( + std::size_t index, util::barrier* startup) const { // wait for all threads to start up before before starting HPX work if (startup != nullptr) diff --git a/libs/core/io_service/src/io_service_thread_pool.cpp b/libs/core/io_service/src/io_service_thread_pool.cpp index 2ab7303c6d4d..ec05ba5775a4 100644 --- a/libs/core/io_service/src/io_service_thread_pool.cpp +++ b/libs/core/io_service/src/io_service_thread_pool.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include #include #include diff --git a/libs/core/runtime_local/include/hpx/runtime_local/run_as_hpx_thread.hpp b/libs/core/runtime_local/include/hpx/runtime_local/run_as_hpx_thread.hpp index 6443a4c25860..3d917d40158a 100644 --- a/libs/core/runtime_local/include/hpx/runtime_local/run_as_hpx_thread.hpp +++ b/libs/core/runtime_local/include/hpx/runtime_local/run_as_hpx_thread.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2018 Hartmut Kaiser +// Copyright (c) 2016-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/libs/core/schedulers/include/hpx/schedulers/background_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/background_scheduler.hpp index f3c64d8e12ef..62515866a3ea 100644 --- a/libs/core/schedulers/include/hpx/schedulers/background_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/background_scheduler.hpp @@ -8,21 +8,15 @@ #pragma once #include -#include -#include -#include #include #include #include #include -#include #include #include -#include #include #include -#include #include @@ -67,8 +61,10 @@ namespace hpx::threads::policies { { // this scheduler does not support stealing or numa stealing, but // needs to enable background work - mode = scheduler_mode(mode & ~scheduler_mode::enable_stealing); - mode = scheduler_mode(mode & ~scheduler_mode::enable_stealing_numa); + mode = static_cast( + mode & ~scheduler_mode::enable_stealing); + mode = static_cast( + mode & ~scheduler_mode::enable_stealing_numa); mode = mode | ~scheduler_mode::do_background_work; mode = mode | ~scheduler_mode::do_background_work_only; scheduler_base::set_scheduler_mode(mode); @@ -76,7 +72,7 @@ namespace hpx::threads::policies { // Return the next thread to be executed, return false if none is // available - constexpr bool get_next_thread( + static constexpr bool get_next_thread( std::size_t, bool, threads::thread_id_ref_type&, bool) noexcept { // this scheduler does not maintain any thread queues @@ -87,8 +83,9 @@ namespace hpx::threads::policies { // manager to allow for maintenance tasks to be executed in the // scheduler. Returns true if the OS thread calling this function has to // be terminated (i.e. no more work has to be done). - constexpr bool wait_or_add_new(std::size_t, bool running, std::int64_t&, - bool, std::size_t&, thread_id_ref_type* = nullptr) + static constexpr bool wait_or_add_new(std::size_t, bool running, + std::int64_t&, bool, std::size_t&, + thread_id_ref_type* = nullptr) noexcept { return !running; } diff --git a/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp index 19b7010173fc..fa46c396fc81 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_priority_queue_scheduler.hpp @@ -78,7 +78,7 @@ namespace hpx::threads::policies { detail::affinity_data const& affinity_data, std::size_t num_high_priority_queues = static_cast( -1), - thread_queue_init_parameters thread_queue_init = + thread_queue_init_parameters const& thread_queue_init = thread_queue_init_parameters{}, char const* description = "local_priority_queue_scheduler") noexcept @@ -111,7 +111,7 @@ namespace hpx::threads::policies { }; using init_parameter_type = init_parameter; - local_priority_queue_scheduler(init_parameter_type const& init, + explicit local_priority_queue_scheduler(init_parameter_type const& init, bool deferred_initialization = true) : scheduler_base( init.num_queues_, init.description_, init.thread_queue_init_) @@ -801,7 +801,7 @@ namespace hpx::threads::policies { num_thread = select_active_pu(num_thread, allow_fallback); - [[maybe_unused]] auto* thrdptr = get_thread_id_data(thrd); + [[maybe_unused]] auto const* thrdptr = get_thread_id_data(thrd); switch (priority) { case thread_priority::high_recursive: @@ -1007,7 +1007,7 @@ namespace hpx::threads::policies { std::int64_t get_thread_count( thread_schedule_state state = thread_schedule_state::unknown, thread_priority priority = thread_priority::default_, - std::size_t num_thread = std::size_t(-1), + std::size_t num_thread = static_cast(-1), bool /* reset */ = false) const override { // Return thread count of one specific queue. diff --git a/libs/core/schedulers/include/hpx/schedulers/local_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_queue_scheduler.hpp index 56437c8180c8..27c8423a85c7 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_queue_scheduler.hpp @@ -34,8 +34,6 @@ #include -// TODO: add branch prediction and function heat - /////////////////////////////////////////////////////////////////////////////// namespace hpx::threads::policies { @@ -67,7 +65,7 @@ namespace hpx::threads::policies { { init_parameter(std::size_t num_queues, detail::affinity_data const& affinity_data, - thread_queue_init_parameters thread_queue_init = + thread_queue_init_parameters const& thread_queue_init = thread_queue_init_parameters{}, char const* description = "local_queue_scheduler") : num_queues_(num_queues) @@ -81,7 +79,6 @@ namespace hpx::threads::policies { detail::affinity_data const& affinity_data, char const* description) : num_queues_(num_queues) - , thread_queue_init_() , affinity_data_(affinity_data) , description_(description) { @@ -101,11 +98,6 @@ namespace hpx::threads::policies { , queues_(init.num_queues_) , curr_queue_(0) , affinity_data_(init.affinity_data_) - // we know that the MIC has one NUMA domain only) -#if !defined(HPX_NATIVE_MIC) - , steals_in_numa_domain_() - , steals_outside_numa_domain_() -#endif , numa_domain_masks_( init.num_queues_, create_topology().get_machine_affinity_mask()) , outside_numa_domain_masks_( @@ -125,7 +117,12 @@ namespace hpx::threads::policies { } } - ~local_queue_scheduler() + local_queue_scheduler(local_queue_scheduler const&) = delete; + local_queue_scheduler(local_queue_scheduler&&) = delete; + local_queue_scheduler& operator=(local_queue_scheduler const&) = delete; + local_queue_scheduler& operator=(local_queue_scheduler&&) = delete; + + ~local_queue_scheduler() override { for (std::size_t i = 0; i != queues_.size(); ++i) delete queues_[i]; @@ -296,11 +293,11 @@ namespace hpx::threads::policies { std::size_t num_thread = data.schedulehint.mode == thread_schedule_hint_mode::thread ? data.schedulehint.hint : - std::size_t(-1); + static_cast(-1); - std::size_t queue_size = queues_.size(); + std::size_t const queue_size = queues_.size(); - if (std::size_t(-1) == num_thread) + if (static_cast(-1) == num_thread) { num_thread = curr_queue_++ % queue_size; } @@ -330,25 +327,24 @@ namespace hpx::threads::policies { bool get_next_thread(std::size_t num_thread, bool running, threads::thread_id_ref_type& thrd, bool /*enable_stealing*/) { - std::size_t queues_size = queues_.size(); + std::size_t const queues_size = queues_.size(); { HPX_ASSERT(num_thread < queues_size); thread_queue_type* q = queues_[num_thread]; - bool result = q->get_next_thread(thrd); + bool const result = q->get_next_thread(thrd); q->increment_num_pending_accesses(); if (result) return true; q->increment_num_pending_misses(); - bool have_staged = - q->get_staged_queue_length(std::memory_order_relaxed) != 0; - // Give up, we should have work to convert. - if (have_staged) + if (q->get_staged_queue_length(std::memory_order_relaxed) != 0) + { return false; + } } if (!running) @@ -356,13 +352,14 @@ namespace hpx::threads::policies { return false; } - bool numa_stealing = has_scheduler_mode( + bool const numa_stealing = has_scheduler_mode( policies::scheduler_mode::enable_stealing_numa); if (!numa_stealing) { // steal work items: first try to steal from other cores in // the same NUMA node - std::size_t pu_number = affinity_data_.get_pu_num(num_thread); + std::size_t const pu_number = + affinity_data_.get_pu_num(num_thread); // we know that the MIC has one NUMA domain only #if !defined(HPX_NATIVE_MIC) @@ -380,10 +377,13 @@ namespace hpx::threads::policies { HPX_ASSERT(idx != num_thread); - std::size_t pu_num = affinity_data_.get_pu_num(idx); - if (!test(this_numa_domain, + if (std::size_t const pu_num = + affinity_data_.get_pu_num(idx); + !test(this_numa_domain, pu_num)) //-V560 //-V600 //-V111 + { continue; + } thread_queue_type* q = queues_[idx]; if (q->get_next_thread(thrd, running)) @@ -413,7 +413,8 @@ namespace hpx::threads::policies { HPX_ASSERT(idx != num_thread); - std::size_t pu_num = affinity_data_.get_pu_num(idx); + std::size_t const pu_num = + affinity_data_.get_pu_num(idx); if (!test(numa_domain, pu_num)) //-V560 //-V600 //-V111 continue; @@ -459,7 +460,7 @@ namespace hpx::threads::policies { thread_priority /* priority */ = thread_priority::default_) override { // NOTE: This scheduler ignores NUMA hints. - std::size_t num_thread = std::size_t(-1); + auto num_thread = static_cast(-1); if (schedulehint.mode == thread_schedule_hint_mode::thread) //-V1051 { @@ -470,9 +471,9 @@ namespace hpx::threads::policies { allow_fallback = false; } - std::size_t queue_size = queues_.size(); + std::size_t const queue_size = queues_.size(); - if (std::size_t(-1) == num_thread) + if (static_cast(-1) == num_thread) { num_thread = curr_queue_++ % queue_size; } @@ -502,7 +503,7 @@ namespace hpx::threads::policies { thread_priority /* priority */ = thread_priority::default_) override { // NOTE: This scheduler ignores NUMA hints. - std::size_t num_thread = std::size_t(-1); + auto num_thread = static_cast(-1); if (schedulehint.mode == thread_schedule_hint_mode::thread) //-V1051 { @@ -513,9 +514,9 @@ namespace hpx::threads::policies { allow_fallback = false; } - std::size_t queue_size = queues_.size(); + std::size_t const queue_size = queues_.size(); - if (std::size_t(-1) == num_thread) + if (static_cast(-1) == num_thread) { num_thread = curr_queue_++ % queue_size; } @@ -541,12 +542,11 @@ namespace hpx::threads::policies { /////////////////////////////////////////////////////////////////////// // This returns the current length of the queues (work items and new items) - std::int64_t get_queue_length( - std::size_t num_thread = std::size_t(-1)) const override + std::int64_t get_queue_length(std::size_t num_thread) const override { // Return queue length of one specific queue. std::int64_t count = 0; - if (std::size_t(-1) != num_thread) + if (static_cast(-1) != num_thread) { HPX_ASSERT(num_thread < queues_.size()); @@ -564,12 +564,12 @@ namespace hpx::threads::policies { std::int64_t get_thread_count( thread_schedule_state state = thread_schedule_state::unknown, thread_priority priority = thread_priority::default_, - std::size_t num_thread = std::size_t(-1), + std::size_t num_thread = static_cast(-1), bool /* reset */ = false) const override { // Return thread count of one specific queue. std::int64_t count = 0; - if (std::size_t(-1) != num_thread) + if (static_cast(-1) != num_thread) { HPX_ASSERT(num_thread < queues_.size()); @@ -593,10 +593,8 @@ namespace hpx::threads::policies { "local_queue_scheduler::get_thread_count", "unknown thread priority value " "(thread_priority::unknown)"); - return 0; } } - return 0; } // Return the cumulative count for all queues. @@ -624,7 +622,6 @@ namespace hpx::threads::policies { "local_queue_scheduler::get_thread_count", "unknown thread priority value " "(thread_priority::unknown)"); - return 0; } } return count; @@ -711,7 +708,7 @@ namespace hpx::threads::policies { std::int64_t& idle_loop_count, bool /* enable_stealing */, std::size_t& added, thread_id_ref_type* = nullptr) { - std::size_t queues_size = queues_.size(); + std::size_t const queues_size = queues_.size(); HPX_ASSERT(num_thread < queues_.size()); added = 0; @@ -729,14 +726,14 @@ namespace hpx::threads::policies { return true; } - bool numa_stealing_ = has_scheduler_mode( - policies::scheduler_mode::enable_stealing_numa); // limited or no stealing across domains - if (!numa_stealing_) + if (!has_scheduler_mode( + policies::scheduler_mode::enable_stealing_numa)) { // steal work items: first try to steal from other cores in the // same NUMA node - std::size_t pu_number = affinity_data_.get_pu_num(num_thread); + std::size_t const pu_number = + affinity_data_.get_pu_num(num_thread); // we know that the MIC has one NUMA domain only #if !defined(HPX_NATIVE_MIC) @@ -875,7 +872,7 @@ namespace hpx::threads::policies { auto const& topo = create_topology(); // pre-calculate certain constants for the given thread number - std::size_t num_pu = affinity_data_.get_pu_num(num_thread); + std::size_t const num_pu = affinity_data_.get_pu_num(num_thread); mask_cref_type machine_mask = topo.get_machine_affinity_mask(); mask_cref_type core_mask = topo.get_thread_affinity_mask(num_pu); mask_cref_type node_mask = topo.get_numa_node_affinity_mask(num_pu); @@ -892,15 +889,15 @@ namespace hpx::threads::policies { mask_type first_mask = mask_type(); resize(first_mask, mask_size(core_mask)); - std::size_t first = find_first(node_mask); - if (first != std::size_t(-1)) + std::size_t const first = find_first(node_mask); + if (first != static_cast(-1)) set(first_mask, first); else first_mask = core_mask; - bool numa_stealing = has_scheduler_mode( - policies::scheduler_mode::enable_stealing_numa); - if (numa_stealing && any(first_mask & core_mask)) + if (has_scheduler_mode( + policies::scheduler_mode::enable_stealing_numa) && + any(first_mask & core_mask)) { #if !defined(HPX_NATIVE_MIC) // we know that the MIC has one NUMA domain only set(steals_outside_numa_domain_, num_pu); diff --git a/libs/core/schedulers/include/hpx/schedulers/lockfree_queue_backends.hpp b/libs/core/schedulers/include/hpx/schedulers/lockfree_queue_backends.hpp index c9ef72bdc07e..65296f65cdd0 100644 --- a/libs/core/schedulers/include/hpx/schedulers/lockfree_queue_backends.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/lockfree_queue_backends.hpp @@ -51,8 +51,8 @@ namespace hpx::threads::policies { using size_type = std::uint64_t; explicit lockfree_fifo_backend(size_type initial_size = 0, - size_type /* num_thread */ = size_type(-1)) - : queue_(std::size_t(initial_size)) + size_type /* num_thread */ = static_cast(-1)) + : queue_(static_cast(initial_size)) { } @@ -115,8 +115,8 @@ namespace hpx::threads::policies { using size_type = std::uint64_t; explicit moodycamel_fifo_backend(size_type initial_size = 0, - size_type /* num_thread */ = size_type(-1)) - : queue_(std::size_t(initial_size)) + size_type /* num_thread */ = static_cast(-1)) + : queue_(static_cast(initial_size)) { } @@ -171,8 +171,8 @@ namespace hpx::threads::policies { using size_type = std::uint64_t; explicit lockfree_lifo_backend(size_type initial_size = 0, - size_type /* num_thread */ = size_type(-1)) - : queue_(std::size_t(initial_size)) + size_type /* num_thread */ = static_cast(-1)) + : queue_(static_cast(initial_size)) { } @@ -231,8 +231,8 @@ namespace hpx::threads::policies { using size_type = std::uint64_t; explicit lockfree_abp_fifo_backend(size_type initial_size = 0, - size_type /* num_thread */ = size_type(-1)) - : queue_(std::size_t(initial_size)) + size_type /* num_thread */ = static_cast(-1)) + : queue_(static_cast(initial_size)) { } @@ -288,8 +288,8 @@ namespace hpx::threads::policies { using size_type = std::uint64_t; explicit lockfree_abp_lifo_backend(size_type initial_size = 0, - size_type /* num_thread */ = size_type(-1)) - : queue_(std::size_t(initial_size)) + size_type /* num_thread */ = static_cast(-1)) + : queue_(static_cast(initial_size)) { } diff --git a/libs/core/schedulers/include/hpx/schedulers/maintain_queue_wait_times.hpp b/libs/core/schedulers/include/hpx/schedulers/maintain_queue_wait_times.hpp index 0f407a85f5a7..ef7674aae59c 100644 --- a/libs/core/schedulers/include/hpx/schedulers/maintain_queue_wait_times.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/maintain_queue_wait_times.hpp @@ -9,11 +9,11 @@ #include +#ifdef HPX_HAVE_THREAD_QUEUE_WAITTIME namespace hpx::threads::policies { -#ifdef HPX_HAVE_THREAD_QUEUE_WAITTIME HPX_CORE_EXPORT void set_maintain_queue_wait_times_enabled( bool enabled) noexcept; HPX_CORE_EXPORT bool get_maintain_queue_wait_times_enabled() noexcept; -#endif } // namespace hpx::threads::policies +#endif diff --git a/libs/core/schedulers/include/hpx/schedulers/queue_helpers.hpp b/libs/core/schedulers/include/hpx/schedulers/queue_helpers.hpp index 1b18ad17eb29..1e5f13203d13 100644 --- a/libs/core/schedulers/include/hpx/schedulers/queue_helpers.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/queue_helpers.hpp @@ -14,14 +14,10 @@ #include #include #include -#include #include -#include #include #include -#include -#include /////////////////////////////////////////////////////////////////////////////// namespace hpx::threads::policies { @@ -56,9 +52,9 @@ namespace hpx::threads::policies { for (typename Map::const_iterator it = tm.begin(); it != end; ++it) { threads::thread_data const* thrd = get_thread_id_data(*it); - threads::thread_schedule_state state = + threads::thread_schedule_state const state = thrd->get_state().state(); - threads::thread_schedule_state marked_state = + threads::thread_schedule_state const marked_state = thrd->get_marked_state(); if (state != marked_state) diff --git a/libs/core/schedulers/include/hpx/schedulers/queue_holder_numa.hpp b/libs/core/schedulers/include/hpx/schedulers/queue_holder_numa.hpp index 0d26d2502db7..aa46a09d8fcd 100644 --- a/libs/core/schedulers/include/hpx/schedulers/queue_holder_numa.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/queue_holder_numa.hpp @@ -7,32 +7,16 @@ #pragma once #include -#include #include #include #include // -#include -#include -#include -// #include // -#include #include #include -#include -#include -#include -#include - -#include #include -#include -#include -#include -#include -#include +#include #if !defined(QUEUE_HOLDER_NUMA_DEBUG) #if defined(HPX_DEBUG) @@ -232,7 +216,7 @@ namespace hpx::threads::policies { } // ---------------------------------------------------------------- - inline std::int64_t get_thread_count( + inline std::size_t get_thread_count( thread_schedule_state state = thread_schedule_state::unknown, thread_priority priority = thread_priority::default_) const { @@ -265,33 +249,33 @@ namespace hpx::threads::policies { std::vector queues_; public: - constexpr void increment_num_pending_misses( + static constexpr void increment_num_pending_misses( std::size_t /* num */ = 1) noexcept { } - constexpr void increment_num_pending_accesses( + static constexpr void increment_num_pending_accesses( std::size_t /* num */ = 1) noexcept { } - constexpr void increment_num_stolen_from_pending( + static constexpr void increment_num_stolen_from_pending( std::size_t /* num */ = 1) noexcept { } - constexpr void increment_num_stolen_from_staged( + static constexpr void increment_num_stolen_from_staged( std::size_t /* num */ = 1) noexcept { } - constexpr void increment_num_stolen_to_pending( + static constexpr void increment_num_stolen_to_pending( std::size_t /* num */ = 1) noexcept { } - constexpr void increment_num_stolen_to_staged( + static constexpr void increment_num_stolen_to_staged( std::size_t /* num */ = 1) noexcept { } // ------------------------------------------------------------ - bool dump_suspended_threads(std::size_t /* num_thread */, + static bool dump_suspended_threads(std::size_t /* num_thread */, std::int64_t& /* idle_loop_count */, bool /* running */) noexcept { return false; @@ -305,9 +289,15 @@ namespace hpx::threads::policies { } // ------------------------------------------------------------ - constexpr void on_start_thread(std::size_t /* num_thread */) noexcept {} - constexpr void on_stop_thread(std::size_t /* num_thread */) noexcept {} - constexpr void on_error(std::size_t /* num_thread */, + static constexpr void on_start_thread( + std::size_t /* num_thread */) noexcept + { + } + static constexpr void on_stop_thread( + std::size_t /* num_thread */) noexcept + { + } + static constexpr void on_error(std::size_t /* num_thread */, std::exception_ptr const& /* e */) noexcept { } diff --git a/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp b/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp index 4837cbe5623f..b5de546e3351 100644 --- a/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/queue_holder_thread.hpp @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -17,7 +16,6 @@ #include #include #include -#include #include #include @@ -25,12 +23,11 @@ #include #include #include -#include #include -#include #include #include #include +#include #include #include #include @@ -356,14 +353,14 @@ namespace hpx::threads::policies { tq_deb.debug(debug::str<>("cleanup"), "delete", queue_data_print(this), debug::threadinfo(todelete)); - thread_id_type tid(todelete); + thread_id_type const tid(todelete); remove_from_thread_map(tid, true); } } else { // delete only this many threads - std::int64_t delete_count = static_cast( + auto delete_count = static_cast( terminated_items_count_.data_.load( std::memory_order_relaxed) / 2); @@ -505,7 +502,7 @@ namespace hpx::threads::policies { #endif { // Allocate a new thread object. - threads::thread_data* p = nullptr; + threads::thread_data* p; if (stacksize == parameters_.nostack_stacksize_) { p = threads::thread_data_stackless::create( @@ -528,7 +525,7 @@ namespace hpx::threads::policies { void recycle_thread(thread_id_type tid) { #if !defined(HPX_HAVE_ADDRESS_SANITIZER) - std::ptrdiff_t stacksize = + std::ptrdiff_t const stacksize = get_thread_id_data(tid)->get_stack_size(); if (stacksize == parameters_.small_stacksize_) @@ -575,12 +572,12 @@ namespace hpx::threads::policies { scoped_lock lk(thread_map_mtx_.data_); // add a new entry in the map for this thread - std::pair p = + std::pair const p = thread_map_.insert(tid); if (/*HPX_UNLIKELY*/ (!p.second)) { - std::string map_size = std::to_string(thread_map_.size()); + std::string const map_size = std::to_string(thread_map_.size()); tq_deb.error(debug::str<>("map add"), "Couldn't add new thread to the thread map", @@ -609,7 +606,7 @@ namespace hpx::threads::policies { HPX_ASSERT(thread_map_.find(tid) != thread_map_.end()); HPX_ASSERT(thread_map_count_.data_ >= 0); - [[maybe_unused]] bool deleted = thread_map_.erase(tid) != 0; + [[maybe_unused]] bool const deleted = thread_map_.erase(tid) != 0; HPX_ASSERT(deleted); tq_deb.debug(debug::str<>("map remove"), queue_data_print(this), @@ -680,8 +677,8 @@ namespace hpx::threads::policies { } // ---------------------------------------------------------------- - std::size_t add_new_HP( - std::int64_t add_count, thread_holder_type* addfrom, bool stealing) + std::size_t add_new_HP(std::int64_t add_count, + thread_holder_type const* addfrom, bool stealing) { std::size_t added; if (owns_bp_queue() && !stealing) @@ -794,7 +791,6 @@ namespace hpx::threads::policies { "unknown thread priority value (thread_priority::unknown)"); } } - return 0; } // ---------------------------------------------------------------- @@ -845,7 +841,6 @@ namespace hpx::threads::policies { "unknown thread priority value (thread_priority::unknown)"); } } - return 0; } // ---------------------------------------------------------------- @@ -873,9 +868,8 @@ namespace hpx::threads::policies { scoped_lock lk(thread_map_mtx_.data_); std::int64_t num_threads = 0; - thread_map_type::const_iterator end = thread_map_.end(); - for (thread_map_type::const_iterator it = thread_map_.begin(); - it != end; ++it) + auto const end = thread_map_.end(); + for (auto it = thread_map_.begin(); it != end; ++it) { if (get_thread_id_data(*it)->get_state().state() == state) ++num_threads; @@ -896,8 +890,8 @@ namespace hpx::threads::policies { "xthread", xthread, queue_data_print(this), debug::threadinfo(thrd)); terminated_items_.push(thrd); - std::int64_t count = ++terminated_items_count_.data_; + std::int64_t const count = ++terminated_items_count_.data_; if (!xthread && (count > parameters_.max_terminated_threads_)) { // clean up all terminated threads @@ -909,16 +903,17 @@ namespace hpx::threads::policies { void abort_all_suspended_threads() { scoped_lock lk(thread_map_mtx_.data_); - thread_map_type::iterator end = thread_map_.end(); - for (thread_map_type::iterator it = thread_map_.begin(); it != end; - ++it) + auto const end = thread_map_.end(); + for (auto it = thread_map_.begin(); it != end; ++it) { if (get_thread_id_data(*it)->get_state().state() == thread_schedule_state::suspended) { - get_thread_id_data(*it)->set_state( - thread_schedule_state::pending, - thread_restart_state::abort); + [[maybe_unused]] auto const s = + get_thread_id_data(*it)->set_state( + thread_schedule_state::pending, + thread_restart_state::abort); + // np queue always exists so use that as priority doesn't matter np_queue_->schedule_work(*it, true); } @@ -940,7 +935,6 @@ namespace hpx::threads::policies { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "queue_holder_thread::iterate_threads", "can't iterate over thread ids of staged threads"); - return false; } std::vector tids; @@ -949,9 +943,8 @@ namespace hpx::threads::policies { if (state == thread_schedule_state::unknown) { scoped_lock lk(thread_map_mtx_.data_); - thread_map_type::const_iterator end = thread_map_.end(); - for (thread_map_type::const_iterator it = thread_map_.begin(); - it != end; ++it) + auto const end = thread_map_.end(); + for (auto it = thread_map_.begin(); it != end; ++it) { tids.push_back(*it); } @@ -959,9 +952,8 @@ namespace hpx::threads::policies { else { scoped_lock lk(thread_map_mtx_.data_); - thread_map_type::const_iterator end = thread_map_.end(); - for (thread_map_type::const_iterator it = thread_map_.begin(); - it != end; ++it) + auto const end = thread_map_.end(); + for (auto it = thread_map_.begin(); it != end; ++it) { if (get_thread_id_data(*it)->get_state().state() == state) tids.push_back(*it); diff --git a/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp index d37ffb9c915b..69ad15e9343d 100644 --- a/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/shared_priority_queue_scheduler.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -64,12 +63,10 @@ namespace hpx { hpx::debug::enable_print; using print_on = hpx::debug::enable_print; - static print_onoff spq_deb("SPQUEUE"); - static print_on spq_arr("SPQUEUE"); + inline constexpr print_onoff spq_deb("SPQUEUE"); + inline constexpr print_on spq_arr("SPQUEUE"); } // namespace hpx -#define SHARED_PRIORITY_QUEUE_SCHEDULER_API 3 - namespace hpx::threads::policies { /////////////////////////////////////////////////////////////////////////// @@ -141,7 +138,6 @@ namespace hpx::threads::policies { char const* description) : num_worker_threads_(num_worker_threads) , cores_per_queue_(cores_per_queue) - , thread_queue_init_() , affinity_data_(affinity_data) , description_(description) { @@ -173,12 +169,21 @@ namespace hpx::threads::policies { , initialized_(false) , debug_init_(false) , thread_init_counter_(0) - , pool_index_(std::size_t(-1)) + , pool_index_(static_cast(-1)) { scheduler_base::set_scheduler_mode(scheduler_mode::default_); HPX_ASSERT(num_workers_ != 0); } + shared_priority_queue_scheduler( + shared_priority_queue_scheduler const&) = delete; + shared_priority_queue_scheduler( + shared_priority_queue_scheduler&&) = delete; + shared_priority_queue_scheduler& operator=( + shared_priority_queue_scheduler const&) = delete; + shared_priority_queue_scheduler& operator=( + shared_priority_queue_scheduler&&) = delete; + virtual ~shared_priority_queue_scheduler() = default; static std::string_view get_scheduler_name() @@ -221,14 +226,14 @@ namespace hpx::threads::policies { // node. It should not be used without care as the thread numbering // internal to the scheduler is not a simple linear indexing // returns -1 to indicate an invalid thread/value/state - inline std::size_t local_thread_number() + inline std::size_t local_thread_number() const { using namespace hpx::threads::detail; std::size_t const thread_pool_num = get_thread_pool_num_tss(); // if the thread belongs to this pool return local Id if (pool_index_ == thread_pool_num) return get_local_thread_num_tss(); - return std::size_t(-1); + return static_cast(-1); } // ------------------------------------------------------------ @@ -240,7 +245,7 @@ namespace hpx::threads::policies { spq_deb.timed(cleanup); std::size_t local_num = local_thread_number(); - if (local_num == std::size_t(-1)) + if (local_num == static_cast(-1)) { // clang-format off using namespace hpx::threads::detail; @@ -258,7 +263,7 @@ namespace hpx::threads::policies { } std::size_t domain_num = d_lookup_[local_num]; - std::size_t q_index = q_lookup_[local_num]; + std::size_t const q_index = q_lookup_[local_num]; spq_deb.debug(debug::str<>("cleanup_terminated"), "v1", "D", debug::dec<2>(domain_num), "Q", debug::dec<3>(q_index), @@ -278,7 +283,7 @@ namespace hpx::threads::policies { // find the numa domain from the local thread index std::size_t domain_num = d_lookup_[local_num]; - std::size_t q_index = q_lookup_[local_num]; + std::size_t const q_index = q_lookup_[local_num]; spq_deb.debug(debug::str<>("cleanup_terminated"), "v2", "D", debug::dec<2>(domain_num), "Q", debug::dec<3>(q_index), @@ -319,7 +324,7 @@ namespace hpx::threads::policies { { spq_deb.set(msg, "HINT_NONE "); // Create thread on this worker thread if possible - if (local_num == std::size_t(-1)) + if (local_num == static_cast(-1)) { // clang-format off using namespace hpx::threads::detail; @@ -340,14 +345,15 @@ namespace hpx::threads::policies { } else if (!round_robin_) /* thread parent */ { - if (spq_deb.is_enabled()) + if constexpr (spq_deb.is_enabled()) { domain_num = d_lookup_[thread_num]; q_index = q_lookup_[thread_num]; + + spq_deb.debug(debug::str<>("create_thread"), + "assign_work_thread_parent", "thread_num", + thread_num, "pool", parent_pool_->get_pool_name()); } - spq_deb.debug(debug::str<>("create_thread"), - "assign_work_thread_parent", "thread_num", thread_num, - "pool", parent_pool_->get_pool_name()); } else /*(round_robin)*/ { @@ -390,7 +396,7 @@ namespace hpx::threads::policies { domain_num = fast_mod(data.schedulehint.hint, num_domains_); // if the thread creating the new task is on the domain // assigned to the new task - try to reuse the core as well - if (local_num != std::size_t(-1) && + if (local_num != static_cast(-1) && d_lookup_[local_num] == domain_num) { thread_num = local_num; //-V1048 @@ -603,7 +609,7 @@ namespace hpx::threads::policies { bool get_next_thread(std::size_t thread_num, bool running, threads::thread_id_ref_type& thrd, bool enable_stealing) { - std::size_t this_thread = local_thread_number(); + std::size_t const this_thread = local_thread_number(); HPX_ASSERT(this_thread < num_workers_); // just cleanup the thread we were called by rather than all threads @@ -615,19 +621,19 @@ namespace hpx::threads::policies { auto get_next_thread_function_HP = [&](std::size_t domain, std::size_t q_index, thread_holder_type* /* receiver */, - threads::thread_id_ref_type& thrd, bool stealing, + threads::thread_id_ref_type& th, bool stealing, bool allow_stealing) { return numa_holder_[domain].get_next_thread_HP( - q_index, thrd, stealing, allow_stealing); + q_index, th, stealing, allow_stealing); }; auto get_next_thread_function = [&](std::size_t domain, std::size_t q_index, thread_holder_type* /* receiver */, - threads::thread_id_ref_type& thrd, bool stealing, + threads::thread_id_ref_type& th, bool stealing, bool allow_stealing) { return numa_holder_[domain].get_next_thread( - q_index, thrd, stealing, allow_stealing); + q_index, th, stealing, allow_stealing); }; std::size_t domain = d_lookup_[this_thread]; @@ -636,13 +642,15 @@ namespace hpx::threads::policies { // first try a high priority task, allow stealing if stealing of HP // tasks in on, this will be fine but send a null function for // normal tasks - bool result = steal_by_function(domain, - q_index, numa_stealing_, core_stealing_, nullptr, thrd, - "SBF-get_next_thread", get_next_thread_function_HP, - get_next_thread_function); - if (result) + if (bool const result = + steal_by_function(domain, + q_index, numa_stealing_, core_stealing_, nullptr, thrd, + "SBF-get_next_thread", get_next_thread_function_HP, + get_next_thread_function)) + { return result; + } // if we did not get a task at all, then try converting tasks in the // pending queue into staged ones @@ -650,8 +658,10 @@ namespace hpx::threads::policies { std::int64_t idle_loop_count = 0; wait_or_add_new(thread_num, true, idle_loop_count, true, added); if (added > 0) + { return get_next_thread( this_thread, running, thrd, enable_stealing); + } return false; } @@ -660,7 +670,7 @@ namespace hpx::threads::policies { std::int64_t& /* idle_loop_count */, bool /*enable_stealing*/, std::size_t& added, thread_id_ref_type* = nullptr) { - std::size_t this_thread = local_thread_number(); + std::size_t const this_thread = local_thread_number(); HPX_ASSERT(this_thread < num_workers_); // just cleanup the thread we were called by rather than all threads @@ -671,18 +681,18 @@ namespace hpx::threads::policies { auto add_new_function_HP = [&](std::size_t domain, std::size_t q_index, - thread_holder_type* receiver, std::size_t& added, + thread_holder_type* receiver, std::size_t& add, bool stealing, bool allow_stealing) { return numa_holder_[domain].add_new_HP( - receiver, q_index, added, stealing, allow_stealing); + receiver, q_index, add, stealing, allow_stealing); }; auto add_new_function = [&](std::size_t domain, std::size_t q_index, thread_holder_type* receiver, - std::size_t& added, bool stealing, + std::size_t& add, bool stealing, bool allow_stealing) { return numa_holder_[domain].add_new( - receiver, q_index, added, stealing, allow_stealing); + receiver, q_index, add, stealing, allow_stealing); }; std::size_t domain = d_lookup_[this_thread]; @@ -694,8 +704,8 @@ namespace hpx::threads::policies { q_index, "numa_stealing ", numa_stealing_, "core_stealing ", core_stealing_); - bool added_tasks = steal_by_function(domain, q_index, - numa_stealing_, core_stealing_, receiver, added, + bool const added_tasks = steal_by_function(domain, + q_index, numa_stealing_, core_stealing_, receiver, added, "wait_or_add_new", add_new_function_HP, add_new_function); return !added_tasks; @@ -711,7 +721,7 @@ namespace hpx::threads::policies { std::size_t local_num = local_thread_number(); std::size_t thread_num = local_num; std::size_t domain_num = 0; - std::size_t q_index = std::size_t(-1); + auto q_index = static_cast(-1); auto msg = spq_deb.declare_variable(nullptr); @@ -725,7 +735,7 @@ namespace hpx::threads::policies { { // Create thread on this worker thread if possible spq_deb.set(msg, "HINT_NONE "); - if (local_num == std::size_t(-1)) + if (local_num == static_cast(-1)) { // This is a task being injected from a thread on another // pool - we can schedule on any thread available @@ -863,16 +873,15 @@ namespace hpx::threads::policies { // This returns the current length of the queues // (work items and new items) //--------------------------------------------------------------------- - std::int64_t get_queue_length( - std::size_t thread_num = std::size_t(-1)) const override + std::int64_t get_queue_length(std::size_t thread_num) const override { spq_deb.debug(debug::str<>("get_queue_length"), "thread_num ", debug::dec<>(thread_num)); - HPX_ASSERT(thread_num != std::size_t(-1)); + HPX_ASSERT(thread_num != static_cast(-1)); std::int64_t count = 0; - if (thread_num != std::size_t(-1)) + if (thread_num != static_cast(-1)) { std::size_t domain_num = d_lookup_[thread_num]; std::size_t q_index = q_lookup_[thread_num]; @@ -893,13 +902,13 @@ namespace hpx::threads::policies { std::int64_t get_thread_count( thread_schedule_state state = thread_schedule_state::unknown, thread_priority priority = thread_priority::default_, - std::size_t thread_num = std::size_t(-1), + std::size_t thread_num = static_cast(-1), bool /* reset */ = false) const override { spq_deb.debug(debug::str<>("get_thread_count"), "thread_num ", debug::dec<3>(thread_num)); - if (thread_num != std::size_t(-1)) + if (thread_num != static_cast(-1)) { std::size_t domain_num = d_lookup_[thread_num]; std::size_t q_index = q_lookup_[thread_num]; @@ -962,7 +971,8 @@ namespace hpx::threads::policies { initialized_ = true; // used to make sure thread ids are valid for this scheduler - pool_index_ = std::size_t(parent_pool_->get_pool_index()); + pool_index_ = + static_cast(parent_pool_->get_pool_index()); // For each worker thread, count which numa domain each // belongs to and build lists of useful indexes/refs @@ -976,9 +986,10 @@ namespace hpx::threads::policies { for (std::size_t local_id = 0; local_id != num_workers_; ++local_id) { - std::size_t global_id = + std::size_t const global_id = local_to_global_thread_index(local_id); - std::size_t pu_num = affinity_data_.get_pu_num(global_id); + std::size_t const pu_num = + affinity_data_.get_pu_num(global_id); std::size_t domain = topo.get_numa_node_number(pu_num); #if defined(SHARED_PRIORITY_SCHEDULER_DEBUG_NUMA) if (local_id >= (num_workers_ + 1) / 2) @@ -1028,13 +1039,13 @@ namespace hpx::threads::policies { std::vector locations; for (std::size_t local_id = 0; local_id != num_workers_; ++local_id) { - std::size_t global_id = local_to_global_thread_index(local_id); + std::size_t const global_id = + local_to_global_thread_index(local_id); std::size_t pu_num = affinity_data_.get_pu_num(global_id); std::size_t core = topo.get_core_number(pu_num); std::size_t domain = d_lookup_[local_id]; // - locations.push_back( - std::make_tuple(domain, core, pu_num, local_id)); + locations.emplace_back(domain, core, pu_num, local_id); } // sort by 1) domain, 2) core, 3) pu so that we can iterate over @@ -1067,7 +1078,6 @@ namespace hpx::threads::policies { parent_pool_->get_pool_id().index()); // one thread holder per core (shared by PUs) - thread_holder_type* thread_holder = nullptr; // queue pointers we will assign to each thread thread_queue_type* bp_queue = nullptr; @@ -1076,12 +1086,11 @@ namespace hpx::threads::policies { thread_queue_type* lp_queue = nullptr; // for each worker thread, assign queues - std::size_t previous_domain = std::size_t(-1); + auto previous_domain = static_cast(-1); std::size_t index = 0; for (auto& tup : locations) { - std::int16_t owner_mask = 0; std::size_t domain = std::get<0>(tup); std::size_t local_id = std::get<3>(tup); std::size_t numa_id = local_id - q_offset_[domain]; @@ -1102,6 +1111,7 @@ namespace hpx::threads::policies { if (local_thread == local_id) { + std::int16_t owner_mask = 0; q_lookup_[local_thread] = static_cast(index); // bound queues are never shared @@ -1171,13 +1181,14 @@ namespace hpx::threads::policies { local_thread, "domain", domain, "index", index, "local_id", local_id, "owner_mask", owner_mask); - thread_holder = new queue_holder_thread( - bp_queue, hp_queue, np_queue, lp_queue, - static_cast(domain), - static_cast(index), - static_cast(local_id), - static_cast(owner_mask), - queue_parameters_); + thread_holder_type* thread_holder = + new queue_holder_thread(bp_queue, + hp_queue, np_queue, lp_queue, + static_cast(domain), + static_cast(index), + static_cast(local_id), + static_cast(owner_mask), + queue_parameters_); numa_holder_[domain].queues_[numa_id] = thread_holder; } @@ -1191,7 +1202,7 @@ namespace hpx::threads::policies { } // increment the thread counter and allow the next thread to init - thread_init_counter_++; + ++thread_init_counter_; // we do not want to allow threads to start stealing from others // until all threads have initialized their structures. We therefore @@ -1206,10 +1217,10 @@ namespace hpx::threads::policies { if (!debug_init_) { debug_init_ = true; - spq_arr.array("# d_lookup_ ", &d_lookup_[0], num_workers_); - spq_arr.array("# q_lookup_ ", &q_lookup_[0], num_workers_); - spq_arr.array("# q_counts_ ", &q_counts_[0], num_domains_); - spq_arr.array("# q_offset_ ", &q_offset_[0], num_domains_); + spq_arr.array("# d_lookup_ ", d_lookup_.data(), num_workers_); + spq_arr.array("# q_lookup_ ", q_lookup_.data(), num_workers_); + spq_arr.array("# q_counts_ ", q_counts_.data(), num_domains_); + spq_arr.array("# q_offset_ ", q_offset_.data(), num_domains_); #ifdef SHARED_PRIORITY_SCHEDULER_LINUX spq_arr.array("# schedcpu_ ", &schedcpu_[0], num_workers_); #endif diff --git a/libs/core/schedulers/include/hpx/schedulers/static_priority_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/static_priority_queue_scheduler.hpp index 1933fe3dade4..57c03ff3af21 100644 --- a/libs/core/schedulers/include/hpx/schedulers/static_priority_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/static_priority_queue_scheduler.hpp @@ -69,9 +69,9 @@ namespace hpx::threads::policies { void set_scheduler_mode(scheduler_mode mode) noexcept override { // this scheduler does not support stealing or numa stealing - mode = policies::scheduler_mode( + mode = static_cast( mode & ~policies::scheduler_mode::enable_stealing); - mode = policies::scheduler_mode( + mode = static_cast( mode & ~policies::scheduler_mode::enable_stealing_numa); scheduler_base::set_scheduler_mode(mode); } diff --git a/libs/core/schedulers/include/hpx/schedulers/static_queue_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/static_queue_scheduler.hpp index d1b0bcb234b7..e74457cd6a12 100644 --- a/libs/core/schedulers/include/hpx/schedulers/static_queue_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/static_queue_scheduler.hpp @@ -13,16 +13,12 @@ #include #include #include -#include #include -#include #include #include -#include #include #include -#include /////////////////////////////////////////////////////////////////////////////// namespace hpx::threads::policies { @@ -66,8 +62,10 @@ namespace hpx::threads::policies { void set_scheduler_mode(scheduler_mode mode) noexcept override { // this scheduler does not support stealing or numa stealing - mode = scheduler_mode(mode & ~scheduler_mode::enable_stealing); - mode = scheduler_mode(mode & ~scheduler_mode::enable_stealing_numa); + mode = static_cast( + mode & ~scheduler_mode::enable_stealing); + mode = static_cast( + mode & ~scheduler_mode::enable_stealing_numa); scheduler_base::set_scheduler_mode(mode); } diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 7758770b9306..4c8524bad734 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -342,7 +342,7 @@ namespace hpx::threads::policies { return addednew != 0; } - void recycle_thread(thread_id_type thrd) + void recycle_thread(thread_id_type const& thrd) { std::ptrdiff_t const stacksize = get_thread_id_data(thrd)->get_stack_size(); @@ -961,8 +961,9 @@ namespace hpx::threads::policies { if (thrd->get_state().state() == thread_schedule_state::suspended) { - thrd->set_state(thread_schedule_state::pending, - thread_restart_state::abort); + [[maybe_unused]] auto const s = + thrd->set_state(thread_schedule_state::pending, + thread_restart_state::abort); // thread holds self-reference HPX_ASSERT(thrd->count_ > 1); diff --git a/libs/core/threading_base/CMakeLists.txt b/libs/core/threading_base/CMakeLists.txt index 3da1b8e31110..72f7b64b6ffc 100644 --- a/libs/core/threading_base/CMakeLists.txt +++ b/libs/core/threading_base/CMakeLists.txt @@ -73,6 +73,7 @@ set(threading_base_sources get_default_pool.cpp get_default_timer_service.cpp print.cpp + register_thread.cpp scheduler_base.cpp set_thread_state.cpp set_thread_state_timed.cpp diff --git a/libs/core/threading_base/include/hpx/threading_base/annotated_function.hpp b/libs/core/threading_base/include/hpx/threading_base/annotated_function.hpp index e7f1f6942ddd..b212ff01625a 100644 --- a/libs/core/threading_base/include/hpx/threading_base/annotated_function.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/annotated_function.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Hartmut Kaiser +// Copyright (c) 2017-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -25,7 +25,6 @@ #endif #include -#include #include #include #include @@ -77,7 +76,6 @@ namespace hpx { /// \brief Returns the function address /// /// This function returns the passed function address. - /// \param none constexpr std::size_t get_function_address() const noexcept { return traits::get_function_address::call(f_); @@ -89,8 +87,6 @@ namespace hpx { /// This function returns the function annotation, if it has a name /// name is returned, name is returned; if name is empty the typeid /// is returned - /// - /// \param none constexpr char const* get_function_annotation() const noexcept { return name_ ? name_ : typeid(f_).name(); @@ -150,31 +146,28 @@ namespace hpx { #endif } // namespace hpx -namespace hpx::traits { - #if defined(HPX_HAVE_THREAD_DESCRIPTION) - /////////////////////////////////////////////////////////////////////////// - template - struct get_function_address> +/////////////////////////////////////////////////////////////////////////// +template +struct hpx::traits::get_function_address> +{ + static constexpr std::size_t call( + hpx::detail::annotated_function const& f) noexcept { - static constexpr std::size_t call( - hpx::detail::annotated_function const& f) noexcept - { - return f.get_function_address(); - } - }; + return f.get_function_address(); + } +}; - template - struct get_function_annotation> +template +struct hpx::traits::get_function_annotation> +{ + static constexpr char const* call( + hpx::detail::annotated_function const& f) noexcept { - static constexpr char const* call( - hpx::detail::annotated_function const& f) noexcept - { - return f.get_function_annotation(); - } - }; + return f.get_function_annotation(); + } +}; #endif -} // namespace hpx::traits namespace hpx::util { diff --git a/libs/core/threading_base/include/hpx/threading_base/detail/get_default_pool.hpp b/libs/core/threading_base/include/hpx/threading_base/detail/get_default_pool.hpp index aa6be5bea26d..59eddd1eaf77 100644 --- a/libs/core/threading_base/include/hpx/threading_base/detail/get_default_pool.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/detail/get_default_pool.hpp @@ -11,11 +11,14 @@ #include #include -#include -namespace hpx::threads::detail { +namespace hpx::threads { - using get_default_pool_type = hpx::function; - HPX_CORE_EXPORT void set_get_default_pool(get_default_pool_type f); - HPX_CORE_EXPORT thread_pool_base* get_self_or_default_pool(); -} // namespace hpx::threads::detail + class HPX_CORE_EXPORT thread_pool_base; + + namespace detail { + using get_default_pool_type = hpx::function; + HPX_CORE_EXPORT void set_get_default_pool(get_default_pool_type f); + HPX_CORE_EXPORT thread_pool_base* get_self_or_default_pool(); + } // namespace detail +} // namespace hpx::threads diff --git a/libs/core/threading_base/include/hpx/threading_base/detail/switch_status.hpp b/libs/core/threading_base/include/hpx/threading_base/detail/switch_status.hpp index 9e1e4cdec822..6a735c0f8f0c 100644 --- a/libs/core/threading_base/include/hpx/threading_base/detail/switch_status.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/detail/switch_status.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -11,8 +11,6 @@ #include #include -#include - namespace hpx::threads::detail { /////////////////////////////////////////////////////////////////////// @@ -30,6 +28,11 @@ namespace hpx::threads::detail { { } + switch_status(switch_status const&) = delete; + switch_status(switch_status&&) = delete; + switch_status& operator=(switch_status const&) = delete; + switch_status& operator=(switch_status&&) = delete; + ~switch_status() { if (need_restore_state_) diff --git a/libs/core/threading_base/include/hpx/threading_base/execution_agent.hpp b/libs/core/threading_base/include/hpx/threading_base/execution_agent.hpp index b3af3fff6ae5..9fd5653a2399 100644 --- a/libs/core/threading_base/include/hpx/threading_base/execution_agent.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/execution_agent.hpp @@ -64,7 +64,7 @@ namespace hpx::threads { char const* desc, threads::thread_schedule_state state); void do_resume( - char const* desc, hpx::threads::thread_restart_state statex); + char const* desc, hpx::threads::thread_restart_state statex) const; execution_context context_; }; diff --git a/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp b/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp index 5fb4df759336..5de9b8fc2b67 100644 --- a/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/register_thread.hpp @@ -10,13 +10,10 @@ #pragma once #include -#include #include -#include -#include -#include +#include +#include -#include #include #include @@ -24,6 +21,8 @@ namespace hpx::threads { namespace detail { + HPX_CORE_EXPORT threads::thread_result_type cleanup_thread(); + template struct thread_function { @@ -35,19 +34,7 @@ namespace hpx::threads { // execute the actual thread function f(threads::thread_restart_state::signaled); - // Verify that there are no more registered locks for this - // OS-thread. This will throw if there are still any locks held. - util::force_error_on_lock(); - - // run and free all registered exit functions for this thread - auto* p = get_self_id_data(); - - p->run_thread_exit_callbacks(); - p->free_thread_exit_callbacks(); - - return threads::thread_result_type( - threads::thread_schedule_state::terminated, - threads::invalid_thread_id); + return cleanup_thread(); } }; @@ -62,19 +49,7 @@ namespace hpx::threads { // execute the actual thread function f(); - // Verify that there are no more registered locks for this - // OS-thread. This will throw if there are still any locks held. - util::force_error_on_lock(); - - // run and free all registered exit functions for this thread - auto* p = get_self_id_data(); - - p->run_thread_exit_callbacks(); - p->free_thread_exit_callbacks(); - - return threads::thread_result_type( - threads::thread_schedule_state::terminated, - threads::invalid_thread_id); + return cleanup_thread(); } }; } // namespace detail @@ -112,23 +87,13 @@ namespace hpx::threads { /// throw but returns the result code using the parameter /// \a ec. Otherwise it throws an instance /// of hpx#exception. - inline void register_thread(threads::thread_init_data& data, + HPX_CORE_EXPORT void register_thread(threads::thread_init_data& data, threads::thread_pool_base* pool, threads::thread_id_ref_type& id, - error_code& ec = throws) - { - HPX_ASSERT(pool); - data.run_now = true; - pool->create_thread(data, id, ec); - } + error_code& ec = hpx::throws); - inline threads::thread_id_ref_type register_thread( + HPX_CORE_EXPORT threads::thread_id_ref_type register_thread( threads::thread_init_data& data, threads::thread_pool_base* pool, - error_code& ec = throws) - { - threads::thread_id_ref_type id = threads::invalid_thread_id; - register_thread(data, pool, id, ec); - return id; - } + error_code& ec = hpx::throws); /////////////////////////////////////////////////////////////////////////// /// \brief Create a new \a thread using the given data on the same thread @@ -150,17 +115,11 @@ namespace hpx::threads { /// \a hpx#throws this function doesn't throw but returns /// the result code using the parameter \a ec. Otherwise /// it throws an instance of hpx#exception. - inline void register_thread(threads::thread_init_data& data, - threads::thread_id_ref_type& id, error_code& ec = throws) - { - register_thread(data, detail::get_self_or_default_pool(), id, ec); - } + HPX_CORE_EXPORT void register_thread(threads::thread_init_data& data, + threads::thread_id_ref_type& id, error_code& ec = throws); - inline threads::thread_id_ref_type register_thread( - threads::thread_init_data& data, error_code& ec = throws) - { - return register_thread(data, detail::get_self_or_default_pool(), ec); - } + HPX_CORE_EXPORT threads::thread_id_ref_type register_thread( + threads::thread_init_data& data, error_code& ec = throws); /// \brief Create a new work item using the given data. /// @@ -176,13 +135,9 @@ namespace hpx::threads { /// \a hpx#throws this function doesn't throw but returns /// the result code using the parameter \a ec. Otherwise /// it throws an instance of hpx#exception. - inline thread_id_ref_type register_work(threads::thread_init_data& data, - threads::thread_pool_base* pool, error_code& ec = throws) - { - HPX_ASSERT(pool); - data.run_now = false; - return pool->create_work(data, ec); - } + HPX_CORE_EXPORT thread_id_ref_type register_work( + threads::thread_init_data& data, threads::thread_pool_base* pool, + error_code& ec = hpx::throws); /// \brief Create a new work item using the given data on the same thread /// pool as the calling thread, or on the default thread pool if @@ -200,11 +155,8 @@ namespace hpx::threads { /// throw but returns the result code using the /// parameter \a ec. Otherwise it throws an instance /// of hpx#exception. - inline thread_id_ref_type register_work( - threads::thread_init_data& data, error_code& ec = throws) - { - return register_work(data, detail::get_self_or_default_pool(), ec); - } + HPX_CORE_EXPORT thread_id_ref_type register_work( + threads::thread_init_data& data, error_code& ec = throws); } // namespace hpx::threads /// \endcond diff --git a/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp b/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp index 23a1ba2b1485..0974fc98245a 100644 --- a/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/scheduler_base.hpp @@ -57,10 +57,11 @@ namespace hpx::threads::policies { /// scheduler policies struct scheduler_base { - public: - HPX_NON_COPYABLE(scheduler_base); + scheduler_base(scheduler_base const&) = delete; + scheduler_base(scheduler_base&&) = delete; + scheduler_base& operator=(scheduler_base const&) = delete; + scheduler_base& operator=(scheduler_base&&) = delete; - public: using pu_mutex_type = std::mutex; explicit scheduler_base(std::size_t num_threads, diff --git a/libs/core/threading_base/include/hpx/threading_base/scoped_annotation.hpp b/libs/core/threading_base/include/hpx/threading_base/scoped_annotation.hpp index 9f33c17d5494..4d7cbc5271c7 100644 --- a/libs/core/threading_base/include/hpx/threading_base/scoped_annotation.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/scoped_annotation.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 Hartmut Kaiser +// Copyright (c) 2017-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -11,6 +11,7 @@ #include #if defined(HPX_HAVE_THREAD_DESCRIPTION) +#include #include #include diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp index cb47ff33fabc..51d80608cbae 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #if defined(HPX_HAVE_APEX) #include #endif @@ -50,15 +51,6 @@ namespace hpx::threads { HPX_CORE_EXPORT std::uint32_t get_locality_id(hpx::error_code&); } // namespace detail - //////////////////////////////////////////////////////////////////////////// - class HPX_CORE_EXPORT thread_data; // forward declaration only - - //////////////////////////////////////////////////////////////////////////// - /// The function \a get_self_id_data returns the data of the HPX thread id - /// associated with the current thread (or nullptr if the current thread is - /// not a HPX thread). - HPX_CORE_EXPORT thread_data* get_self_id_data() noexcept; - //////////////////////////////////////////////////////////////////////////// /// A \a thread is the representation of a HPX thread. It's a first class /// object in HPX. In our implementation this is a user level thread running @@ -674,82 +666,6 @@ namespace hpx::threads { { return static_cast(tid.get()); } - - namespace detail { - - HPX_CORE_EXPORT void set_self_ptr(thread_self*) noexcept; - } - - /////////////////////////////////////////////////////////////////////// - /// The function \a get_self returns a reference to the (OS thread - /// specific) self reference to the current HPX thread. - HPX_CORE_EXPORT thread_self& get_self(); - - /// The function \a get_self_ptr returns a pointer to the (OS thread - /// specific) self reference to the current HPX thread. - HPX_CORE_EXPORT thread_self* get_self_ptr() noexcept; - - /// The function \a get_ctx_ptr returns a pointer to the internal data - /// associated with each coroutine. - HPX_CORE_EXPORT thread_self_impl_type* get_ctx_ptr(); - - /// The function \a get_self_ptr_checked returns a pointer to the (OS - /// thread specific) self reference to the current HPX thread. - HPX_CORE_EXPORT thread_self* get_self_ptr_checked(error_code& ec = throws); - - /// The function \a get_self_id returns the HPX thread id of the current - /// thread (or zero if the current thread is not a HPX thread). - HPX_CORE_EXPORT thread_id_type get_self_id() noexcept; - - /// The function \a get_outer_self_id returns the HPX thread id of - /// the current outer thread (or zero if the current thread is not a HPX - /// thread). This usually returns the same as \a get_self_id, except for - /// directly executed threads, in which case this returns the thread id - /// of the outermost HPX thread. - HPX_CORE_EXPORT thread_id_type get_outer_self_id() noexcept; - - /// The function \a get_parent_id returns the HPX thread id of the - /// current thread's parent (or zero if the current thread is not a - /// HPX thread). - /// - /// \note This function will return a meaningful value only if the - /// code was compiled with HPX_HAVE_THREAD_PARENT_REFERENCE - /// being defined. - HPX_CORE_EXPORT thread_id_type get_parent_id() noexcept; - - /// The function \a get_parent_phase returns the HPX phase of the - /// current thread's parent (or zero if the current thread is not a - /// HPX thread). - /// - /// \note This function will return a meaningful value only if the - /// code was compiled with HPX_HAVE_THREAD_PARENT_REFERENCE - /// being defined. - HPX_CORE_EXPORT std::size_t get_parent_phase() noexcept; - - /// The function \a get_self_stacksize returns the stack size of the - /// current thread (or zero if the current thread is not a HPX thread). - HPX_CORE_EXPORT std::ptrdiff_t get_self_stacksize() noexcept; - - /// The function \a get_self_stacksize_enum returns the stack size of the / - //current thread (or thread_stacksize::default if the current thread is not - //a HPX thread). - HPX_CORE_EXPORT thread_stacksize get_self_stacksize_enum() noexcept; - - /// The function \a get_parent_locality_id returns the id of the locality of - /// the current thread's parent (or zero if the current thread is not a - /// HPX thread). - /// - /// \note This function will return a meaningful value only if the - /// code was compiled with HPX_HAVE_THREAD_PARENT_REFERENCE - /// being defined. - HPX_CORE_EXPORT std::uint32_t get_parent_locality_id() noexcept; - - /// The function \a get_self_component_id returns the lva of the component - /// the current thread is acting on - /// - /// \note This function will return a meaningful value only if the code was - /// compiled with HPX_HAVE_THREAD_TARGET_ADDRESS being defined. - HPX_CORE_EXPORT std::uint64_t get_self_component_id() noexcept; } // namespace hpx::threads #include diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp index 82db59ebe895..131369ccdc6c 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp @@ -17,6 +17,9 @@ #include #include #include +#if !defined(HPX_HAVE_THREAD_FULLBACKTRACE_ON_SUSPENSION) +#include +#endif #include #include diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp index c27e0babb6da..89d6789c33d9 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_pool_base.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -107,7 +108,7 @@ namespace hpx::threads { // note: this data structure has to be protected from races from the outside /// \brief The base class used to manage a pool of OS threads. - class HPX_CORE_EXPORT thread_pool_base + class thread_pool_base { public: /// \cond NOINTERNAL diff --git a/libs/core/threading_base/include/hpx/threading_base/threading_base_fwd.hpp b/libs/core/threading_base/include/hpx/threading_base/threading_base_fwd.hpp index 97e9780e88df..230468b44c42 100644 --- a/libs/core/threading_base/include/hpx/threading_base/threading_base_fwd.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/threading_base_fwd.hpp @@ -12,8 +12,11 @@ #include #include #include +#include #include +#include +#include #include #if defined(HPX_HAVE_APEX) @@ -27,7 +30,7 @@ namespace hpx::util::external_timer { namespace hpx::threads { - class thread_data; + class HPX_CORE_EXPORT thread_data; // forward declaration only class thread_data_stackful; class thread_data_stackless; @@ -63,4 +66,86 @@ namespace hpx::threads { std::shared_ptr data); #endif /// \endcond + + //////////////////////////////////////////////////////////////////////////// + /// The function \a get_self_id_data returns the data of the HPX thread id + /// associated with the current thread (or nullptr if the current thread is + /// not a HPX thread). + HPX_CORE_EXPORT thread_data* get_self_id_data() noexcept; + + namespace detail { + + HPX_CORE_EXPORT void set_self_ptr(thread_self*) noexcept; + } + + /////////////////////////////////////////////////////////////////////// + /// The function \a get_self returns a reference to the (OS thread + /// specific) self reference to the current HPX thread. + HPX_CORE_EXPORT thread_self& get_self(); + + /// The function \a get_self_ptr returns a pointer to the (OS thread + /// specific) self reference to the current HPX thread. + HPX_CORE_EXPORT thread_self* get_self_ptr() noexcept; + + /// The function \a get_ctx_ptr returns a pointer to the internal data + /// associated with each coroutine. + HPX_CORE_EXPORT thread_self_impl_type* get_ctx_ptr(); + + /// The function \a get_self_ptr_checked returns a pointer to the (OS + /// thread specific) self reference to the current HPX thread. + HPX_CORE_EXPORT thread_self* get_self_ptr_checked(error_code& ec = throws); + + /// The function \a get_self_id returns the HPX thread id of the current + /// thread (or zero if the current thread is not a HPX thread). + HPX_CORE_EXPORT thread_id_type get_self_id() noexcept; + + /// The function \a get_outer_self_id returns the HPX thread id of + /// the current outer thread (or zero if the current thread is not a HPX + /// thread). This usually returns the same as \a get_self_id, except for + /// directly executed threads, in which case this returns the thread id + /// of the outermost HPX thread. + HPX_CORE_EXPORT thread_id_type get_outer_self_id() noexcept; + + /// The function \a get_parent_id returns the HPX thread id of the + /// current thread's parent (or zero if the current thread is not a + /// HPX thread). + /// + /// \note This function will return a meaningful value only if the + /// code was compiled with HPX_HAVE_THREAD_PARENT_REFERENCE + /// being defined. + HPX_CORE_EXPORT thread_id_type get_parent_id() noexcept; + + /// The function \a get_parent_phase returns the HPX phase of the + /// current thread's parent (or zero if the current thread is not a + /// HPX thread). + /// + /// \note This function will return a meaningful value only if the + /// code was compiled with HPX_HAVE_THREAD_PARENT_REFERENCE + /// being defined. + HPX_CORE_EXPORT std::size_t get_parent_phase() noexcept; + + /// The function \a get_self_stacksize returns the stack size of the + /// current thread (or zero if the current thread is not a HPX thread). + HPX_CORE_EXPORT std::ptrdiff_t get_self_stacksize() noexcept; + + /// The function \a get_self_stacksize_enum returns the stack size of the / + //current thread (or thread_stacksize::default if the current thread is not + //a HPX thread). + HPX_CORE_EXPORT thread_stacksize get_self_stacksize_enum() noexcept; + + /// The function \a get_parent_locality_id returns the id of the locality of + /// the current thread's parent (or zero if the current thread is not a + /// HPX thread). + /// + /// \note This function will return a meaningful value only if the + /// code was compiled with HPX_HAVE_THREAD_PARENT_REFERENCE + /// being defined. + HPX_CORE_EXPORT std::uint32_t get_parent_locality_id() noexcept; + + /// The function \a get_self_component_id returns the lva of the component + /// the current thread is acting on + /// + /// \note This function will return a meaningful value only if the code was + /// compiled with HPX_HAVE_THREAD_TARGET_ADDRESS being defined. + HPX_CORE_EXPORT std::uint64_t get_self_component_id() noexcept; } // namespace hpx::threads diff --git a/libs/core/threading_base/src/annotated_function.cpp b/libs/core/threading_base/src/annotated_function.cpp index 59f41a9d6f64..6274002b5255 100644 --- a/libs/core/threading_base/src/annotated_function.cpp +++ b/libs/core/threading_base/src/annotated_function.cpp @@ -18,7 +18,7 @@ namespace hpx::detail { char const* store_function_annotation(std::string name) { static thread_local std::unordered_set names; - auto r = names.emplace(HPX_MOVE(name)); + auto const r = names.emplace(HPX_MOVE(name)); return (*std::get<0>(r)).c_str(); } } // namespace hpx::detail diff --git a/libs/core/threading_base/src/create_thread.cpp b/libs/core/threading_base/src/create_thread.cpp index 14e3168baa66..dfbc4deaa6ad 100644 --- a/libs/core/threading_base/src/create_thread.cpp +++ b/libs/core/threading_base/src/create_thread.cpp @@ -13,8 +13,6 @@ #include #include -#include - namespace hpx::threads::detail { void create_thread(policies::scheduler_base* scheduler, @@ -52,7 +50,7 @@ namespace hpx::threads::detail { } #endif - thread_self* self = get_self_ptr(); + thread_self const* self = get_self_ptr(); #ifdef HPX_HAVE_THREAD_PARENT_REFERENCE if (nullptr == data.parent_id) diff --git a/libs/core/threading_base/src/create_work.cpp b/libs/core/threading_base/src/create_work.cpp index c0aaca600949..ed3ebeee2d91 100644 --- a/libs/core/threading_base/src/create_work.cpp +++ b/libs/core/threading_base/src/create_work.cpp @@ -61,7 +61,7 @@ namespace hpx::threads::detail { #endif ; - thread_self* self = get_self_ptr(); + thread_self const* self = get_self_ptr(); #ifdef HPX_HAVE_THREAD_PARENT_REFERENCE if (nullptr == data.parent_id) diff --git a/libs/core/threading_base/src/execution_agent.cpp b/libs/core/threading_base/src/execution_agent.cpp index 64952d6f736a..7369490260a1 100644 --- a/libs/core/threading_base/src/execution_agent.cpp +++ b/libs/core/threading_base/src/execution_agent.cpp @@ -43,7 +43,7 @@ namespace hpx::threads { std::string execution_agent::description() const { - thread_id_type id = self_.get_thread_id(); + thread_id_type const id = self_.get_thread_id(); if (HPX_UNLIKELY(!id)) { HPX_THROW_EXCEPTION(hpx::error::null_thread_id, @@ -109,9 +109,9 @@ namespace hpx::threads { // Just yield until time has passed by... auto now = std::chrono::steady_clock::now(); - // Note: we yield at least once to allow for other threads to - // make progress in any case. We also use yield instead of yield_k - // for the same reason. + // Note: we yield at least once to allow for other threads to make + // progress in any case. We also use yield instead of yield_k for the + // same reason. std::size_t k = 0; do { @@ -177,16 +177,15 @@ namespace hpx::threads { thrd_data->set_last_worker_thread_num( hpx::get_local_worker_thread_num()); - threads::thread_restart_state statex = - threads::thread_restart_state::unknown; + threads::thread_restart_state statex; { #ifdef HPX_HAVE_THREAD_DESCRIPTION - threads::detail::reset_lco_description desc( + [[maybe_unused]] threads::detail::reset_lco_description reset_desc( id.noref(), threads::thread_description(desc)); #endif #ifdef HPX_HAVE_THREAD_BACKTRACE_ON_SUSPENSION - threads::detail::reset_backtrace bt(id); + [[maybe_unused]] threads::detail::reset_backtrace reset_bt(id); #endif [[maybe_unused]] on_exit_reset_held_lock_data held_locks; @@ -219,7 +218,7 @@ namespace hpx::threads { } void execution_agent::do_resume( - char const* /* desc */, hpx::threads::thread_restart_state statex) + char const* /* desc */, hpx::threads::thread_restart_state statex) const { threads::detail::set_thread_state(self_.get_thread_id(), thread_schedule_state::pending, statex, thread_priority::normal, diff --git a/libs/core/threading_base/src/get_default_pool.cpp b/libs/core/threading_base/src/get_default_pool.cpp index ce63964f077c..2a5454df4cdd 100644 --- a/libs/core/threading_base/src/get_default_pool.cpp +++ b/libs/core/threading_base/src/get_default_pool.cpp @@ -22,7 +22,7 @@ namespace hpx_start { // Redefining weak variables defined in hpx_main.hpp to facilitate error // checking and make sure correct errors are thrown. It is added again to // make sure that these variables are defined correctly in cases where - // hpx_main functionalities are not used. + // hpx_main functionality is not used. HPX_SYMBOL_EXPORT bool is_linked __attribute__((weak)) = false; HPX_SYMBOL_EXPORT bool include_libhpx_wrap __attribute__((weak)) = false; } // namespace hpx_start @@ -41,8 +41,7 @@ namespace hpx::threads::detail { thread_pool_base* get_self_or_default_pool() { thread_pool_base* pool = nullptr; - auto thrd_data = get_self_id_data(); - if (thrd_data) + if (auto const* thrd_data = get_self_id_data()) { pool = thrd_data->get_scheduler_base()->get_parent_pool(); } diff --git a/libs/core/threading_base/src/print.cpp b/libs/core/threading_base/src/print.cpp index 0b70873c92f4..3f9e5540bda9 100644 --- a/libs/core/threading_base/src/print.cpp +++ b/libs/core/threading_base/src/print.cpp @@ -9,6 +9,7 @@ #include #include +#include #include /// \cond NODETAIL @@ -76,7 +77,7 @@ namespace hpx::debug { } else { - hpx::threads::thread_data* dummy = + hpx::threads::thread_data const* dummy = hpx::threads::get_self_id_data(); os << dummy << " "; } diff --git a/libs/core/threading_base/src/register_thread.cpp b/libs/core/threading_base/src/register_thread.cpp new file mode 100644 index 000000000000..b8184162d40b --- /dev/null +++ b/libs/core/threading_base/src/register_thread.cpp @@ -0,0 +1,103 @@ +// Copyright (c) 2007-2023 Hartmut Kaiser +// Copyright (c) 2018 Thomas Heller +// Copyright (c) 2011 Bryce Lelbach +// Copyright (c) 2008-2009 Chirag Dekate, Anshul Tandon +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::threads { + + /////////////////////////////////////////////////////////////////////////// + namespace detail { + + threads::thread_result_type cleanup_thread() + { + // Verify that there are no more registered locks for this + // OS-thread. This will throw if there are still any locks held. + util::force_error_on_lock(); + + // run and free all registered exit functions for this thread + auto* p = get_self_id_data(); + + p->run_thread_exit_callbacks(); + p->free_thread_exit_callbacks(); + + return threads::thread_result_type( + threads::thread_schedule_state::terminated, + threads::invalid_thread_id); + } + } // namespace detail + + /////////////////////////////////////////////////////////////////////////// + threads::thread_id_ref_type register_thread(threads::thread_init_data& data, + threads::thread_pool_base* pool, error_code& ec) + { + HPX_ASSERT(pool); + + threads::thread_id_ref_type id = threads::invalid_thread_id; + data.run_now = true; + pool->create_thread(data, id, ec); + return id; + } + + void register_thread(threads::thread_init_data& data, + threads::thread_pool_base* pool, threads::thread_id_ref_type& id, + error_code& ec) + { + HPX_ASSERT(pool); + + data.run_now = true; + pool->create_thread(data, id, ec); + } + + void register_thread(threads::thread_init_data& data, + threads::thread_id_ref_type& id, error_code& ec) + { + auto* pool = detail::get_self_or_default_pool(); + HPX_ASSERT(pool); + + data.run_now = true; + pool->create_thread(data, id, ec); + } + + threads::thread_id_ref_type register_thread( + threads::thread_init_data& data, error_code& ec) + { + auto* pool = detail::get_self_or_default_pool(); + HPX_ASSERT(pool); + + threads::thread_id_ref_type id = threads::invalid_thread_id; + data.run_now = true; + pool->create_thread(data, id, ec); + return id; + } + + /////////////////////////////////////////////////////////////////////////// + thread_id_ref_type register_work(threads::thread_init_data& data, + threads::thread_pool_base* pool, error_code& ec) + { + HPX_ASSERT(pool); + data.run_now = false; + return pool->create_work(data, ec); + } + + thread_id_ref_type register_work( + threads::thread_init_data& data, error_code& ec) + { + auto* pool = detail::get_self_or_default_pool(); + HPX_ASSERT(pool); + + data.run_now = false; + return pool->create_work(data, ec); + } +} // namespace hpx::threads diff --git a/libs/core/threading_base/src/thread_description.cpp b/libs/core/threading_base/src/thread_description.cpp index 358c192deb94..684082e512f9 100644 --- a/libs/core/threading_base/src/thread_description.cpp +++ b/libs/core/threading_base/src/thread_description.cpp @@ -60,8 +60,7 @@ namespace hpx::threads { return; } - hpx::threads::thread_id_type const id = hpx::threads::get_self_id(); - if (id) + if (hpx::threads::thread_id_type const id = hpx::threads::get_self_id()) { // get the current task description thread_description const desc = diff --git a/libs/full/command_line_handling/include/hpx/command_line_handling/command_line_handling.hpp b/libs/full/command_line_handling/include/hpx/command_line_handling/command_line_handling.hpp index 55ed47532afd..3d31a6703527 100644 --- a/libs/full/command_line_handling/include/hpx/command_line_handling/command_line_handling.hpp +++ b/libs/full/command_line_handling/include/hpx/command_line_handling/command_line_handling.hpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index b50ed8b8a14e..34437c3e8534 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -28,7 +28,9 @@ #include #include #include +#if defined(HPX_HAVE_MAX_CPU_COUNT) #include +#endif #include #include @@ -36,9 +38,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -109,18 +109,23 @@ namespace hpx::util { } /////////////////////////////////////////////////////////////////////// - std::size_t handle_num_localities(util::manage_config& cfgmap, - hpx::program_options::variables_map& vm, - util::batch_environment& env, bool using_nodelist, + std::size_t handle_num_localities(util::manage_config const& cfgmap, + hpx::program_options::variables_map const& vm, + util::batch_environment const& env, bool using_nodelist, std::size_t num_localities, bool initial) { - std::size_t batch_localities = env.retrieve_number_of_localities(); - if (num_localities == 1 && batch_localities != std::size_t(-1)) + std::size_t const batch_localities = + env.retrieve_number_of_localities(); + if (num_localities == 1 && + batch_localities != static_cast(-1)) { - std::size_t cfg_num_localities = cfgmap.get_value( - "hpx.localities", batch_localities); - if (cfg_num_localities > 1) + if (auto const cfg_num_localities = + cfgmap.get_value( + "hpx.localities", batch_localities); + cfg_num_localities > 1) + { num_localities = cfg_num_localities; + } } if (!initial && env.found_batch_environment() && using_nodelist && @@ -132,7 +137,8 @@ namespace hpx::util { if (vm.count("hpx:localities")) { - std::size_t localities = vm["hpx:localities"].as(); + std::size_t const localities = + vm["hpx:localities"].as(); if (localities == 0) { @@ -164,9 +170,9 @@ namespace hpx::util { /////////////////////////////////////////////////////////////////////// std::size_t get_number_of_default_cores( - util::batch_environment& env, bool use_process_mask) + util::batch_environment const& env, bool use_process_mask) { - std::size_t num_cores = + std::size_t const num_cores = hpx::local::detail::get_number_of_default_cores( use_process_mask); if (use_process_mask) @@ -175,13 +181,13 @@ namespace hpx::util { } std::size_t batch_threads = env.retrieve_number_of_threads(); - if (batch_threads == std::size_t(-1)) + if (batch_threads == static_cast(-1)) { return num_cores; } // assuming we assign the first N cores ... - threads::topology& top = threads::create_topology(); + threads::topology const& top = threads::create_topology(); std::size_t core = 0; for (/**/; core < num_cores; ++core) { @@ -193,11 +199,11 @@ namespace hpx::util { } /////////////////////////////////////////////////////////////////////// - std::size_t handle_num_threads(util::manage_config& cfgmap, + std::size_t handle_num_threads(util::manage_config const& cfgmap, util::runtime_configuration const& rtcfg, - hpx::program_options::variables_map& vm, - util::batch_environment& env, bool using_nodelist, bool initial, - bool use_process_mask) + hpx::program_options::variables_map const& vm, + util::batch_environment const& env, bool using_nodelist, + bool initial, bool use_process_mask) { // If using the process mask we override "cores" and "all" options // but keep explicit numeric values. @@ -208,16 +214,15 @@ namespace hpx::util { detail::get_number_of_default_cores(env, use_process_mask); std::size_t const batch_threads = env.retrieve_number_of_threads(); - std::string threads_str = - cfgmap.get_value("hpx.os_threads", - rtcfg.get_entry( - "hpx.os_threads", std::to_string(init_threads))); + auto threads_str = cfgmap.get_value("hpx.os_threads", + rtcfg.get_entry( + "hpx.os_threads", std::to_string(init_threads))); - std::size_t threads = 0; + std::size_t threads; if ("cores" == threads_str) { threads = init_cores; - if (batch_threads != std::size_t(-1)) + if (batch_threads != static_cast(-1)) { threads = batch_threads; } @@ -225,12 +230,12 @@ namespace hpx::util { else if ("all" == threads_str) { threads = init_threads; - if (batch_threads != std::size_t(-1)) + if (batch_threads != static_cast(-1)) { threads = batch_threads; } } - else if (batch_threads != std::size_t(-1)) + else if (batch_threads != static_cast(-1)) { threads = batch_threads; } @@ -246,7 +251,7 @@ namespace hpx::util { if ("all" == threads_str) { threads = init_threads; - if (batch_threads != std::size_t(-1)) + if (batch_threads != static_cast(-1)) { threads = batch_threads; } @@ -254,7 +259,7 @@ namespace hpx::util { else if ("cores" == threads_str) { threads = init_cores; - if (batch_threads != std::size_t(-1)) + if (batch_threads != static_cast(-1)) { threads = batch_threads; } @@ -284,7 +289,7 @@ namespace hpx::util { } // make sure minimal requested number of threads is observed - std::size_t min_os_threads = cfgmap.get_value( + auto min_os_threads = cfgmap.get_value( "hpx.force_min_os_threads", threads); if (min_os_threads == 0) @@ -321,7 +326,7 @@ namespace hpx::util { /////////////////////////////////////////////////////////////////////// #if !defined(HPX_HAVE_NETWORKING) void check_networking_option( - hpx::program_options::variables_map& vm, char const* option) + hpx::program_options::variables_map const& vm, char const* option) { if (vm.count(option) != 0) { @@ -334,7 +339,7 @@ namespace hpx::util { #endif void check_networking_options( - [[maybe_unused]] hpx::program_options::variables_map& vm) + [[maybe_unused]] hpx::program_options::variables_map const& vm) { #if !defined(HPX_HAVE_NETWORKING) check_networking_option(vm, "hpx:agas"); @@ -358,7 +363,7 @@ namespace hpx::util { std::vector ini_config, hpx::function hpx_main_f) : base_type(HPX_MOVE(rtcfg), HPX_MOVE(ini_config), HPX_MOVE(hpx_main_f)) - , node_(std::size_t(-1)) + , node_(static_cast(-1)) , num_localities_(1) { } @@ -374,7 +379,8 @@ namespace hpx::util { // disabled detail::check_networking_options(vm); - bool debug_clp = node != std::size_t(-1) && vm.count("hpx:debug-clp"); + bool debug_clp = + node != static_cast(-1) && vm.count("hpx:debug-clp"); // create host name mapping util::map_hostnames mapnames(debug_clp); @@ -387,12 +393,11 @@ namespace hpx::util { // The AGAS host name and port number are pre-initialized from //the command line - std::string agas_host = cfgmap.get_value( + auto agas_host = cfgmap.get_value( "hpx.agas.address", rtcfg_.get_entry("hpx.agas.address", "")); - std::uint16_t agas_port = - cfgmap.get_value("hpx.agas.port", - hpx::util::from_string( - rtcfg_.get_entry("hpx.agas.port", HPX_INITIAL_IP_PORT))); + auto agas_port = cfgmap.get_value("hpx.agas.port", + hpx::util::from_string( + rtcfg_.get_entry("hpx.agas.port", HPX_INITIAL_IP_PORT))); if (vm.count("hpx:agas")) { @@ -419,8 +424,8 @@ namespace hpx::util { vm["hpx:iftransform"].as())); } - typedef util::map_hostnames::transform_function_type - transform_function_type; + using transform_function_type = + util::map_hostnames::transform_function_type; mapnames.use_transform(transform_function_type(iftransform)); } @@ -438,6 +443,7 @@ namespace hpx::util { "one of the --hpx:nodefile and --hpx:nodes options at the " "same time."); } + std::string node_file = vm["hpx:nodefile"].as(); ini_config.emplace_back("hpx.nodefile!=" + node_file); std::ifstream ifs(node_file.c_str()); @@ -540,7 +546,7 @@ namespace hpx::util { "hpx.parcel.port", initial_hpx_port); run_agas_server = vm.count("hpx:run-agas-server") != 0; - if (node == std::size_t(-1)) + if (node == static_cast(-1)) node = env.retrieve_node_number(); #else num_localities_ = 1; @@ -625,7 +631,8 @@ namespace hpx::util { } #endif } - else if (node != std::size_t(-1) || vm.count("hpx:node")) + else if (node != static_cast(-1) || + vm.count("hpx:node")) { // command line overwrites the environment if (vm.count("hpx:node")) @@ -770,7 +777,8 @@ namespace hpx::util { #endif } - // write HPX and AGAS network parameters to the proper ini-file entries + // write HPX and AGAS network parameters to the proper ini-file + // entries ini_config.emplace_back("hpx.parcel.address=" + hpx_host); ini_config.emplace_back( "hpx.parcel.port=" + std::to_string(hpx_port)); @@ -952,7 +960,8 @@ namespace hpx::util { error_mode |= util::commandline_error_mode::ignore_aliases; hpx::program_options::variables_map prevm; if (!util::parse_commandline(rtcfg_, desc_cmdline, argv[0], args, - prevm, std::size_t(-1), error_mode, rtcfg_.mode_)) + prevm, static_cast(-1), error_mode, + rtcfg_.mode_)) { return -1; } @@ -1012,7 +1021,7 @@ namespace hpx::util { // be considered now. // minimally assume one locality and this is the console - if (node_ == std::size_t(-1)) + if (node_ == static_cast(-1)) node_ = 0; for (std::shared_ptr& reg : diff --git a/libs/full/command_line_handling/src/late_command_line_handling.cpp b/libs/full/command_line_handling/src/late_command_line_handling.cpp index 962c3194fea3..a8f5598d7647 100644 --- a/libs/full/command_line_handling/src/late_command_line_handling.cpp +++ b/libs/full/command_line_handling/src/late_command_line_handling.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2007-2023 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -43,7 +42,7 @@ namespace hpx::util { hpx::program_options::variables_map vm; std::vector still_unregistered_options; util::parse_commandline(ini, options, unknown_cmd_line, vm, - std::size_t(-1), mode, + static_cast(-1), mode, get_runtime_mode_from_name(runtime_mode), nullptr, &still_unregistered_options); @@ -65,7 +64,7 @@ namespace hpx::util { hpx::program_options::variables_map vm; util::parse_commandline(ini, options, cmd_line, vm, - std::size_t(-1), + static_cast(-1), util::commandline_error_mode::allow_unregistered | util::commandline_error_mode:: report_missing_config_file, diff --git a/libs/full/command_line_handling/src/parse_command_line.cpp b/libs/full/command_line_handling/src/parse_command_line.cpp index a2a6a0a4896e..2729ffa6db87 100644 --- a/libs/full/command_line_handling/src/parse_command_line.cpp +++ b/libs/full/command_line_handling/src/parse_command_line.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -14,9 +13,8 @@ #include #include -#include +#include #include -#include #include #include #include @@ -47,18 +45,19 @@ namespace hpx::util { // any --hpx: option without a second ':' is handled elsewhere as // well - std::string::size_type p = s.find_first_of(':', hpx_prefix_len); + std::string::size_type const p = + s.find_first_of(':', hpx_prefix_len); if (p == std::string::npos) return false; if (hpx::util::from_string( s.substr(hpx_prefix_len, p - hpx_prefix_len), - std::size_t(-1)) == node) + static_cast(-1)) == node) { using hpx::local::detail::trim_whitespace; // this option is for the current locality only - std::string::size_type p1 = s.find_first_of('=', p); + std::string::size_type const p1 = s.find_first_of('=', p); if (p1 != std::string::npos) { // the option has a value @@ -127,9 +126,9 @@ namespace hpx::util { using hpx::program_options::store; using hpx::program_options::command_line_style::unix_style; - util::commandline_error_mode mode = + util::commandline_error_mode const mode = error_mode & util::commandline_error_mode::ignore_aliases; - util::commandline_error_mode notmode = + util::commandline_error_mode const notmode = error_mode & ~util::commandline_error_mode::ignore_aliases; store(hpx::local::detail::get_commandline_parser( @@ -160,22 +159,21 @@ namespace hpx::util { return; filesystem::path dir(filesystem::initial_path()); - filesystem::path app(appname); + filesystem::path const app(appname); appname = filesystem::basename(app.filename()); // walk up the hierarchy, trying to find a file .cfg while (!dir.empty()) { filesystem::path filename = dir / (appname + ".cfg"); - util::commandline_error_mode mode = error_mode & + util::commandline_error_mode const mode = error_mode & ~util::commandline_error_mode::report_missing_config_file; std::vector options = hpx::local::detail::read_config_file_options( filename.string(), mode); - bool result = handle_config_file_options( - options, desc_cfgfile, vm, ini, node, mode); - if (result) + if (handle_config_file_options( + options, desc_cfgfile, vm, ini, node, mode)) { break; // break on the first options file found } @@ -204,7 +202,7 @@ namespace hpx::util { using hpx::program_options::options_description; if (vm.count("hpx:options-file")) { - std::vector const& cfg_files = + auto const& cfg_files = vm["hpx:options-file"].as>(); for (std::string const& cfg_file : cfg_files) @@ -454,7 +452,7 @@ namespace hpx::util { all_options[options_type::desc_cfgfile].add( all_options[options_type::counter_options]); #endif - bool result = hpx::local::detail::parse_commandline(rtcfg, + bool const result = hpx::local::detail::parse_commandline(rtcfg, all_options, app_options, args, vm, error_mode, visible, unregistered_options); @@ -491,8 +489,8 @@ namespace hpx::util { std::string extract_arg0(std::string const& cmdline) { - std::string::size_type p = cmdline.find_first_of(" \t"); - if (p != std::string::npos) + if (std::string::size_type const p = cmdline.find_first_of(" \t"); + p != std::string::npos) { return cmdline.substr(0, p); } From b54c97f2935bce8c9d4d0385efcf3c9f2a97c07b Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 13 Jul 2023 22:58:32 +0200 Subject: [PATCH 181/333] Add MPI migration guide --- docs/sphinx/manual/migration_guide.rst | 122 +++++++++++++++++- .../include/hpx/parallel/algorithm.hpp | 1 + .../examples/channel_communicator.cpp | 5 +- 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 83027781549e..dcfa1a5eb260 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -12,7 +12,7 @@ Migration guide =============== The Migration Guide serves as a valuable resource for developers seeking to transition their -parallel computing applications from different APIs (i.e. |openmp|, |tbb|) to |hpx|. |hpx|, an +parallel computing applications from different APIs (i.e. |openmp|, |tbb|, |mpi|) to |hpx|. |hpx|, an advanced C++ library, offers a versatile and high-performance platform for parallel and distributed computing, providing a wide range of features and capabilities. This guide aims to assist developers in understanding the key differences between different APIs and |hpx|, and it provides step-by-step @@ -1044,3 +1044,123 @@ task_group |hpx| drew inspiration from |tbb| to introduce the :cpp:func:`hpx::experimental::task_group` feature. Therefore, utilizing :cpp:func:`hpx::experimental::task_group` provides an equivalent functionality to `tbb::task_group`. + +|mpi| +===== + +|mpi| is a standardized communication protocol and library that allows multiple processes or +nodes in a parallel computing system to exchange data and coordinate their execution. + +MPI_send & MPI_recv +------------------- + +Let's assume we have the following simple message passing code where each process sends a +message to the next process in a circular manner. The exchanged message is modified and printed +to the console. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + #include + #include + + constexpr int times = 2; + + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_localities; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + + int this_locality; + MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); + + int next_locality = (this_locality + 1) % num_localities; + std::vector msg_vec = {0, 1}; + + int cnt = 0; + int msg = msg_vec[this_locality]; + + int recv_msg; + MPI_Request request_send, request_recv; + MPI_Status status; + + while (cnt < times) { + cnt += 1; + + MPI_Isend(&msg, 1, MPI_INT, next_locality, cnt, MPI_COMM_WORLD, + &request_send); + MPI_Irecv(&recv_msg, 1, MPI_INT, next_locality, cnt, MPI_COMM_WORLD, + &request_recv); + + MPI_Wait(&request_send, &status); + MPI_Wait(&request_recv, &status); + + std::cout << "Time: " << cnt << ", Locality " << this_locality + << " received msg: " << recv_msg << "\n"; + + recv_msg += 10; + msg = recv_msg; + } + + MPI_Finalize(); + return 0; + } + +|hpx| equivalent: + +.. literalinclude:: ../../libs/full/collectives/examples/channel_communicator.cpp + :start-after: //[doc + :end-before: //doc] + +To perform message passing between different processes in |hpx| we can use a channel communicator. +To understand this example, let's focus on the `hpx_main()` function: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. +- `create_channel_communicator` function is used to create a channel to serve the communication. + This function takes several arguments, including the launch policy (`hpx::launch::sync`), the + name of the communicator (`channel_communicator_name`), the number of localities, and the ID + of the current locality. +- The communication follows a ring pattern, where each process (or locality) sends a message to + its neighbor in a circular manner. This means that the messages circulate around the localities, + ensuring that the communication wraps around when reaching the end of the locality sequence. + To achieve this, the `next_locality` variable is calculated as the ID of the next locality in + the ring. +- The initial values for the communication are set (`msg_vec`, `cnt`, `msg`). +- The `set()` function is called to send the message to the next locality in the ring. The message + is sent asynchronously and is associated with a tag (`cnt`). +- The `get()` function is called to receive a message from the next locality. It is also associated + with the same tag as the `set()` operation. +- The `setf.get()` call blocks until the message sending operation is complete. +- A continuation is set up using the function `then()` to handle the received message. + Inside the continuation: + + - The received message value (`rec_msg`) is retrieved using `f.get()`. + + - The received message is printed to the console and then modified by adding 10. + + - The `set()` and `get()` operations are repeated to send and receive the modified message to + the next locality. + + - The `setf.get()` call blocks until the new message sending operation is complete. +- The `done_msg.get()` call blocks until the continuation is complete for the current loop iteration. + +Having said that, we conclude to the following table: + +.. table:: |hpx| equivalent functions of |mpi| + + ========================= ============================================================== + |openmpi| function |hpx| equivalent + ========================= ============================================================== + MPI_Comm_create `hpx::collectives::create_channel_communicator()` + MPI_Comm_size `hpx::get_num_localities` + MPI_Comm_rank `hpx::get_locality_id()` + MPI_Isend `hpx::collectives::set()` + MPI_Irecv `hpx::collectives::get()` + MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` + ========================= ============================================================== diff --git a/libs/core/algorithms/include/hpx/parallel/algorithm.hpp b/libs/core/algorithms/include/hpx/parallel/algorithm.hpp index 127c3f8993b8..af8372c62ffc 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithm.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithm.hpp @@ -51,6 +51,7 @@ #include #include #include +#include #include // Parallelism TS V2 diff --git a/libs/full/collectives/examples/channel_communicator.cpp b/libs/full/collectives/examples/channel_communicator.cpp index 841a896c5bd0..def6f0788a65 100644 --- a/libs/full/collectives/examples/channel_communicator.cpp +++ b/libs/full/collectives/examples/channel_communicator.cpp @@ -5,6 +5,8 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +/////////////////////////////////////////////////////////////////////////////// +//[doc #include #if !defined(HPX_COMPUTE_DEVICE_CODE) @@ -20,14 +22,12 @@ using namespace hpx::collectives; -/////////////////////////////////////////////////////////////////////////////// constexpr char const* channel_communicator_name = "/example/channel_communicator/"; // the number of times constexpr int times = 2; -//////////////////////////////////////////////////////////////////////// int hpx_main() { std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); @@ -89,3 +89,4 @@ int main(int argc, char* argv[]) return 0; #endif } +//doc] From d15eb184cb585e6c62eeb0fa01055bda34a885bd Mon Sep 17 00:00:00 2001 From: dimitraka Date: Tue, 1 Aug 2023 18:53:49 +0200 Subject: [PATCH 182/333] Add MPI_Gather doc --- docs/sphinx/manual/migration_guide.rst | 103 ++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index dcfa1a5eb260..54928c8f60b1 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1051,7 +1051,7 @@ equivalent functionality to `tbb::task_group`. |mpi| is a standardized communication protocol and library that allows multiple processes or nodes in a parallel computing system to exchange data and coordinate their execution. -MPI_send & MPI_recv +MPI_Send & MPI_Recv ------------------- Let's assume we have the following simple message passing code where each process sends a @@ -1164,3 +1164,104 @@ Having said that, we conclude to the following table: MPI_Irecv `hpx::collectives::get()` MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` ========================= ============================================================== + +MPI_Gather +---------- + +The following code gathers data from all processes to the root process and verifies +the gathered data in the root process. + +|mpi| code: + +.. code-block:: c++ + + #include + + int main(int argc, char* argv[]) + { + int num_localities, this_locality; + int gather_data[10]; + + // Initialize MPI + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); + + // Test functionality based on immediate local result value + for (int i = 0; i < 10; ++i) + { + if (this_locality == 0) + { + int value = 42; + MPI_Gather(&value, 1, MPI_INT, gather_data, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (this_locality == 0) + { + for (int j = 0; j < num_localities; ++j) + { + // Verify gathered data + assert(j + 42 == gather_data[j]); + } + } + } + else + { + int value = this_locality + 42; + MPI_Gather(&value, 1, MPI_INT, nullptr, 0, MPI_INT, 0, MPI_COMM_WORLD); + } + } + + // Finalize MPI + MPI_Finalize(); + + return 0; + } + + +|hpx| equivalent: + +.. literalinclude:: ../../libs/full/collectives/tests/unit/gather.cpp + :start-after: //[doc + :end-before: //doc] + +|hpx| uses two functions to implement the functionality of `MPI_Gather`: `hpx::gather_here` and +`hpx::gather_there`. `hpx::gather_here` is gathering data from all localities to the locality +with ID 0 (root locality). `hpx::gather_there` allows non-root localities to participate in the +gather operation by sending data to the root locality. In more detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- If the current locality is the root (its ID is equal to 0): + + - the `hpx::gather_here` function is used to perform the gather operation. It collects data from all + other localities into the `overall_result` future object. The function arguments provide the necessary + information, such as the base name for the gather operation (`gather_direct_basename`), the value + to be gathered (`value`), the number of localities (`num_localities`), the current locality ID + (`this_locality`), and the generation number (related to the gather operation). + + - The `get()` member function of the `overall_result` future is used to retrieve the gathered data. + + - The next `for` loop is used to verify the correctness of the gathered data (`sol`). `HPX_TEST` + is a macro provided by the |hpx| testing utilities to perform similar testing withthe Standard + C++ macro `assert`. + +- If the current locality is not the root: + + - The `hpx::gather_there` function is used to participate in the gather operation initiated by + the root locality. It sends the data (in this case, the value `this_locality + 42`) to the root + locality, indicating that it should be included in the gathering. + + - The `get()` member function of the `overall_result` future is used to wait for the gather operation + to complete for this locality. + + +.. table:: |hpx| equivalent functions of |mpi| + + ========================= ===================================================================== + |openmpi| function |hpx| equivalent + ========================= ===================================================================== + MPI_Comm_size `hpx::get_num_localities` + MPI_Comm_rank `hpx::get_locality_id()` + MPI_Gather `hpx::gather_here()` and `hpx::gather_there()` both used with `get()` + ========================= ===================================================================== From 9325b219dc7a9d02e6c67dfb8e1bffc05c8251bb Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 23 Aug 2023 11:24:12 +0200 Subject: [PATCH 183/333] Add scatter and gather doc --- docs/sphinx/manual/migration_guide.rst | 252 +++++++++++++++++++++---- 1 file changed, 214 insertions(+), 38 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 54928c8f60b1..b0df47b8b369 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1157,7 +1157,6 @@ Having said that, we conclude to the following table: ========================= ============================================================== |openmpi| function |hpx| equivalent ========================= ============================================================== - MPI_Comm_create `hpx::collectives::create_channel_communicator()` MPI_Comm_size `hpx::get_num_localities` MPI_Comm_rank `hpx::get_locality_id()` MPI_Isend `hpx::collectives::set()` @@ -1175,54 +1174,90 @@ the gathered data in the root process. .. code-block:: c++ + #include #include + #include + #include - int main(int argc, char* argv[]) - { - int num_localities, this_locality; - int gather_data[10]; + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_localities, this_locality; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); - // Initialize MPI - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &num_localities); - MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); + std::vector local_data; // Data to be gathered - // Test functionality based on immediate local result value - for (int i = 0; i < 10; ++i) + if (this_locality == 0) { + local_data.resize(num_localities); // Resize the vector on the root process + } + + // Each process calculates its local data value + int my_data = 42 + this_locality; + + for (std::uint32_t i = 0; i != 10; ++i) { + + // Gather data from all processes to the root process (process 0) + MPI_Gather(&my_data, 1, MPI_INT, local_data.data(), 1, MPI_INT, 0, + MPI_COMM_WORLD); + + // Only the root process (process 0) will print the gathered data + if (this_locality == 0) { + std::cout << "Gathered data on the root: "; + for (int i = 0; i < num_localities; ++i) { + std::cout << local_data[i] << " "; + } + std::cout << std::endl; + } + } + + MPI_Finalize(); + return 0; + } + + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + std::uint32_t this_locality = hpx::get_locality_id(); + + // test functionality based on immediate local result value + auto gather_direct_client = create_communicator(gather_direct_basename, + num_sites_arg(num_localities), this_site_arg(this_locality)); + + for (std::uint32_t i = 0; i != 10; ++i) + { + if (this_locality == 0) { - if (this_locality == 0) - { - int value = 42; - MPI_Gather(&value, 1, MPI_INT, gather_data, 1, MPI_INT, 0, MPI_COMM_WORLD); + hpx::future> overall_result = + gather_here(gather_direct_client, std::uint32_t(42)); - if (this_locality == 0) - { - for (int j = 0; j < num_localities; ++j) - { - // Verify gathered data - assert(j + 42 == gather_data[j]); - } - } - } - else + std::vector sol = overall_result.get(); + std::cout << "Gathered data on the root:"; + + for (std::size_t j = 0; j != sol.size(); ++j) { - int value = this_locality + 42; - MPI_Gather(&value, 1, MPI_INT, nullptr, 0, MPI_INT, 0, MPI_COMM_WORLD); + HPX_TEST(j + 42 == sol[j]); + std::cout << " " << sol[j]; } + std::cout << std::endl; + } + else + { + hpx::future overall_result = + gather_there(gather_direct_client, this_locality + 42); + overall_result.get(); } - // Finalize MPI - MPI_Finalize(); - - return 0; } +This code will print 10 times the following message: -|hpx| equivalent: +.. code-block:: c++ -.. literalinclude:: ../../libs/full/collectives/tests/unit/gather.cpp - :start-after: //[doc - :end-before: //doc] + Gathered data on the root: 42 43 |hpx| uses two functions to implement the functionality of `MPI_Gather`: `hpx::gather_here` and `hpx::gather_there`. `hpx::gather_here` is gathering data from all localities to the locality @@ -1232,18 +1267,21 @@ gather operation by sending data to the root locality. In more detail: - `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while `hpx::get_locality_id()` returns the ID of the current locality. +- The function `create_communicator()` is used to create a communicator called + `gather_direct_client`. + - If the current locality is the root (its ID is equal to 0): - the `hpx::gather_here` function is used to perform the gather operation. It collects data from all other localities into the `overall_result` future object. The function arguments provide the necessary - information, such as the base name for the gather operation (`gather_direct_basename`), the value - to be gathered (`value`), the number of localities (`num_localities`), the current locality ID + information, such as the base name for the gather operation (`gather_direct_basename`), the value + to be gathered (`value`), the number of localities (`num_localities`), the current locality ID (`this_locality`), and the generation number (related to the gather operation). - The `get()` member function of the `overall_result` future is used to retrieve the gathered data. - The next `for` loop is used to verify the correctness of the gathered data (`sol`). `HPX_TEST` - is a macro provided by the |hpx| testing utilities to perform similar testing withthe Standard + is a macro provided by the |hpx| testing utilities to perform similar testing with the Standard C++ macro `assert`. - If the current locality is not the root: @@ -1265,3 +1303,141 @@ gather operation by sending data to the root locality. In more detail: MPI_Comm_rank `hpx::get_locality_id()` MPI_Gather `hpx::gather_here()` and `hpx::gather_there()` both used with `get()` ========================= ===================================================================== + +MPI_Scatter +----------- + +The following code gathers data from all processes to the root process and verifies +the gathered data in the root process. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_localities, this_locality; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); + + int num_localities = num_localities; + std::vector data(num_localities); + + if (this_locality == 0) { + // Fill the data vector on the root locality (locality 0) + for (int i = 0; i < num_localities; ++i) { + data[i] = 42 + i; + } + } + + int local_data; // Variable to store the received data + + // Scatter data from the root locality to all other localities + MPI_Scatter(&data[0], 1, MPI_INT, &local_data, 1, MPI_INT, 0, MPI_COMM_WORLD); + + // Now, each locality has its own local_data + + // Print the local_data on each locality + std::cout << "Locality " << this_locality << " received " << local_data + << std::endl; + + MPI_Finalize(); + return 0; + } + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + HPX_TEST_LTE(std::uint32_t(2), num_localities); + + std::uint32_t this_locality = hpx::get_locality_id(); + + auto scatter_direct_client = + hpx::collectives::create_communicator(scatter_direct_basename, + num_sites_arg(num_localities), this_site_arg(this_locality)); + + // test functionality based on immediate local result value + for (std::uint32_t i = 0; i != 10; ++i) + { + if (this_locality == 0) + { + std::vector data(num_localities); + std::iota(data.begin(), data.end(), 42 + i); + + hpx::future result = + scatter_to(scatter_direct_client, std::move(data)); + + HPX_TEST_EQ(i + 42 + this_locality, result.get()); + } + else + { + hpx::future result = + scatter_from(scatter_direct_client); + + HPX_TEST_EQ(i + 42 + this_locality, result.get()); + + std::cout << "Locality " << this_locality << " received " + << i + 42 + this_locality << std::endl; + } + } + +For num_localities = 2 and since we run for 10 iterations this code will print +the following message: + +.. code-block:: c++ + + Locality 1 received 43 + Locality 1 received 44 + Locality 1 received 45 + Locality 1 received 46 + Locality 1 received 47 + Locality 1 received 48 + Locality 1 received 49 + Locality 1 received 50 + Locality 1 received 51 + Locality 1 received 52 + +|hpx| uses two functions to implement the functionality of `MPI_Scatter`: `hpx::scatter_to` and +`hpx::scatter_from`. `hpx::scatter_to` is distributing the data from the locality with ID 0 +(root locality) to all other localities. `hpx::scatter_from` allows non-root localities to receive +the data from the root locality. In more detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `hpx::collectives::create_communicator()` is used to create a communicator called + `scatter_direct_client`. + +- If the current locality is the root (its ID is equal to 0): + + - The data vector is filled with values ranging from `42 + i` to `42 + i + num_localities - 1`. + + - The `hpx::scatter_to` function is used to perform the scatter operation using the communicator + `scatter_direct_client`. This scatters the data vector to other localities and + returns a future representing the result. + + - `HPX_TEST_EQ` is a macro provided by the |hpx| testing utilities to test the distributed values. + +- If the current locality is not the root: + + - The `hpx::scatter_from` function is used to collect the data by the root locality. + + - `HPX_TEST_EQ` is a macro provided by the |hpx| testing utilities to test the collected values. + + +.. table:: |hpx| equivalent functions of |mpi| + + ========================= ============================================= + |openmpi| function |hpx| equivalent + ========================= ============================================= + MPI_Comm_size `hpx::get_num_localities` + MPI_Comm_rank `hpx::get_locality_id()` + MPI_Scatter `hpx::scatter_to()` and `hpx::scatter_from()` + ========================= ============================================= From 54ad817afca03676a0730b82cf123c1524c921ab Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 24 Aug 2023 13:36:18 +0200 Subject: [PATCH 184/333] Add MPI_Allgather doc --- docs/sphinx/manual/migration_guide.rst | 185 +++++++++++++++++++------ 1 file changed, 141 insertions(+), 44 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index b0df47b8b369..44acae9881fc 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1180,39 +1180,39 @@ the gathered data in the root process. #include int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + MPI_Init(&argc, &argv); - int num_localities, this_locality; - MPI_Comm_size(MPI_COMM_WORLD, &num_localities); - MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); + int num_localities, this_locality; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); - std::vector local_data; // Data to be gathered + std::vector local_data; // Data to be gathered - if (this_locality == 0) { - local_data.resize(num_localities); // Resize the vector on the root process - } + if (this_locality == 0) { + local_data.resize(num_localities); // Resize the vector on the root process + } - // Each process calculates its local data value - int my_data = 42 + this_locality; + // Each process calculates its local data value + int my_data = 42 + this_locality; - for (std::uint32_t i = 0; i != 10; ++i) { + for (std::uint32_t i = 0; i != 10; ++i) { - // Gather data from all processes to the root process (process 0) - MPI_Gather(&my_data, 1, MPI_INT, local_data.data(), 1, MPI_INT, 0, - MPI_COMM_WORLD); + // Gather data from all processes to the root process (process 0) + MPI_Gather(&my_data, 1, MPI_INT, local_data.data(), 1, MPI_INT, 0, + MPI_COMM_WORLD); - // Only the root process (process 0) will print the gathered data - if (this_locality == 0) { - std::cout << "Gathered data on the root: "; - for (int i = 0; i < num_localities; ++i) { - std::cout << local_data[i] << " "; - } - std::cout << std::endl; + // Only the root process (process 0) will print the gathered data + if (this_locality == 0) { + std::cout << "Gathered data on the root: "; + for (int i = 0; i < num_localities; ++i) { + std::cout << local_data[i] << " "; + } + std::cout << std::endl; + } } - } - MPI_Finalize(); - return 0; + MPI_Finalize(); + return 0; } @@ -1319,35 +1319,35 @@ the gathered data in the root process. #include int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + MPI_Init(&argc, &argv); - int num_localities, this_locality; - MPI_Comm_size(MPI_COMM_WORLD, &num_localities); - MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); + int num_localities, this_locality; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + MPI_Comm_rank(MPI_COMM_WORLD, &this_locality); - int num_localities = num_localities; - std::vector data(num_localities); + int num_localities = num_localities; + std::vector data(num_localities); - if (this_locality == 0) { - // Fill the data vector on the root locality (locality 0) - for (int i = 0; i < num_localities; ++i) { - data[i] = 42 + i; + if (this_locality == 0) { + // Fill the data vector on the root locality (locality 0) + for (int i = 0; i < num_localities; ++i) { + data[i] = 42 + i; + } } - } - int local_data; // Variable to store the received data + int local_data; // Variable to store the received data - // Scatter data from the root locality to all other localities - MPI_Scatter(&data[0], 1, MPI_INT, &local_data, 1, MPI_INT, 0, MPI_COMM_WORLD); + // Scatter data from the root locality to all other localities + MPI_Scatter(&data[0], 1, MPI_INT, &local_data, 1, MPI_INT, 0, MPI_COMM_WORLD); - // Now, each locality has its own local_data + // Now, each locality has its own local_data - // Print the local_data on each locality - std::cout << "Locality " << this_locality << " received " << local_data - << std::endl; + // Print the local_data on each locality + std::cout << "Locality " << this_locality << " received " << local_data + << std::endl; - MPI_Finalize(); - return 0; + MPI_Finalize(); + return 0; } |hpx| equivalent: @@ -1441,3 +1441,100 @@ the data from the root locality. In more detail: MPI_Comm_rank `hpx::get_locality_id()` MPI_Scatter `hpx::scatter_to()` and `hpx::scatter_from()` ========================= ============================================= + +MPI_Allgather +------------- + +The following code gathers data from all processes and sends the data to all +processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + #include + + int main(int argc, char **argv) { + MPI_Init(&argc, &argv); + + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + // Get the number of MPI processes (equivalent to HPX localities). + int num_localities = size; + + // Get the MPI process rank (equivalent to HPX locality ID). + int here = rank; + + std::uint32_t value = here; + + std::vector r(num_localities); + + // Perform an all-gather operation to gather values from all processes. + MPI_Allgather(&value, 1, MPI_UINT32_T, r.data(), 1, MPI_UINT32_T, + MPI_COMM_WORLD); + + // Print the result. + std::cout << "Locality " << here << " has values:"; + for (size_t j = 0; j < r.size(); ++j) { + std::cout << " " << r[j]; + } + std::cout << std::endl; + + MPI_Finalize(); + return 0; + } + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + std::uint32_t here = hpx::get_locality_id(); + + // test functionality based on immediate local result value + auto all_gather_direct_client = + create_communicator(all_gather_direct_basename, + num_sites_arg(num_localities), this_site_arg(here)); + + std::uint32_t value = here; + + hpx::future> overall_result = + all_gather(all_gather_direct_client, value); + + std::vector r = overall_result.get(); + + std::cout << "Locality " << here << " has values:"; + for (std::size_t j = 0; j != r.size(); ++j) + { + std::cout << " " << j; + } + std::cout << std::endl; + +For num_localities = 2 this code will print the following message: + +.. code-block:: c++ + + Locality 0 has values: 0 1 + Locality 1 has values: 0 1 + +|hpx| uses the function `all_gather` to implement the functionality of `MPI_Allgather`. In more +detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `hpx::collectives::create_communicator()` is used to create a communicator called + `all_gather_direct_client`. + +- The values that the localities exchange with each other are equal to each locality's ID. + +- The gather operation is performed using `all_gather`. The result is stored in an `hpx::future` + object called `overall_result`, which represents a future result that can be retrieved later when + needed. + +- The `get()` function waits until the result is available and then stores it in the vector called `r`. From c3d3c980669fc79dce30d6238e65ac8955c2a2bd Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 24 Aug 2023 14:13:07 +0200 Subject: [PATCH 185/333] Add MPI_Allreduce doc --- docs/sphinx/manual/migration_guide.rst | 89 ++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 44acae9881fc..46e2fe282893 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1538,3 +1538,92 @@ detail: needed. - The `get()` function waits until the result is available and then stores it in the vector called `r`. + + +MPI_Allreduce +------------- + +The following code combines values from all processes and distributes the result back to all processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + + int main(int argc, char **argv) { + MPI_Init(&argc, &argv); + + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + // Get the number of MPI processes + int num_localities = size; + + // Get the MPI process rank + int here = rank; + + // Create a communicator for the all reduce operation. + MPI_Comm all_reduce_direct_client; + MPI_Comm_split(MPI_COMM_WORLD, 0, rank, &all_reduce_direct_client); + + // Perform the all reduce operation to calculate the sum of 'here' values. + std::uint32_t value = here; + std::uint32_t res = 0; + MPI_Allreduce(&value, &res, 1, MPI_UINT32_T, MPI_SUM, + all_reduce_direct_client); + + std::cout << "Locality " << rank << " has value: " << res << std::endl; + + MPI_Finalize(); + return 0; + } + + + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t const num_localities = + hpx::get_num_localities(hpx::launch::sync); + std::uint32_t const here = hpx::get_locality_id(); + + auto const all_reduce_direct_client = + create_communicator(all_reduce_direct_basename, + num_sites_arg(num_localities), this_site_arg(here)); + + std::uint32_t value = here; + + hpx::future overall_result = + all_reduce(all_reduce_direct_client, value, std::plus{}); + + std::uint32_t res = overall_result.get(); + std::cout << "Locality " << here << " has value: " << res << std::endl; + +For num_localities = 2 this code will print the following message: + +.. code-block:: c++ + + Locality 0 has value: 1 + Locality 1 has value: 1 + +|hpx| uses the function `all_reduce` to implement the functionality of `MPI_Allreduce`. In more +detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `hpx::collectives::create_communicator()` is used to create a communicator called + `all_reduce_direct_client`. + +- The value of each locality is equal to its ID. + +- The reduce operation is performed using `all_reduce`. The result is stored in an `hpx::future` + object called `overall_result`, which represents a future result that can be retrieved later when + needed. + +- The `get()` function waits until the result is available and then stores it in the variable `res`. From ac67290f86d66d741483a147cd52ec7b39a89d2e Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 24 Aug 2023 15:59:28 +0200 Subject: [PATCH 186/333] Add MPI_Alltoall doc --- docs/sphinx/manual/migration_guide.rst | 111 ++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 46e2fe282893..7a6051009c83 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1464,10 +1464,10 @@ processes. MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); - // Get the number of MPI processes (equivalent to HPX localities). + // Get the number of MPI processes int num_localities = size; - // Get the MPI process rank (equivalent to HPX locality ID). + // Get the MPI process rank int here = rank; std::uint32_t value = here; @@ -1582,8 +1582,6 @@ The following code combines values from all processes and distributes the result return 0; } - - |hpx| equivalent: .. code-block:: c++ @@ -1627,3 +1625,108 @@ detail: needed. - The `get()` function waits until the result is available and then stores it in the variable `res`. + +MPI_Alltoall +------------- + +The following code cGathers data from and scatters data to all processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + #include + #include + + int main(int argc, char **argv) { + MPI_Init(&argc, &argv); + + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + // Get the number of MPI processes + int num_localities = size; + + // Get the MPI process rank + int this_locality = rank; + + // Create a communicator for all-to-all operation. + MPI_Comm all_to_all_direct_client; + MPI_Comm_split(MPI_COMM_WORLD, 0, rank, &all_to_all_direct_client); + + std::vector values(num_localities); + std::fill(values.begin(), values.end(), this_locality); + + // Create vectors to store received values. + std::vector r(num_localities); + + // Perform an all-to-all operation to exchange values with other localities. + MPI_Alltoall(values.data(), 1, MPI_UINT32_T, r.data(), 1, MPI_UINT32_T, + all_to_all_direct_client); + + // Print the results. + std::cout << "Locality " << this_locality << " has values:"; + for (std::size_t j = 0; j != r.size(); ++j) { + std::cout << " " << r[j]; + } + std::cout << std::endl; + + MPI_Finalize(); + return 0; + } + + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + std::uint32_t this_locality = hpx::get_locality_id(); + + auto all_to_all_direct_client = + create_communicator(all_to_all_direct_basename, + num_sites_arg(num_localities), this_site_arg(this_locality)); + + std::vector values(num_localities); + std::fill(values.begin(), values.end(), this_locality); + + hpx::future> overall_result = + all_to_all(all_to_all_direct_client, std::move(values)); + + std::vector r = overall_result.get(); + std::cout << "Locality " << this_locality << " has values:"; + + for (std::size_t j = 0; j != r.size(); ++j) + { + std::cout << " " << r[j]; + } + std::cout << std::endl; + +For num_localities = 2 this code will print the following message: + +.. code-block:: c++ + + Locality 0 has values: 0 1 + Locality 1 has values: 0 1 + +|hpx| uses the function `all_to_all` to implement the functionality of `MPI_Alltoall`. In more +detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `hpx::collectives::create_communicator()` is used to create a communicator called + `all_to_all_direct_client`. + +- The value each locality sends is equal to its ID. + +- The all-to-all operation is performed using `all_to_all`. The result is stored in an `hpx::future` + object called `overall_result`, which represents a future result that can be retrieved later when + needed. + +- The `get()` function waits until the result is available and then stores it in the variable `r`. + From cca1f0f4c9ddb1e3f144676df08edab7519e05d7 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 28 Aug 2023 10:46:38 -0500 Subject: [PATCH 187/333] Listing HPX_WITH_GIT_BRANCH and HPX_WITH_GIT_TAG only if those are defined --- cmake/HPX_UpdateGitDocs.cmake | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmake/HPX_UpdateGitDocs.cmake b/cmake/HPX_UpdateGitDocs.cmake index 89fca2bb2334..6ecea2906076 100644 --- a/cmake/HPX_UpdateGitDocs.cmake +++ b/cmake/HPX_UpdateGitDocs.cmake @@ -57,9 +57,8 @@ string(REGEX REPLACE " " ";" HPX_WITH_DOCUMENTATION_OUTPUT_FORMATS ) # If a branch name has been set, we copy files to a corresponding directory -message("HPX_WITH_GIT_BRANCH=\"${HPX_WITH_GIT_BRANCH}\"") if(HPX_WITH_GIT_BRANCH) - message("Updating branch directory") + message("Updating branch directory, " "HPX_WITH_GIT_BRANCH=\"${HPX_WITH_GIT_BRANCH}\"") set(DOCS_BRANCH_DEST "${HPX_BINARY_DIR}/docs/gh-pages/branches/${HPX_WITH_GIT_BRANCH}" ) @@ -92,9 +91,8 @@ if(HPX_WITH_GIT_BRANCH) endif() # If a tag name has been set, we copy files to a corresponding directory -message("HPX_WITH_GIT_TAG=\"${HPX_WITH_GIT_TAG}\"") if(HPX_WITH_GIT_TAG) - message("Updating tag directory") + message("Updating tag directory, " "HPX_WITH_GIT_TAG=\"${HPX_WITH_GIT_TAG}\"") set(DOCS_TAG_DEST "${HPX_BINARY_DIR}/docs/gh-pages/tags/${HPX_WITH_GIT_TAG}") file(REMOVE_RECURSE "${DOCS_TAG_DEST}") if("html" IN_LIST HPX_WITH_DOCUMENTATION_OUTPUT_FORMATS) From cadad9569b32e36b3ce432d1a6ede0005c6763a6 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 28 Aug 2023 10:48:48 -0500 Subject: [PATCH 188/333] More tweaks to listing CMake variables --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 28cf94607398..dff5c025e0ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2225,8 +2225,8 @@ hpx_include(SetOutputPaths) # ############################################################################## # Fixing git tag, if necessary -hpx_info("HPX_WITH_GIT_TAG: " ${HPX_WITH_GIT_TAG}) -hpx_info("HPX_WITH_GIT_BRANCH: " ${HPX_WITH_GIT_BRANCH}) +hpx_debug("HPX_WITH_GIT_TAG: " ${HPX_WITH_GIT_TAG}) +hpx_debug("HPX_WITH_GIT_BRANCH: " ${HPX_WITH_GIT_BRANCH}) if(HPX_WITH_GIT_BRANCH AND ((NOT HPX_WITH_GIT_TAG) OR "${HPX_WITH_GIT_TAG}x" STREQUAL "x") From f8c439a17bd2f44f2d4d57bc5c18b5a101f7d1a9 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 22 Oct 2022 14:23:46 -0500 Subject: [PATCH 189/333] Investigating partitioned_vector problems - flyby: reduce the need for make_ready_future for various AGAS operations --- .../partitioned_vector_component_impl.hpp | 6 +- .../partitioned_vector_decl.hpp | 2 +- .../partitioned_vector_impl.hpp | 12 +- .../partitioned_vector_segmented_iterator.hpp | 2 +- examples/throttle/spin.cpp | 3 +- libs/core/futures/CMakeLists.txt | 1 + .../include/hpx/futures/future_or_value.hpp | 71 +++++++++++ .../lcos_local/detail/preprocess_future.hpp | 10 ++ .../include/hpx/actions/transfer_action.hpp | 12 +- .../include/hpx/agas/addressing_service.hpp | 33 ++++-- libs/full/agas/src/addressing_service.cpp | 112 ++++++++++++------ libs/full/agas/src/detail/interface.cpp | 43 +++---- .../hpx/agas_base/primary_namespace.hpp | 11 +- libs/full/agas_base/src/primary_namespace.cpp | 27 +++-- .../src/server/symbol_namespace_server.cpp | 36 +++--- .../hpx/async_colocated/get_colocation_id.hpp | 4 +- .../async_colocated/src/get_colocation_id.cpp | 10 +- .../hpx/async_distributed/put_parcel.hpp | 27 ++++- .../include/hpx/components/client_base.hpp | 10 +- .../include/hpx/components/get_ptr.hpp | 86 +++++++++++--- libs/full/components/src/client_base.cpp | 2 +- .../hpx/components_base/agas_interface.hpp | 10 +- .../detail/agas_interface_functions.hpp | 11 +- .../components_base/src/agas_interface.cpp | 26 +++- .../src/detail/agas_interface_functions.cpp | 13 +- .../naming/include/hpx/naming/split_gid.hpp | 12 +- libs/full/naming/src/credit_handling.cpp | 76 +++++++++--- .../tests/unit/agas/local_address_rebind.cpp | 11 +- .../partitioned_vector_inclusive_scan.cpp | 5 +- .../tests/unit/partitioned_vector_scan.hpp | 4 +- 30 files changed, 483 insertions(+), 205 deletions(-) create mode 100644 libs/core/futures/include/hpx/futures/future_or_value.hpp diff --git a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_component_impl.hpp b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_component_impl.hpp index 944690fb987c..a2c139ca5dab 100644 --- a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_component_impl.hpp +++ b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_component_impl.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2014 Anuj R. Sharma -// Copyright (c) 2014-2017 Hartmut Kaiser +// Copyright (c) 2014-2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -339,8 +339,8 @@ namespace hpx { partitioned_vector_partition::get_ptr() const { error_code ec(throwmode::lightweight); - return hpx::get_ptr>(this->get_id()) - .get(ec); + return hpx::get_ptr>( + hpx::launch::sync, this->get_id(), ec); } template */> diff --git a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_decl.hpp b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_decl.hpp index 17c2d2122213..0e9afacaa09f 100644 --- a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_decl.hpp +++ b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_decl.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2014 Anuj R. Sharma -// Copyright (c) 2014-2017 Hartmut Kaiser +// Copyright (c) 2014-2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying diff --git a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_impl.hpp b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_impl.hpp index d06bbe53c72d..bce6ca1d2229 100644 --- a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_impl.hpp +++ b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_impl.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2014 Anuj R. Sharma -// Copyright (c) 2014-2017 Hartmut Kaiser +// Copyright (c) 2014-2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -73,6 +72,7 @@ namespace hpx { std::uint32_t this_locality = get_locality_id(); std::vector> ptrs; + ptrs.reserve(partitions_.size()); typedef typename partitions_vector_type::const_iterator const_iterator; @@ -346,6 +346,7 @@ namespace hpx { // now initialize our data structures std::uint32_t this_locality = get_locality_id(); std::vector> ptrs; + ptrs.reserve(num_parts); std::size_t num_part = 0; std::size_t allocated_size = 0; @@ -397,7 +398,7 @@ namespace hpx { } HPX_ASSERT(l == num_parts); - hpx::when_all(ptrs).get(); + hpx::wait_all(ptrs); // cache our partition size partition_size_ = get_partition_size(); @@ -440,11 +441,12 @@ namespace hpx { std::uint32_t this_locality = get_locality_id(); std::vector> ptrs; + ptrs.reserve(rhs.partitions_.size()); - partitions_vector_type partitions; // Fixing the size of partitions to avoid race conditions between // possible reallocations during push back and the continuation // to set the local partition data + partitions_vector_type partitions; partitions.resize(rhs.partitions_.size()); for (std::size_t i = 0; i != rhs.partitions_.size(); ++i) { @@ -461,7 +463,7 @@ namespace hpx { } } - hpx::when_all(ptrs).get(); + hpx::wait_all(ptrs); size_ = rhs.size_; partition_size_ = rhs.partition_size_; diff --git a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_segmented_iterator.hpp b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_segmented_iterator.hpp index 0c5571fbe041..86693f038c23 100644 --- a/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_segmented_iterator.hpp +++ b/components/containers/partitioned_vector/include/hpx/components/containers/partitioned_vector/partitioned_vector_segmented_iterator.hpp @@ -1,5 +1,5 @@ // Copyright (c) 2014 Anuj R. Sharma -// Copyright (c) 2014-2016 Hartmut Kaiser +// Copyright (c) 2014-2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying diff --git a/examples/throttle/spin.cpp b/examples/throttle/spin.cpp index 7ccc45a6a07b..06631248f42e 100644 --- a/examples/throttle/spin.cpp +++ b/examples/throttle/spin.cpp @@ -47,7 +47,8 @@ int hpx_main() for (id_type const& locality_ : localities) { - address addr = hpx::agas::resolve(locality_).get(); + address addr = + hpx::agas::resolve(hpx::launch::sync, locality_); hpx::util::format_to(std::cout, " [{1}] {2}\n", get_locality_id_from_gid(locality_.get_gid()), diff --git a/libs/core/futures/CMakeLists.txt b/libs/core/futures/CMakeLists.txt index f914351c40d4..a7b48127e601 100644 --- a/libs/core/futures/CMakeLists.txt +++ b/libs/core/futures/CMakeLists.txt @@ -11,6 +11,7 @@ set(futures_headers hpx/futures/future.hpp hpx/futures/future_fwd.hpp hpx/futures/futures_factory.hpp + hpx/futures/future_or_value.hpp hpx/futures/detail/future_data.hpp hpx/futures/detail/future_transforms.hpp hpx/futures/packaged_continuation.hpp diff --git a/libs/core/futures/include/hpx/futures/future_or_value.hpp b/libs/core/futures/include/hpx/futures/future_or_value.hpp new file mode 100644 index 000000000000..31af55ae0651 --- /dev/null +++ b/libs/core/futures/include/hpx/futures/future_or_value.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2022 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include + +namespace hpx { + + template + struct future_or_value + { + future_or_value(T const& value) + : data(value) + { + } + + future_or_value(T&& value) noexcept + : data(HPX_MOVE(value)) + { + } + + future_or_value(hpx::future&& value) noexcept + : data(HPX_MOVE(value)) + { + } + + constexpr bool has_value() const noexcept + { + return hpx::holds_alternative(data); + } + constexpr bool has_future() const noexcept + { + return hpx::holds_alternative>(data); + } + + T& get_value() & + { + return hpx::get(data); + } + T const& get_value() const& + { + return hpx::get(data); + } + T&& get_value() && + { + return hpx::get(HPX_MOVE(data)); + } + + hpx::future& get_future() & + { + return hpx::get>(data); + } + hpx::future const& get_future() const& + { + return hpx::get>(data); + } + hpx::future&& get_future() && + { + return hpx::get>(HPX_MOVE(data)); + } + + private: + hpx::variant> data; + }; +} // namespace hpx diff --git a/libs/core/lcos_local/include/hpx/lcos_local/detail/preprocess_future.hpp b/libs/core/lcos_local/include/hpx/lcos_local/detail/preprocess_future.hpp index b0774571dc9e..9fa015d211a3 100644 --- a/libs/core/lcos_local/include/hpx/lcos_local/detail/preprocess_future.hpp +++ b/libs/core/lcos_local/include/hpx/lcos_local/detail/preprocess_future.hpp @@ -123,6 +123,16 @@ namespace hpx::serialization::detail { ++num_futures_; } + void decrement_future_count() + { + std::lock_guard l(mtx_); + HPX_ASSERT(num_futures_ > 0); + if (--num_futures_ == 0) + { + done_ = true; + } + } + void reset() { std::lock_guard l(mtx_); diff --git a/libs/full/actions/include/hpx/actions/transfer_action.hpp b/libs/full/actions/include/hpx/actions/transfer_action.hpp index 52ee265ea866..d80bd6a33d50 100644 --- a/libs/full/actions/include/hpx/actions/transfer_action.hpp +++ b/libs/full/actions/include/hpx/actions/transfer_action.hpp @@ -223,16 +223,14 @@ namespace hpx::actions { { // If this is a direct action and deferred schedule was requested, // that is we are not the last parcel, return immediately - if (base_type::direct_execution::value) + if constexpr (base_type::direct_execution::value) { return; } - else - { - // If this is not a direct action, we can safely set - // deferred_schedule to false - deferred_schedule = false; - } + + // If this is not a direct action, we can safely set + // deferred_schedule to false + deferred_schedule = false; } schedule_thread(HPX_MOVE(target), lva, comptype, num_thread); diff --git a/libs/full/agas/include/hpx/agas/addressing_service.hpp b/libs/full/agas/include/hpx/agas/addressing_service.hpp index c7b4e1e3163e..400d491ae522 100644 --- a/libs/full/agas/include/hpx/agas/addressing_service.hpp +++ b/libs/full/agas/include/hpx/agas/addressing_service.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -177,7 +178,7 @@ namespace hpx { namespace agas { void garbage_collect(error_code& ec = throws); static std::int64_t synchronize_with_async_incref( - hpx::future fut, hpx::id_type const& id, + std::int64_t old_credit, hpx::id_type const& id, std::int64_t compensated_credit); server::primary_namespace& get_local_primary_namespace_service() @@ -221,7 +222,7 @@ namespace hpx { namespace agas { util::runtime_configuration& rtcfg); naming::address resolve_full_postproc(naming::gid_type const& id, - future f); + primary_namespace::resolved_type const&); bool bind_postproc( naming::gid_type const& id, gva const& g, future f); @@ -856,15 +857,17 @@ namespace hpx { namespace agas { } /////////////////////////////////////////////////////////////////////////// - hpx::future resolve_async(naming::gid_type const& id); + hpx::future_or_value resolve_async( + naming::gid_type const& id); - hpx::future resolve_async(hpx::id_type const& id) + hpx::future_or_value resolve_async( + hpx::id_type const& id) { return resolve_async(id.get_gid()); } /////////////////////////////////////////////////////////////////////////// - hpx::future get_colocation_id_async( + hpx::future_or_value get_colocation_id_async( hpx::id_type const& id); /////////////////////////////////////////////////////////////////////////// @@ -893,10 +896,11 @@ namespace hpx { namespace agas { return addr; } - hpx::future resolve_full_async( + hpx::future_or_value resolve_full_async( naming::gid_type const& id); - hpx::future resolve_full_async(hpx::id_type const& id) + hpx::future_or_value resolve_full_async( + hpx::id_type const& id) { return resolve_full_async(id.get_gid()); } @@ -978,14 +982,25 @@ namespace hpx { namespace agas { /// throw but returns the result code using the /// parameter \a ec. Otherwise it throws an instance /// of hpx#exception. - hpx::future incref_async(naming::gid_type const& gid, + hpx::future_or_value incref_async( + naming::gid_type const& gid, std::int64_t credits = 1, + hpx::id_type const& keep_alive = hpx::invalid_id); + + /// \cond NOINTERN + std::int64_t incref_async_helper(naming::gid_type const& gid, std::int64_t credits = 1, hpx::id_type const& keep_alive = hpx::invalid_id); + /// \endcond std::int64_t incref(naming::gid_type const& gid, std::int64_t credits = 1, error_code& ec = throws) { - return incref_async(gid, credits).get(ec); + auto result = incref_async(gid, credits); + if (result.has_value()) + { + return HPX_MOVE(result).get_value(); + } + return result.get_future().get(ec); } /// \brief Decrement the global reference count for the given id diff --git a/libs/full/agas/src/addressing_service.cpp b/libs/full/agas/src/addressing_service.cpp index d4a7a1a0a238..c36106c4884c 100644 --- a/libs/full/agas/src/addressing_service.cpp +++ b/libs/full/agas/src/addressing_service.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -291,7 +292,8 @@ namespace hpx::agas { } } - // Search again ... might have been added by a different thread already + // Search again ... might have been added by a different thread + // already it = resolved_localities_.find(gid); if (it == resolved_localities_.end()) { @@ -315,8 +317,8 @@ namespace hpx::agas { return it->second; } - // TODO: We need to ensure that the locality isn't unbound while it still holds - // referenced objects. + // TODO: We need to ensure that the locality isn't unbound while it still + // holds referenced objects. bool addressing_service::unregister_locality( naming::gid_type const& gid, error_code& ec) { @@ -1058,13 +1060,14 @@ namespace hpx::agas { return false; } - hpx::future addressing_service::resolve_async( + hpx::future_or_value addressing_service::resolve_async( naming::gid_type const& gid) { if (!gid) { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "addressing_service::resolve_async", "invalid reference id"); + return naming::address(); } // Try the cache. @@ -1073,7 +1076,9 @@ namespace hpx::agas { naming::address addr; error_code ec; if (resolve_cached(gid, addr, ec)) - return make_ready_future(addr); + { + return addr; + } if (ec) { @@ -1086,7 +1091,7 @@ namespace hpx::agas { return resolve_full_async(gid); } - hpx::future addressing_service::get_colocation_id_async( + hpx::future_or_value addressing_service::get_colocation_id_async( hpx::id_type const& id) { if (!id) @@ -1094,6 +1099,7 @@ namespace hpx::agas { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "addressing_service::get_colocation_id_async", "invalid reference id"); + return hpx::invalid_id; } return primary_ns_.colocate(id.get_gid()); @@ -1101,15 +1107,12 @@ namespace hpx::agas { /////////////////////////////////////////////////////////////////////////// naming::address addressing_service::resolve_full_postproc( - naming::gid_type const& id, future f) + naming::gid_type const& id, primary_namespace::resolved_type const& rep) { - using hpx::get; - naming::address addr; - auto rep = f.get(); - if (get<0>(rep) == naming::invalid_gid || - get<2>(rep) == naming::invalid_gid) + if (hpx::get<0>(rep) == naming::invalid_gid || + hpx::get<2>(rep) == naming::invalid_gid) { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "addressing_service::resolve_full_postproc", @@ -1118,8 +1121,8 @@ namespace hpx::agas { // Resolve the gva to the real resolved address (which is just a gva // with as fully resolved LVA and and offset of zero). - naming::gid_type const base_gid = get<0>(rep); - gva const base_gva = get<1>(rep); + naming::gid_type const base_gid = hpx::get<0>(rep); + gva const base_gva = hpx::get<1>(rep); gva const g = base_gva.resolve(id, base_gid); @@ -1144,23 +1147,37 @@ namespace hpx::agas { return addr; } - hpx::future addressing_service::resolve_full_async( - naming::gid_type const& gid) + hpx::future_or_value + addressing_service::resolve_full_async(naming::gid_type const& gid) { if (!gid) { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "addressing_service::resolve_full_async", "invalid reference id"); + return naming::address(); } // ask server - future f = - primary_ns_.resolve_full(gid); + auto result = primary_ns_.resolve_full(gid); - return f.then(hpx::launch::sync, - util::one_shot(hpx::bind_front( - &addressing_service::resolve_full_postproc, this, gid))); + if (result.has_value()) + { + try + { + return resolve_full_postproc(gid, result.get_value()); + } + catch (...) + { + return hpx::make_exceptional_future( + std::current_exception()); + } + } + + return result.get_future().then( + hpx::launch::sync, [this, gid](auto&& f) { + return resolve_full_postproc(gid, f.get()); + }); } /////////////////////////////////////////////////////////////////////////// @@ -1301,26 +1318,38 @@ namespace hpx::agas { // incref was sent. The pending decref was subtracted from the amount of // credits to incref. std::int64_t addressing_service::synchronize_with_async_incref( - hpx::future fut, hpx::id_type const&, + std::int64_t old_credit, hpx::id_type const&, std::int64_t compensated_credit) { - return fut.get() + compensated_credit; + return old_credit + compensated_credit; } - hpx::future addressing_service::incref_async( + std::int64_t addressing_service::incref_async_helper( naming::gid_type const& id, std::int64_t credit, hpx::id_type const& keep_alive) { + auto result = incref_async(id, credit, keep_alive); + if (result.has_value()) + { + return HPX_MOVE(result).get_value(); + } + return result.get_future().get(); + } + + hpx::future_or_value addressing_service::incref_async( + naming::gid_type const& id, std::int64_t credit, + hpx::id_type const& keep_alive) + { // {{{ incref implementation naming::gid_type raw(naming::detail::get_stripped_gid(id)); if (HPX_UNLIKELY(nullptr == threads::get_self_ptr())) { // reschedule this call as an HPX thread - hpx::future (addressing_service::*incref_async_ptr)( + std::int64_t (addressing_service::*incref_async_ptr)( naming::gid_type const&, std::int64_t, hpx::id_type const&) = - &addressing_service::incref_async; + &addressing_service::incref_async_helper; - return async(incref_async_ptr, this, raw, credit, keep_alive); + return hpx::async(incref_async_ptr, this, raw, credit, keep_alive); } if (HPX_UNLIKELY(0 >= credit)) @@ -1328,6 +1357,7 @@ namespace hpx::agas { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, "addressing_service::incref_async", "invalid credit count of {1}", credit); + return std::int64_t(-1); } HPX_ASSERT(keep_alive != hpx::invalid_id); @@ -1390,23 +1420,29 @@ namespace hpx::agas { } } + // no need to talk to AGAS, acknowledge the incref immediately if (!has_pending_incref) { - // no need to talk to AGAS, acknowledge the incref immediately - return hpx::make_ready_future(pending_decrefs); + return pending_decrefs; } naming::gid_type const e_lower = pending_incref.first; - - hpx::future f = primary_ns_.increment_credit( + auto result = primary_ns_.increment_credit( pending_incref.second, e_lower, e_lower); // pass the amount of compensated decrefs to the callback - return f.then(hpx::launch::sync, - util::one_shot( - hpx::bind(&addressing_service::synchronize_with_async_incref, - hpx::placeholders::_1, keep_alive, pending_decrefs))); - } + if (result.has_value()) + { + return synchronize_with_async_incref( + result.get_value(), keep_alive, pending_decrefs); + } + + return result.get_future().then( + hpx::launch::sync, [keep_alive, pending_decrefs](auto&& f) { + return synchronize_with_async_incref( + f.get(), keep_alive, pending_decrefs); + }); + } // }}} /////////////////////////////////////////////////////////////////////////// void addressing_service::decref( @@ -1509,7 +1545,7 @@ namespace hpx::agas { // We need to modify the reference count. naming::gid_type& mutable_gid = const_cast(id).get_gid(); naming::gid_type const new_gid = - naming::detail::split_gid_if_needed(mutable_gid).get(); + naming::detail::split_gid_if_needed(hpx::launch::sync, mutable_gid); std::int64_t const new_credit = naming::detail::get_credit_from_gid(new_gid); @@ -1534,7 +1570,7 @@ namespace hpx::agas { // We need to modify the reference count. naming::gid_type& mutable_gid = const_cast(id).get_gid(); naming::gid_type const new_gid = - naming::detail::split_gid_if_needed(mutable_gid).get(); + naming::detail::split_gid_if_needed(hpx::launch::sync, mutable_gid); std::int64_t new_credit = naming::detail::get_credit_from_gid(new_gid); future f = symbol_ns_.bind_async(name, new_gid); diff --git a/libs/full/agas/src/detail/interface.cpp b/libs/full/agas/src/detail/interface.cpp index 727b8f970c85..e9e1cd63567e 100644 --- a/libs/full/agas/src/detail/interface.cpp +++ b/libs/full/agas/src/detail/interface.cpp @@ -10,8 +10,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -154,14 +154,19 @@ namespace hpx::agas::detail::impl { } /////////////////////////////////////////////////////////////////////////// - hpx::future resolve_async(hpx::id_type const& id) + hpx::future_or_value resolve_async(hpx::id_type const& id) { return naming::get_agas_client().resolve_async(id); } naming::address resolve(hpx::id_type const& id, error_code& ec) { - return naming::get_agas_client().resolve_async(id).get(ec); + auto result = naming::get_agas_client().resolve_async(id); + if (result.has_value()) + { + return HPX_MOVE(result).get_value(); + } + return result.get_future().get(ec); } bool resolve_local( @@ -396,13 +401,12 @@ namespace hpx::agas::detail::impl { } /////////////////////////////////////////////////////////////////////////// - hpx::future incref_async(naming::gid_type const& gid, + hpx::future_or_value incref_async(naming::gid_type const& gid, std::int64_t credits, hpx::id_type const& keep_alive_) { HPX_ASSERT(!naming::detail::is_locked(gid)); naming::resolver_client& resolver = naming::get_agas_client(); - if (keep_alive_) return resolver.incref_async(gid, credits, keep_alive_); @@ -411,31 +415,21 @@ namespace hpx::agas::detail::impl { return resolver.incref_async(gid, credits, keep_alive); } - std::int64_t incref(naming::gid_type const& gid, std::int64_t credits, - hpx::id_type const& keep_alive_, error_code&) - { - HPX_ASSERT(!naming::detail::is_locked(gid)); - - naming::resolver_client& resolver = naming::get_agas_client(); - - if (keep_alive_) - { - return resolver.incref_async(gid, credits, keep_alive_).get(); - } - hpx::id_type const keep_alive = - hpx::id_type(gid, hpx::id_type::management_type::unmanaged); - return resolver.incref_async(gid, credits, keep_alive).get(); - } - /////////////////////////////////////////////////////////////////////////// - hpx::future get_colocation_id_async(hpx::id_type const& id) + hpx::future_or_value get_colocation_id_async( + hpx::id_type const& id) { return naming::get_agas_client().get_colocation_id_async(id); } hpx::id_type get_colocation_id(hpx::id_type const& id, error_code& ec) { - return get_colocation_id_async(id).get(ec); + auto result = get_colocation_id_async(id); + if (result.has_value()) + { + return HPX_MOVE(result).get_value(); + } + return result.get_future().get(ec); } /////////////////////////////////////////////////////////////////////////// @@ -616,7 +610,6 @@ namespace hpx::agas { detail::decref = &detail::impl::decref; detail::incref_async = &detail::impl::incref_async; - detail::incref = &detail::impl::incref; detail::get_colocation_id_async = &detail::impl::get_colocation_id_async; diff --git a/libs/full/agas_base/include/hpx/agas_base/primary_namespace.hpp b/libs/full/agas_base/include/hpx/agas_base/primary_namespace.hpp index 993ff5b4c7f9..1a2afc353c30 100644 --- a/libs/full/agas_base/include/hpx/agas_base/primary_namespace.hpp +++ b/libs/full/agas_base/include/hpx/agas_base/primary_namespace.hpp @@ -1,5 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // Copyright (c) 2016 Thomas Heller +// Copyright (c) 2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -11,8 +12,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -72,16 +73,16 @@ namespace hpx { namespace agas { #endif resolved_type resolve_gid(naming::gid_type const& id); - future resolve_full(naming::gid_type id); + hpx::future_or_value resolve_full(naming::gid_type id); - future colocate(naming::gid_type id); + hpx::future_or_value colocate(naming::gid_type id); naming::address unbind_gid( std::uint64_t count, naming::gid_type const& id); future unbind_gid_async( std::uint64_t count, naming::gid_type const& id); - future increment_credit(std::int64_t credits, + future_or_value increment_credit(std::int64_t credits, naming::gid_type lower, naming::gid_type upper); std::pair allocate( diff --git a/libs/full/agas_base/src/primary_namespace.cpp b/libs/full/agas_base/src/primary_namespace.cpp index 1793bb16dc73..3d5a3bfd7d89 100644 --- a/libs/full/agas_base/src/primary_namespace.cpp +++ b/libs/full/agas_base/src/primary_namespace.cpp @@ -1,5 +1,6 @@ //////////////////////////////////////////////////////////////////////////////// // Copyright (c) 2016 Thomas Heller +// Copyright (c) 2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -238,40 +240,43 @@ namespace hpx { namespace agas { return server_->resolve_gid(id); } - future primary_namespace::resolve_full( - naming::gid_type id) + hpx::future_or_value + primary_namespace::resolve_full(naming::gid_type id) { hpx::id_type dest = hpx::id_type( get_service_instance(id), hpx::id_type::management_type::unmanaged); if (naming::get_locality_id_from_id(dest) == agas::get_locality_id()) { - return hpx::make_ready_future(server_->resolve_gid(id)); + return server_->resolve_gid(id); } + #if !defined(HPX_COMPUTE_DEVICE_CODE) server::primary_namespace::resolve_gid_action action; return hpx::async(action, HPX_MOVE(dest), id); #else HPX_ASSERT(false); - return hpx::make_ready_future(primary_namespace::resolved_type{}); + return primary_namespace::resolved_type{}; #endif } - hpx::future primary_namespace::colocate(naming::gid_type id) + hpx::future_or_value primary_namespace::colocate( + naming::gid_type id) { hpx::id_type dest = hpx::id_type( get_service_instance(id), hpx::id_type::management_type::unmanaged); if (naming::get_locality_id_from_id(dest) == agas::get_locality_id()) { - return hpx::make_ready_future(server_->colocate(id)); + return server_->colocate(id); } + #if !defined(HPX_COMPUTE_DEVICE_CODE) server::primary_namespace::colocate_action action; return hpx::async(action, HPX_MOVE(dest), id); #else HPX_ASSERT(false); - return hpx::make_ready_future(hpx::invalid_id); + return hpx::invalid_id; #endif } @@ -316,7 +321,7 @@ namespace hpx { namespace agas { #endif } - future primary_namespace::increment_credit( + future_or_value primary_namespace::increment_credit( std::int64_t credits, naming::gid_type lower, naming::gid_type upper) { hpx::id_type dest = hpx::id_type(get_service_instance(lower), @@ -324,15 +329,15 @@ namespace hpx { namespace agas { if (naming::get_locality_id_from_id(dest) == agas::get_locality_id()) { - return hpx::make_ready_future( - server_->increment_credit(credits, lower, upper)); + return server_->increment_credit(credits, lower, upper); } + #if !defined(HPX_COMPUTE_DEVICE_CODE) server::primary_namespace::increment_credit_action action; return hpx::async(action, HPX_MOVE(dest), credits, lower, upper); #else HPX_ASSERT(false); - return hpx::make_ready_future(std::int64_t{}); + return std::int64_t(-1); #endif } diff --git a/libs/full/agas_base/src/server/symbol_namespace_server.cpp b/libs/full/agas_base/src/server/symbol_namespace_server.cpp index 3c0eedd171b2..a30561eea37a 100644 --- a/libs/full/agas_base/src/server/symbol_namespace_server.cpp +++ b/libs/full/agas_base/src/server/symbol_namespace_server.cpp @@ -91,7 +91,7 @@ namespace hpx::agas::server { naming::gid_type gid = gid_; - gid_table_type::iterator const it = gids_.find(key); + auto const it = gids_.find(key); if (auto const end = gids_.end(); it != end) { std::int64_t const credits = @@ -155,7 +155,7 @@ namespace hpx::agas::server { auto iter = first; while (iter != last) { - lcos.push_back((*iter).second); + lcos.push_back(iter->second); ++iter; } @@ -187,7 +187,8 @@ namespace hpx::agas::server { // split the credit as the receiving end will expect to keep // the object alive naming::gid_type new_gid = - naming::detail::split_gid_if_needed(*current_gid).get(); + naming::detail::split_gid_if_needed( + hpx::launch::sync, *current_gid); // trigger the lco set_lco_value(id, new_gid); @@ -216,7 +217,7 @@ namespace hpx::agas::server { std::unique_lock l(mutex_); - gid_table_type::iterator const it = gids_.find(key); + auto const it = gids_.find(key); if (auto const end = gids_.end(); it == end) { l.unlock(); @@ -233,8 +234,8 @@ namespace hpx::agas::server { l.unlock(); - naming::gid_type gid = - naming::detail::split_gid_if_needed(*current_gid).get(); + naming::gid_type gid = naming::detail::split_gid_if_needed( + hpx::launch::sync, *current_gid); LAGAS_(info).format("symbol_namespace::resolve, key({1}), " "stored_gid({2}), gid({3})", @@ -292,33 +293,33 @@ namespace hpx::agas::server { std::regex const rx(str_rx); std::unique_lock l(mutex_); - for (auto it = gids_.begin(); it != gids_.end(); ++it) + for (auto& gid : gids_) { - if (!std::regex_match(it->first, rx)) + if (!std::regex_match(gid.first, rx)) continue; // hold on to entry while map is unlocked - std::shared_ptr current_gid(it->second); + std::shared_ptr current_gid(gid.second); unlock_guard> ul(l); - found[it->first] = - naming::detail::split_gid_if_needed(*current_gid).get(); + found[gid.first] = naming::detail::split_gid_if_needed( + hpx::launch::sync, *current_gid); } } else { std::unique_lock l(mutex_); - for (auto it = gids_.begin(); it != gids_.end(); ++it) + for (auto& gid : gids_) { - if (!pattern.empty() && pattern != it->first) + if (!pattern.empty() && pattern != gid.first) continue; // hold on to entry while map is unlocked - std::shared_ptr current_gid(it->second); + std::shared_ptr current_gid(gid.second); unlock_guard> ul(l); - found[it->first] = - naming::detail::split_gid_if_needed(*current_gid).get(); + found[gid.first] = naming::detail::split_gid_if_needed( + hpx::launch::sync, *current_gid); } } @@ -351,7 +352,8 @@ namespace hpx::agas::server { unlock_guard> ul(l); naming::gid_type new_gid = - naming::detail::split_gid_if_needed(*current_gid).get(); + naming::detail::split_gid_if_needed( + hpx::launch::sync, *current_gid); // trigger the lco handled = true; diff --git a/libs/full/async_colocated/include/hpx/async_colocated/get_colocation_id.hpp b/libs/full/async_colocated/include/hpx/async_colocated/get_colocation_id.hpp index e48f1fe757ff..a434294d6154 100644 --- a/libs/full/async_colocated/include/hpx/async_colocated/get_colocation_id.hpp +++ b/libs/full/async_colocated/include/hpx/async_colocated/get_colocation_id.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2020 Hartmut Kaiser +// Copyright (c) 2007-2022 Hartmut Kaiser // Copyright (c) 2011 Bryce Lelbach // // SPDX-License-Identifier: BSL-1.0 @@ -9,8 +9,8 @@ #pragma once -#include #include +#include #include #include diff --git a/libs/full/async_colocated/src/get_colocation_id.cpp b/libs/full/async_colocated/src/get_colocation_id.cpp index 9a3c85dd87d1..fa6b48a1bffb 100644 --- a/libs/full/async_colocated/src/get_colocation_id.cpp +++ b/libs/full/async_colocated/src/get_colocation_id.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace hpx { @@ -20,8 +21,13 @@ namespace hpx { return agas::get_colocation_id(launch::sync, id, ec); } - future get_colocation_id(hpx::id_type const& id) + hpx::future get_colocation_id(hpx::id_type const& id) { - return agas::get_colocation_id(id); + auto result = agas::get_colocation_id(id); + if (result.has_value()) + { + return hpx::make_ready_future(HPX_MOVE(result).get_value()); + } + return HPX_MOVE(result).get_future(); } } // namespace hpx diff --git a/libs/full/async_distributed/include/hpx/async_distributed/put_parcel.hpp b/libs/full/async_distributed/include/hpx/async_distributed/put_parcel.hpp index 7486883b3f9c..a1e2a26ddb6c 100644 --- a/libs/full/async_distributed/include/hpx/async_distributed/put_parcel.hpp +++ b/libs/full/async_distributed/include/hpx/async_distributed/put_parcel.hpp @@ -1,4 +1,5 @@ // Copyright (c) 2016 Thomas Heller +// Copyright (c) 2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -10,6 +11,7 @@ #if defined(HPX_HAVE_NETWORKING) #include +#include #include #include #include @@ -126,19 +128,32 @@ namespace hpx::parcelset { } else { - future split_gid = + auto result = naming::detail::split_gid_if_needed(dest.get_gid()); - if (split_gid.is_ready()) + if (result.has_value()) { pp(detail::create_parcel::call_with_action( - split_gid.get(), HPX_MOVE(addr), HPX_MOVE(action))); + HPX_MOVE(result).get_value(), HPX_MOVE(addr), + HPX_MOVE(action))); } else { - split_gid.then(hpx::launch::sync, - put_parcel_cont{HPX_FORWARD(PutParcel, pp), - HPX_MOVE(dest), HPX_MOVE(addr), HPX_MOVE(action)}); + HPX_ASSERT(result.has_future()); + + auto&& split_gid = HPX_MOVE(result).get_future(); + if (split_gid.is_ready()) + { + pp(detail::create_parcel::call_with_action( + split_gid.get(), HPX_MOVE(addr), HPX_MOVE(action))); + } + else + { + split_gid.then(hpx::launch::sync, + put_parcel_cont{ + HPX_FORWARD(PutParcel, pp), HPX_MOVE(dest), + HPX_MOVE(addr), HPX_MOVE(action)}); + } } } } diff --git a/libs/full/components/include/hpx/components/client_base.hpp b/libs/full/components/include/hpx/components/client_base.hpp index 7dbca12b5c12..2869d547ecc7 100644 --- a/libs/full/components/include/hpx/components/client_base.hpp +++ b/libs/full/components/include/hpx/components/client_base.hpp @@ -218,7 +218,15 @@ struct HPX_EXPORT hpx::lcos::detail::future_data { } - ~future_data() noexcept override; + // The constructor for the base template is defaulted. Some compilers + // generate duplicate symbols for this is the destructor definition is not + // visible to it. We hide the implementation in tidy() instead. + ~future_data() noexcept override + { + tidy(); + } + + void tidy() const noexcept; [[nodiscard]] std::string const& get_registered_name() const noexcept; void set_registered_name(std::string name); diff --git a/libs/full/components/include/hpx/components/get_ptr.hpp b/libs/full/components/include/hpx/components/get_ptr.hpp index c39a6dee9b14..5c072fbc9c45 100644 --- a/libs/full/components/include/hpx/components/get_ptr.hpp +++ b/libs/full/components/include/hpx/components/get_ptr.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2007-2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -17,8 +17,8 @@ #include #include #include -#include #include +#include #include #include @@ -134,6 +134,61 @@ namespace hpx { return get_ptr_postproc( addr, id, false); } + + /////////////////////////////////////////////////////////////////////// + template + std::shared_ptr resolve_get_ptr_local(hpx::id_type const& id) + { + return std::shared_ptr( + get_lva::call( + reinterpret_cast(id.get_lsb())), + detail::get_ptr_no_unpin_deleter(id)); + } + + template + hpx::future_or_value> get_ptr( + hpx::id_type const& id) + { + hpx::error_code ec(hpx::throwmode::lightweight); + + // shortcut for local, non-migratable objects + naming::gid_type gid = id.get_gid(); + if (naming::refers_to_local_lva(gid) && + naming::get_locality_id_from_gid(gid) == + agas::get_locality_id(ec) && + !ec) + { + return resolve_get_ptr_local(id); + } + + if (ec) + { + return hpx::make_exceptional_future>( + hpx::detail::access_exception(ec)); + } + + auto result = agas::resolve_async(id); + if (result.has_value()) + { + try + { + return get_ptr_postproc( + HPX_MOVE(result).get_value(), id); + } + catch (...) + { + return hpx::make_exceptional_future< + std::shared_ptr>(std::current_exception()); + } + } + + return result.get_future().then(hpx::launch::sync, + [=](hpx::future&& f) + -> std::shared_ptr { + return get_ptr_postproc( + f.get(), id); + }); + } } // namespace detail /// \endcond @@ -164,12 +219,12 @@ namespace hpx { template hpx::future> get_ptr(hpx::id_type const& id) { - hpx::future f = agas::resolve(id); - return f.then(hpx::launch::sync, - [=](hpx::future f) -> std::shared_ptr { - return detail::get_ptr_postproc(f.get(), id); - }); + auto result = detail::get_ptr(id); + if (result.has_value()) + { + return hpx::make_ready_future(HPX_MOVE(result).get_value()); + } + return HPX_MOVE(result).get_future(); } /// \brief Returns a future referring to the pointer to the @@ -247,19 +302,12 @@ namespace hpx { std::shared_ptr get_ptr( launch::sync_policy, hpx::id_type const& id, error_code& ec = throws) { - // shortcut for local, non-migratable objects - naming::gid_type gid = id.get_gid(); - if (naming::refers_to_local_lva(gid) && - naming::get_locality_id_from_gid(gid) == agas::get_locality_id(ec)) + auto result = detail::get_ptr(id); + if (result.has_value()) { - return std::shared_ptr( - get_lva::call( - reinterpret_cast(gid.get_lsb())), - detail::get_ptr_no_unpin_deleter(id)); + return HPX_MOVE(result).get_value(); } - - hpx::future> ptr = get_ptr(id); - return ptr.get(ec); + return result.get_future().get(ec); } #endif diff --git a/libs/full/components/src/client_base.cpp b/libs/full/components/src/client_base.cpp index ffbef9cd9de9..347f15949095 100644 --- a/libs/full/components/src/client_base.cpp +++ b/libs/full/components/src/client_base.cpp @@ -43,7 +43,7 @@ namespace hpx::util { namespace hpx::lcos::detail { - future_data::~future_data() noexcept + void future_data::tidy() const noexcept { auto* registered_name = try_get_extra_data(); if (registered_name != nullptr && !registered_name->empty()) diff --git a/libs/full/components_base/include/hpx/components_base/agas_interface.hpp b/libs/full/components_base/include/hpx/components_base/agas_interface.hpp index 6349abe92c74..15162135a02f 100644 --- a/libs/full/components_base/include/hpx/components_base/agas_interface.hpp +++ b/libs/full/components_base/include/hpx/components_base/agas_interface.hpp @@ -165,6 +165,9 @@ namespace hpx::agas { } /////////////////////////////////////////////////////////////////////////// + HPX_EXPORT hpx::future_or_value resolve_async( + hpx::id_type const& id); + HPX_EXPORT hpx::future resolve(hpx::id_type const& id); HPX_EXPORT naming::address resolve( @@ -240,8 +243,9 @@ namespace hpx::agas { error_code& ec = throws); /////////////////////////////////////////////////////////////////////////// - HPX_EXPORT hpx::future incref(naming::gid_type const& gid, - std::int64_t credits, hpx::id_type const& keep_alive = hpx::invalid_id); + HPX_EXPORT hpx::future_or_value incref( + naming::gid_type const& gid, std::int64_t credits, + hpx::id_type const& keep_alive = hpx::invalid_id); HPX_EXPORT std::int64_t incref(launch::sync_policy, naming::gid_type const& gid, std::int64_t credits = 1, @@ -252,7 +256,7 @@ namespace hpx::agas { HPX_EXPORT std::int64_t replenish_credits(naming::gid_type& gid); /////////////////////////////////////////////////////////////////////////// - HPX_EXPORT hpx::future get_colocation_id( + HPX_EXPORT hpx::future_or_value get_colocation_id( hpx::id_type const& id); HPX_EXPORT hpx::id_type get_colocation_id( diff --git a/libs/full/components_base/include/hpx/components_base/detail/agas_interface_functions.hpp b/libs/full/components_base/include/hpx/components_base/detail/agas_interface_functions.hpp index ec6044dbf322..866f04115609 100644 --- a/libs/full/components_base/include/hpx/components_base/detail/agas_interface_functions.hpp +++ b/libs/full/components_base/include/hpx/components_base/detail/agas_interface_functions.hpp @@ -11,8 +11,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -115,7 +115,7 @@ namespace hpx::agas::detail { naming::gid_type const& gid); /////////////////////////////////////////////////////////////////////////// - extern HPX_EXPORT hpx::future (*resolve_async)( + extern HPX_EXPORT hpx::future_or_value (*resolve_async)( hpx::id_type const& id); extern HPX_EXPORT naming::address (*resolve)( @@ -184,18 +184,15 @@ namespace hpx::agas::detail { naming::gid_type const& id, std::int64_t credits, error_code& ec); /////////////////////////////////////////////////////////////////////////// - extern HPX_EXPORT hpx::future (*incref_async)( + extern HPX_EXPORT hpx::future_or_value (*incref_async)( naming::gid_type const& gid, std::int64_t credits, hpx::id_type const& keep_alive); - extern HPX_EXPORT std::int64_t (*incref)(naming::gid_type const& gid, - std::int64_t credits, hpx::id_type const& keep_alive, error_code& ec); - /////////////////////////////////////////////////////////////////////////// extern HPX_EXPORT std::int64_t (*replenish_credits)(naming::gid_type& gid); /////////////////////////////////////////////////////////////////////////// - extern HPX_EXPORT hpx::future (*get_colocation_id_async)( + extern HPX_EXPORT hpx::future_or_value (*get_colocation_id_async)( hpx::id_type const& id); extern HPX_EXPORT hpx::id_type (*get_colocation_id)( diff --git a/libs/full/components_base/src/agas_interface.cpp b/libs/full/components_base/src/agas_interface.cpp index 1f9562ab3dff..9f152c84888a 100644 --- a/libs/full/components_base/src/agas_interface.cpp +++ b/libs/full/components_base/src/agas_interface.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2011 Bryce Adelstein-Lelbach -// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2007-2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) #include +#include #include #include #include @@ -150,11 +151,21 @@ namespace hpx::agas { } /////////////////////////////////////////////////////////////////////////// - hpx::future resolve(hpx::id_type const& id) + hpx::future_or_value resolve_async(hpx::id_type const& id) { return detail::resolve_async(id); } + hpx::future resolve(hpx::id_type const& id) + { + auto result = detail::resolve_async(id); + if (result.has_value()) + { + return hpx::make_ready_future(HPX_MOVE(result).get_value()); + } + return HPX_MOVE(result).get_future(); + } + naming::address resolve( launch::sync_policy, hpx::id_type const& id, error_code& ec) { @@ -303,7 +314,7 @@ namespace hpx::agas { } /////////////////////////////////////////////////////////////////////////// - hpx::future incref(naming::gid_type const& gid, + hpx::future_or_value incref(naming::gid_type const& gid, std::int64_t credits, hpx::id_type const& keep_alive) { return detail::incref_async(gid, credits, keep_alive); @@ -312,7 +323,12 @@ namespace hpx::agas { std::int64_t incref(launch::sync_policy, naming::gid_type const& gid, std::int64_t credits, hpx::id_type const& keep_alive, error_code& ec) { - return detail::incref(gid, credits, keep_alive, ec); + auto result = detail::incref_async(gid, credits, keep_alive); + if (result.has_value()) + { + return HPX_MOVE(result).get_value(); + } + return result.get_future().get(ec); } /////////////////////////////////////////////////////////////////////////// @@ -322,7 +338,7 @@ namespace hpx::agas { } /////////////////////////////////////////////////////////////////////////// - hpx::future get_colocation_id(hpx::id_type const& id) + hpx::future_or_value get_colocation_id(hpx::id_type const& id) { return detail::get_colocation_id_async(id); } diff --git a/libs/full/components_base/src/detail/agas_interface_functions.cpp b/libs/full/components_base/src/detail/agas_interface_functions.cpp index 8aa6bb325079..2061138779aa 100644 --- a/libs/full/components_base/src/detail/agas_interface_functions.cpp +++ b/libs/full/components_base/src/detail/agas_interface_functions.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -109,7 +110,7 @@ namespace hpx::agas::detail { bool (*is_local_lva_encoded_address)(naming::gid_type const& gid) = nullptr; /////////////////////////////////////////////////////////////////////////// - hpx::future (*resolve_async)( + hpx::future_or_value (*resolve_async)( hpx::id_type const& id) = nullptr; naming::address (*resolve)( @@ -177,17 +178,15 @@ namespace hpx::agas::detail { error_code& ec) = nullptr; /////////////////////////////////////////////////////////////////////////// - hpx::future (*incref_async)(naming::gid_type const& gid, - std::int64_t credits, hpx::id_type const& keep_alive) = nullptr; - - std::int64_t (*incref)(naming::gid_type const& gid, std::int64_t credits, - hpx::id_type const& keep_alive, error_code& ec) = nullptr; + hpx::future_or_value (*incref_async)( + naming::gid_type const& gid, std::int64_t credits, + hpx::id_type const& keep_alive) = nullptr; /////////////////////////////////////////////////////////////////////////// std::int64_t (*replenish_credits)(naming::gid_type& gid) = nullptr; /////////////////////////////////////////////////////////////////////////// - hpx::future (*get_colocation_id_async)( + hpx::future_or_value (*get_colocation_id_async)( hpx::id_type const& id) = nullptr; hpx::id_type (*get_colocation_id)( diff --git a/libs/full/naming/include/hpx/naming/split_gid.hpp b/libs/full/naming/include/hpx/naming/split_gid.hpp index 6aabf9073b16..78b0423250a2 100644 --- a/libs/full/naming/include/hpx/naming/split_gid.hpp +++ b/libs/full/naming/include/hpx/naming/split_gid.hpp @@ -9,14 +9,20 @@ #pragma once #include -#include +#include +#include +#include #include #include namespace hpx::naming::detail { - HPX_EXPORT hpx::future split_gid_if_needed(gid_type& id); - HPX_EXPORT hpx::future split_gid_if_needed_locked( + HPX_EXPORT hpx::future_or_value split_gid_if_needed(gid_type& id); + + HPX_EXPORT gid_type split_gid_if_needed( + hpx::launch::sync_policy, gid_type& id); + + HPX_EXPORT hpx::future_or_value split_gid_if_needed_locked( std::unique_lock& l, gid_type& gid); } // namespace hpx::naming::detail diff --git a/libs/full/naming/src/credit_handling.cpp b/libs/full/naming/src/credit_handling.cpp index 230faac199e8..bbea22c11dc0 100644 --- a/libs/full/naming/src/credit_handling.cpp +++ b/libs/full/naming/src/credit_handling.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -178,12 +179,25 @@ namespace hpx::naming { } /////////////////////////////////////////////////////////////////////// - hpx::future split_gid_if_needed(gid_type& gid) + hpx::future_or_value split_gid_if_needed(gid_type& gid) { std::unique_lock l(gid.get_mutex()); return split_gid_if_needed_locked(l, gid); } + gid_type split_gid_if_needed(hpx::launch::sync_policy, gid_type& gid) + { + auto result = split_gid_if_needed(gid); + if (result.has_value()) + { + return result.get_value(); + } + + HPX_ASSERT(result.has_future()); + return result.get_future().get(); + } + + /////////////////////////////////////////////////////////////////////// gid_type postprocess_incref(gid_type& gid) { std::unique_lock l(gid.get_mutex()); @@ -234,7 +248,7 @@ namespace hpx::naming { return new_gid; } - hpx::future split_gid_if_needed_locked( + hpx::future_or_value split_gid_if_needed_locked( std::unique_lock& l, gid_type& gid) { HPX_ASSERT_OWNS_LOCK(l); @@ -286,9 +300,26 @@ namespace hpx::naming { naming::gid_type const new_gid = gid; // strips lock-bit HPX_ASSERT(new_gid != invalid_gid); - return agas::incref(new_gid, new_credit) - .then(hpx::launch::sync, - hpx::bind(postprocess_incref, std::ref(gid))); + + auto result = agas::incref(new_gid, new_credit); + if (result.has_value()) + { + try + { + return postprocess_incref(gid); + } + catch (...) + { + return hpx::make_exceptional_future( + std::current_exception()); + } + } + + return result.get_future().then( + hpx::launch::sync, [&gid](auto&& f) { + f.get(); + return postprocess_incref(gid); + }); } HPX_ASSERT(src_log2credits > 1); @@ -298,11 +329,11 @@ namespace hpx::naming { HPX_ASSERT(detail::has_credits(gid)); HPX_ASSERT(detail::has_credits(new_gid)); - return hpx::make_ready_future(new_gid); + return new_gid; } naming::gid_type new_gid = gid; // strips lock-bit - return hpx::make_ready_future(new_gid); + return new_gid; } /////////////////////////////////////////////////////////////////////// @@ -479,17 +510,26 @@ namespace hpx::naming { // preprocessing handle_futures.increment_future_count(); - auto f = split_gid_if_needed(gid).then(hpx::launch::sync, - [&split_gids, &handle_futures, &gid]( - hpx::future&& gid_future) { - HPX_UNUSED(handle_futures); - HPX_ASSERT(handle_futures.has_futures()); - split_gids.add_gid(gid, gid_future.get()); - }); - - handle_futures.await_future( - *traits::future_access::get_shared_state(f), - false); + auto result = split_gid_if_needed(gid); + if (result.has_value()) + { + handle_futures.decrement_future_count(); + split_gids.add_gid(gid, result.get_value()); + } + else + { + auto f = result.get_future().then(hpx::launch::sync, + [&split_gids, &handle_futures, &gid]( + hpx::future&& gid_future) { + HPX_UNUSED(handle_futures); + HPX_ASSERT(handle_futures.has_futures()); + split_gids.add_gid(gid, gid_future.get()); + }); + + handle_futures.await_future( + *traits::future_access::get_shared_state(f), + false); + } } void preprocess_gid( diff --git a/libs/full/runtime_components/tests/unit/agas/local_address_rebind.cpp b/libs/full/runtime_components/tests/unit/agas/local_address_rebind.cpp index 3c4286b4d8c1..3d807fad0744 100644 --- a/libs/full/runtime_components/tests/unit/agas/local_address_rebind.cpp +++ b/libs/full/runtime_components/tests/unit/agas/local_address_rebind.cpp @@ -47,11 +47,11 @@ int hpx_main() std::uint64_t b_lva = b.get_lva(); // Resolve a_gid. - address addr = hpx::agas::resolve(a_id).get(); + address addr = hpx::agas::resolve(hpx::launch::sync, a_id); /////////////////////////////////////////////////////////////////////// HPX_TEST_EQ(addr.address_, reinterpret_cast(a.get_lva())); - HPX_SANITY_EQ(hpx::agas::resolve(a_id).get().address_, + HPX_SANITY_EQ(hpx::agas::resolve(hpx::launch::sync, a_id).address_, reinterpret_cast(a.get_lva())); /////////////////////////////////////////////////////////////////////// @@ -67,7 +67,7 @@ int hpx_main() /////////////////////////////////////////////////////////////////////// HPX_TEST_EQ(b_lva, a.get_lva()); - HPX_SANITY_EQ(hpx::agas::resolve(a_id).get().address_, + HPX_SANITY_EQ(hpx::agas::resolve(hpx::launch::sync, a_id).address_, reinterpret_cast(a.get_lva())); /////////////////////////////////////////////////////////////////////// @@ -81,8 +81,9 @@ int hpx_main() hpx::agas::update_cache_entry(a_gid, addr); /////////////////////////////////////////////////////////////////////// - HPX_TEST_EQ(hpx::agas::resolve(a_id).get().address_, a_lva); - HPX_SANITY_EQ(hpx::agas::resolve(a_id).get().address_, + HPX_TEST_EQ( + hpx::agas::resolve(hpx::launch::sync, a_id).address_, a_lva); + HPX_SANITY_EQ(hpx::agas::resolve(hpx::launch::sync, a_id).address_, reinterpret_cast(a.get_lva())); } diff --git a/libs/full/segmented_algorithms/tests/unit/partitioned_vector_inclusive_scan.cpp b/libs/full/segmented_algorithms/tests/unit/partitioned_vector_inclusive_scan.cpp index b896a740a160..16587be0207b 100644 --- a/libs/full/segmented_algorithms/tests/unit/partitioned_vector_inclusive_scan.cpp +++ b/libs/full/segmented_algorithms/tests/unit/partitioned_vector_inclusive_scan.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2016 Minh-Khanh Do +// Copyright (c) 2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -75,7 +76,7 @@ void inclusive_scan_algo_tests_with_policy(std::size_t size, template void inclusive_scan_algo_tests_segmented_out_with_policy(std::size_t size, DistPolicy const& in_dist_policy, DistPolicy const& out_dist_policy, - hpx::partitioned_vector& in, hpx::partitioned_vector out, + hpx::partitioned_vector& in, hpx::partitioned_vector& out, std::vector ver, ExPolicy const& policy) { msg9(typeid(ExPolicy).name(), typeid(DistPolicy).name(), typeid(T).name(), @@ -168,7 +169,7 @@ void inclusive_scan_algo_tests_with_policy_async(std::size_t size, template void inclusive_scan_algo_tests_segmented_out_with_policy_async(std::size_t size, DistPolicy const& in_dist_policy, DistPolicy const& out_dist_policy, - hpx::partitioned_vector& in, hpx::partitioned_vector out, + hpx::partitioned_vector& in, hpx::partitioned_vector& out, std::vector ver, ExPolicy const& policy) { msg9(typeid(ExPolicy).name(), typeid(DistPolicy).name(), typeid(T).name(), diff --git a/libs/full/segmented_algorithms/tests/unit/partitioned_vector_scan.hpp b/libs/full/segmented_algorithms/tests/unit/partitioned_vector_scan.hpp index e8f6164b1910..742c9bad33c2 100644 --- a/libs/full/segmented_algorithms/tests/unit/partitioned_vector_scan.hpp +++ b/libs/full/segmented_algorithms/tests/unit/partitioned_vector_scan.hpp @@ -1,4 +1,5 @@ // Copyright (c) 2016 Minh-Khanh Do +// Copyright (c) 2022 Hartmut Kaiser // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -87,7 +88,8 @@ struct verify_ : public hpx::parallel::detail::algorithm, Value> }; template -void verify_values(hpx::partitioned_vector v1, std::vector v2) +void verify_values( + hpx::partitioned_vector const& v1, std::vector const& v2) { auto first = v1.begin(); auto last = v1.end(); From 0884a80f8bdd7d0f198bfdf41b64055b45b4f418 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 29 Aug 2023 09:58:41 -0500 Subject: [PATCH 190/333] Fixing codacy badge --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fb46b5268338..a02bd0b404b2 100644 --- a/README.rst +++ b/README.rst @@ -151,8 +151,8 @@ Past and current funding and support for HPX is listed `here :target: https://doi.org/10.5281/zenodo.598202 :alt: Latest software release of HPX -.. |codacy| image:: https://api.codacy.com/project/badge/Grade/0b8cd5a874914edaba67ce3bb711e688 - :target: https://www.codacy.com/gh/STEllAR-GROUP/hpx +.. |codacy| .. image:: https://app.codacy.com/project/badge/Grade/0b8cd5a874914edaba67ce3bb711e688 + :target: https://app.codacy.com/gh/STEllAR-GROUP/hpx/dashboard :alt: HPX Code Quality Assessment .. |coveralls| image:: https://coveralls.io/repos/github/STEllAR-GROUP/hpx/badge.svg From 88911f124b4c35e233ff87ebb6450178b0211159 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 29 Aug 2023 09:59:39 -0500 Subject: [PATCH 191/333] More codacy badge changes --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a02bd0b404b2..928941eb7038 100644 --- a/README.rst +++ b/README.rst @@ -151,7 +151,7 @@ Past and current funding and support for HPX is listed `here :target: https://doi.org/10.5281/zenodo.598202 :alt: Latest software release of HPX -.. |codacy| .. image:: https://app.codacy.com/project/badge/Grade/0b8cd5a874914edaba67ce3bb711e688 +.. |codacy| image:: https://app.codacy.com/project/badge/Grade/0b8cd5a874914edaba67ce3bb711e688 :target: https://app.codacy.com/gh/STEllAR-GROUP/hpx/dashboard :alt: HPX Code Quality Assessment From fedf160e4c471107d26d3431bf676074bda4b4e4 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 25 Nov 2019 12:49:34 -0600 Subject: [PATCH 192/333] Adding local work stealing scheduler that is based on message passing internally - Using uniform_int_distribution with proper bounds - Removing queue index from thread_queues as it was unused - flyby: remove commented out options from .clang-format - Renaming workstealing --> workrequesting - Adding adaptive work stealing (steal half/steal one) - this makes this scheduler consistently (albeit only slightly) faster than the (default) local-priority scheduler - Adding LIFO and FIFO variations of local work-stealing scheduler - flyby: fixing HPX_WITH_SWAP_CONTEXT_EMULATION - flyby: minor changes to fibonacci_local example - Adding high- and low- priority queues - flyby: cache_line_data now does not generate warnings errors if padding is not needed - Adding bound queues - flyby: using cache_line_data for scheduler states --- .circleci/config.yml | 3 +- .jenkins/lsu/env-clang-13.sh | 2 + .jenkins/lsu/env-gcc-11.sh | 3 + CMakeLists.txt | 7 + cmake/HPX_AddTest.cmake | 5 + .../manual/hpx_runtime_and_resources.rst | 37 +- ...ching_and_configuring_hpx_applications.rst | 4 +- examples/quickstart/fibonacci_local.cpp | 25 +- .../src/parse_command_line_local.cpp | 13 +- .../resource_partitioner/partitioner_fwd.hpp | 2 + .../src/detail_partitioner.cpp | 18 + libs/core/schedulers/CMakeLists.txt | 1 + .../include/hpx/modules/schedulers.hpp | 3 +- .../local_workrequesting_scheduler.hpp | 1920 +++++++++++++++++ .../include/hpx/schedulers/thread_queue.hpp | 3 +- libs/core/testing/src/performance.cpp | 5 +- .../hpx/thread_pools/scheduling_loop.hpp | 12 +- .../src/scheduled_thread_pool.cpp | 12 + .../hpx/threading_base/thread_helpers.hpp | 3 - .../threading_base/src/scheduler_base.cpp | 3 +- .../threading_base/src/thread_helpers.cpp | 16 - .../regressions/thread_stacksize_current.cpp | 4 +- .../include/hpx/modules/threadmanager.hpp | 6 + libs/core/threadmanager/src/threadmanager.cpp | 89 + tools/perftests_ci/perftest/plot.py | 6 +- 25 files changed, 2146 insertions(+), 56 deletions(-) create mode 100644 libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp diff --git a/.circleci/config.yml b/.circleci/config.yml index 2298868cac25..eb498cbc59b3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -250,7 +250,8 @@ jobs: -DHPX_WITH_FETCH_JSON=On \ -DCMAKE_EXPORT_COMPILE_COMMANDS=On \ -DHPX_WITH_DOCUMENTATION=On \ - -DHPX_WITH_DOCUMENTATION_OUTPUT_FORMATS="${DOCUMENTATION_OUTPUT_FORMATS}" + -DHPX_WITH_DOCUMENTATION_OUTPUT_FORMATS="${DOCUMENTATION_OUTPUT_FORMATS}" \ + -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo - persist_to_workspace: root: /hpx paths: diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index e23c2ccd28bb..9b00cbd75a03 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" # configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" +configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" + # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-11.sh b/.jenkins/lsu/env-gcc-11.sh index b9f7be125f7f..882cc60eb73d 100644 --- a/.jenkins/lsu/env-gcc-11.sh +++ b/.jenkins/lsu/env-gcc-11.sh @@ -15,6 +15,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" +configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" @@ -25,3 +26,5 @@ configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=STD_EXPERIMENTAL_SIMD" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" + +configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-lifo" diff --git a/CMakeLists.txt b/CMakeLists.txt index dff5c025e0ab..43edac1a68d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1423,6 +1423,13 @@ hpx_option( ADVANCED ) +hpx_option( + HPX_WITH_TESTS_COMMAND_LINE STRING + "Add given command line options to all tests run" "" + CATEGORY "Debugging" + ADVANCED +) + hpx_option( HPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY STRING diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 34793a49febd..abacd140f062 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -68,6 +68,11 @@ function(add_hpx_test category name) ) endif() + # add additional command line arguments to all test executions + if(NOT x${HPX_WITH_TESTS_COMMAND_LINE} STREQUAL x"") + set(args ${args} "${HPX_WITH_TESTS_COMMAND_LINE}") + endif() + if(${HPX_WITH_PARALLEL_TESTS_BIND_NONE} AND NOT run_serial AND NOT "${name}_RUNWRAPPER" diff --git a/docs/sphinx/manual/hpx_runtime_and_resources.rst b/docs/sphinx/manual/hpx_runtime_and_resources.rst index 667c7ea7f535..195427972ed9 100644 --- a/docs/sphinx/manual/hpx_runtime_and_resources.rst +++ b/docs/sphinx/manual/hpx_runtime_and_resources.rst @@ -16,13 +16,13 @@ |hpx| thread scheduling policies ================================ -The |hpx| runtime has five thread scheduling policies: local-priority, -static-priority, local, static and abp-priority. These policies can be specified -from the command line using the command line option :option:`--hpx:queuing`. In -order to use a particular scheduling policy, the runtime system must be built -with the appropriate scheduler flag turned on (e.g. ``cmake --DHPX_THREAD_SCHEDULERS=local``, see :ref:`cmake_variables` for more -information). +The |hpx| runtime has six thread scheduling policies: local-priority, +static-priority, local, static, local-workrequesting-fifo, and abp-priority. +These policies can be specified from the command line using the command line +option :option:`--hpx:queuing`. In order to use a particular scheduling policy, +the runtime system must be built with the appropriate scheduler flag turned on +(e.g. ``cmake -DHPX_THREAD_SCHEDULERS=local``, see :ref:`cmake_variables` for +more information). Priority local scheduling policy (default policy) ------------------------------------------------- @@ -51,9 +51,7 @@ policy and must be invoked using the command line option Static priority scheduling policy --------------------------------- -* invoke using: :option:`--hpx:queuing`\ ``=static-priority`` (or ``-qs``) -* flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or - ``HPX_THREAD_SCHEDULERS=static-priority`` +* invoke using: :option:`--hpx:queuing`\ ``static-priority`` (or ``-qs``) The static scheduling policy maintains one queue per OS thread from which each OS thread pulls its tasks (user threads). Threads are distributed in a round @@ -102,7 +100,7 @@ domain first, only after that work is stolen from other NUMA domains. This scheduler can be used with two underlying queuing policies (FIFO: first-in-first-out, and LIFO: last-in-first-out). In order to use the LIFO policy use the command line option -:option:`--hpx:queuing`\ ``=abp-priority-lifo``. +:option:`--hpx:queuing`\ ``abp-priority-lifo``. .. Questions, concerns and notes: @@ -151,6 +149,23 @@ policy use the command line option I see both FIFO and double ended queues in ABP policies? +Work requesting scheduling policies +----------------------------------- + +* invoke using: :option:`--hpx:queuing`\ ``local-workrequesting-fifo`` + or using :option:`--hpx:queuing`\ ``local-workrequesting-lifo`` + +The work-requesting policies rely on a different mechanism of balancing work +between cores (compared to the other policies listed above). Instead of actively +trying to steal work from other cores, requesting work relies on a less +disruptive mechanism. If a core runs out of work, instead of actively looking at +the queues of neighboring cores, in this case a request is posted to another +core. This core now (whenever it is not busy with other work) either responds to +the original core by sending back work or passes the request on to the next +possible core in the system. In general, this scheme avoids contention on the +work queues as those are always accessed by their own cores only. + + The |hpx| resource partitioner ============================== diff --git a/docs/sphinx/manual/launching_and_configuring_hpx_applications.rst b/docs/sphinx/manual/launching_and_configuring_hpx_applications.rst index bc08a401b736..2b92572f4e10 100644 --- a/docs/sphinx/manual/launching_and_configuring_hpx_applications.rst +++ b/docs/sphinx/manual/launching_and_configuring_hpx_applications.rst @@ -1571,7 +1571,9 @@ The predefined command line options for any application using The queue scheduling policy to use. Options are ``local``, ``local-priority-fifo``, ``local-priority-lifo``, ``static``, - ``static-priority``, ``abp-priority-fifo`` and ``abp-priority-lifo`` + ``static-priority``, ``abp-priority-fifo``, + ``local-workrequesting-fifo``, ``local-workrequesting-lifo`` + and ``abp-priority-lifo`` (default: ``local-priority-fifo``). .. option:: --hpx:high-priority-threads arg diff --git a/examples/quickstart/fibonacci_local.cpp b/examples/quickstart/fibonacci_local.cpp index 910a6f5285ab..2b1da660eed5 100644 --- a/examples/quickstart/fibonacci_local.cpp +++ b/examples/quickstart/fibonacci_local.cpp @@ -18,6 +18,9 @@ #include #include +#include +#include +#include /////////////////////////////////////////////////////////////////////////////// //[fibonacci @@ -26,15 +29,10 @@ std::uint64_t fibonacci(std::uint64_t n) if (n < 2) return n; - // Invoking the Fibonacci algorithm twice is inefficient. - // However, we intentionally demonstrate it this way to create some - // heavy workload. - hpx::future n1 = hpx::async(fibonacci, n - 1); - hpx::future n2 = hpx::async(fibonacci, n - 2); + std::uint64_t n2 = fibonacci(n - 2); - return n1.get() + - n2.get(); // wait for the Futures to return their values + return n1.get() + n2; // wait for the Future to return their values } //fibonacci] @@ -42,6 +40,9 @@ std::uint64_t fibonacci(std::uint64_t n) //[hpx_main int hpx_main(hpx::program_options::variables_map& vm) { + hpx::threads::add_scheduler_mode( + hpx::threads::policies::scheduler_mode::fast_idle_mode); + // extract command line argument, i.e. fib(N) std::uint64_t n = vm["n-value"].as(); @@ -67,9 +68,13 @@ int main(int argc, char* argv[]) hpx::program_options::options_description desc_commandline( "Usage: " HPX_APPLICATION_STRING " [options]"); - desc_commandline.add_options()("n-value", - hpx::program_options::value()->default_value(10), - "n value for the Fibonacci function"); + // clang-format off + desc_commandline.add_options() + ("n-value", + hpx::program_options::value()->default_value(10), + "n value for the Fibonacci function") + ; + // clang-format on // Initialize and run HPX hpx::local::init_params init_args; diff --git a/libs/core/command_line_handling_local/src/parse_command_line_local.cpp b/libs/core/command_line_handling_local/src/parse_command_line_local.cpp index 646d805b19b3..6352b81c22e0 100644 --- a/libs/core/command_line_handling_local/src/parse_command_line_local.cpp +++ b/libs/core/command_line_handling_local/src/parse_command_line_local.cpp @@ -478,18 +478,24 @@ namespace hpx::local::detail { "run on (default: 0), valid for " "--hpx:queuing=local, --hpx:queuing=abp-priority, " "--hpx:queuing=static, --hpx:queuing=static-priority, " + "--hpx:queuing=local-workrequesting-fifo, " + "--hpx:queuing=local-workrequesting-lifo, " "and --hpx:queuing=local-priority only") ("hpx:pu-step", value(), "the step between used processing unit numbers for this " "instance of HPX (default: 1), valid for " "--hpx:queuing=local, --hpx:queuing=abp-priority, " "--hpx:queuing=static, --hpx:queuing=static-priority " + "--hpx:queuing=local-workrequesting-fifo, " + "--hpx:queuing=local-workrequesting-lifo, " "and --hpx:queuing=local-priority only") ("hpx:affinity", value(), "the affinity domain the OS threads will be confined to, " "possible values: pu, core, numa, machine (default: pu), valid for " "--hpx:queuing=local, --hpx:queuing=abp-priority, " "--hpx:queuing=static, --hpx:queuing=static-priority " + "--hpx:queuing=local-workrequesting-fifo, " + "--hpx:queuing=local-workrequesting-lifo, " " and --hpx:queuing=local-priority only") ("hpx:bind", value >()->composing(), "the detailed affinity description for the OS threads, see " @@ -514,13 +520,16 @@ namespace hpx::local::detail { ("hpx:queuing", value(), "the queue scheduling policy to use, options are " "'local', 'local-priority-fifo','local-priority-lifo', " - "'abp-priority-fifo', 'abp-priority-lifo', 'static', and " - "'static-priority' (default: 'local-priority'; " + "'abp-priority-fifo', 'abp-priority-lifo', 'static', " + "'static-priority', 'local-workrequesting-fifo', and " + "'local-workrequesting-lifo' (default: 'local-priority'; " "all option values can be abbreviated)") ("hpx:high-priority-threads", value(), "the number of operating system threads maintaining a high " "priority queue (default: number of OS threads), valid for " "--hpx:queuing=local-priority,--hpx:queuing=static-priority, " + "--hpx:queuing=local-workrequesting-fifo, " + "--hpx:queuing=local-workrequesting-lifo, " " and --hpx:queuing=abp-priority only)") ("hpx:numa-sensitive", value()->implicit_value(0), "makes the local-priority scheduler NUMA sensitive (" diff --git a/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner_fwd.hpp b/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner_fwd.hpp index 5a3df6be1c4a..379879be4b01 100644 --- a/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner_fwd.hpp +++ b/libs/core/resource_partitioner/include/hpx/resource_partitioner/partitioner_fwd.hpp @@ -105,6 +105,8 @@ namespace hpx::resource { abp_priority_fifo = 5, abp_priority_lifo = 6, shared_priority = 7, + local_workrequesting_fifo = 8, + local_workrequesting_lifo = 9, }; #define HPX_SCHEDULING_POLICY_UNSCOPED_ENUM_DEPRECATION_MSG \ diff --git a/libs/core/resource_partitioner/src/detail_partitioner.cpp b/libs/core/resource_partitioner/src/detail_partitioner.cpp index 68fb708814e7..8dad2d6379ce 100644 --- a/libs/core/resource_partitioner/src/detail_partitioner.cpp +++ b/libs/core/resource_partitioner/src/detail_partitioner.cpp @@ -142,6 +142,12 @@ namespace hpx::resource::detail { case resource::scheduling_policy::local_priority_lifo: sched = "local_priority_lifo"; break; + case resource::scheduling_policy::local_workrequesting_fifo: + sched = "local_workrequesting_fifo"; + break; + case resource::scheduling_policy::local_workrequesting_lifo: + sched = "local_workrequesting_lifo"; + break; case resource::scheduling_policy::static_: sched = "static"; break; @@ -475,6 +481,18 @@ namespace hpx::resource::detail { { default_scheduler = scheduling_policy::local_priority_lifo; } + else if (0 == + std::string("local-workrequesting-fifo") + .find(default_scheduler_str)) + { + default_scheduler = scheduling_policy::local_workrequesting_fifo; + } + else if (0 == + std::string("local-workrequesting-lifo") + .find(default_scheduler_str)) + { + default_scheduler = scheduling_policy::local_workrequesting_lifo; + } else if (0 == std::string("static").find(default_scheduler_str)) { default_scheduler = scheduling_policy::static_; diff --git a/libs/core/schedulers/CMakeLists.txt b/libs/core/schedulers/CMakeLists.txt index 70b038c539a3..0476a76a8893 100644 --- a/libs/core/schedulers/CMakeLists.txt +++ b/libs/core/schedulers/CMakeLists.txt @@ -11,6 +11,7 @@ set(schedulers_headers hpx/schedulers/deadlock_detection.hpp hpx/schedulers/local_priority_queue_scheduler.hpp hpx/schedulers/local_queue_scheduler.hpp + hpx/schedulers/local_workrequesting_scheduler.hpp hpx/schedulers/lockfree_queue_backends.hpp hpx/schedulers/maintain_queue_wait_times.hpp hpx/schedulers/queue_helpers.hpp diff --git a/libs/core/schedulers/include/hpx/modules/schedulers.hpp b/libs/core/schedulers/include/hpx/modules/schedulers.hpp index d77ba076c7e9..d6790d352d73 100644 --- a/libs/core/schedulers/include/hpx/modules/schedulers.hpp +++ b/libs/core/schedulers/include/hpx/modules/schedulers.hpp @@ -6,11 +6,10 @@ #pragma once -#include - #include #include #include +#include #include #include #include diff --git a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp new file mode 100644 index 000000000000..bb025b80e987 --- /dev/null +++ b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp @@ -0,0 +1,1920 @@ +// Copyright (c) 2007-2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/////////////////////////////////////////////////////////////////////////////// +// The scheduler implemented here is adapted from the ideas described in chapter +// 3 of Andreas Prell's PhD thesis 'Embracing Explicit Communication in +// Work-Stealing Runtime Systems' (see: https://epub.uni-bayreuth.de/2990/). +// While it's being described as a work-stealing scheduler, it relies on a +// different working principle if compared to the classic work-stealing. Instead +// of actively pulling work from the work queues of neighboring cores it relies +// on a push model. Cores that run out of work post steal requests that are +// handled by cores that have work awailable by actively sending tasks to the +// requesting core. +// +// When a worker runs out of tasks, it becomes a thief by sending steal requests +// to selected victim workers, those either reply with tasks or signal that they +// have no tasks left. A steal request is a message containing the thief's ID, a +// reference to a channel for sending tasks from victim to thief, and other +// information needed for thread coordination. +// +// When the runtime system starts up, every worker allocates two channels: a +// channel for receiving steal requests and a channel for receiving tasks. A +// reference to the latter is stored in steal requests, and workers use this +// reference to send tasks. By "owning" two channels, workers are able to +// receive steal requests and tasks independently of other workers, which in +// turn enables efficient channel implementations based on single-consumer +// queues. The total number of channels grows linearly with the number of +// workers: n workers allocate 2n channels to communicate with each other. +// +// Matching traditional work stealing, we allow one outstanding steal request +// per worker. This decision has two important consequences: (1) The number of +// steal requests is bounded by n, the number of workers. (2) A thief will never +// receive tasks from more than one victim at a time. It follows from (1) that a +// channel capacity of n - 1 is sufficient to deal with other workers' steal +// requests since no more than n - 1 thieves may request tasks from a single +// victim. We actually increase the capacity to n so that steal requests can be +// returned to their senders, for instance, in case of repeated failure. (2) +// implies that, at any given time, a task channel has at most one sender and +// one receiver, meeting the requirements for an SPSC implementation. +// +// In summary, every worker allocates two specialized channels: an MPSC channel +// where it receives steal requests and an SPSC channel where it receives tasks. +// Steal requests are forwarded rather than acknowledged, letting workers steal +// on behalf of others upon receiving steal requests that cannot be handled. +// Random victim selection fits in well with forwarding steal requests, but may +// cause a lot of communication if only few workers have tasks left. Stealing +// half of a victim's tasks (steal-half) is straightforward to implement with +// private task queues, especially when shared memory is available, in which +// case tasks do not need to be copied. While steal-half is important to tackle +// fine-grained parallelism, polling is necessary to achieve short message +// handling delays when workers schedule long-running tasks. + +namespace hpx::threads::policies { + + /////////////////////////////////////////////////////////////////////////// +#if defined(HPX_HAVE_CXX11_STD_ATOMIC_128BIT) + using default_local_workrequesting_scheduler_terminated_queue = + lockfree_lifo; +#else + using default_local_workrequesting_scheduler_terminated_queue = + lockfree_fifo; +#endif + + /////////////////////////////////////////////////////////////////////////// + // The local_workrequesting_scheduler maintains exactly one queue of work + // items (threads) per OS thread, where this OS thread pulls its next work + // from. + template + class local_workrequesting_scheduler : public scheduler_base + { + public: + using has_periodic_maintenance = std::false_type; + + using thread_queue_type = thread_queue; + + public: + struct init_parameter + { + init_parameter(std::size_t num_queues, + detail::affinity_data const& affinity_data, + std::size_t num_high_priority_queues = std::size_t(-1), + thread_queue_init_parameters const& thread_queue_init = + thread_queue_init_parameters{}, + char const* description = "local_workrequesting_scheduler") + : num_queues_(num_queues) + , num_high_priority_queues_( + num_high_priority_queues == std::size_t(-1) ? + num_queues : + num_high_priority_queues) + , thread_queue_init_(thread_queue_init) + , affinity_data_(affinity_data) + , description_(description) + { + } + + init_parameter(std::size_t num_queues, + detail::affinity_data const& affinity_data, + char const* description) + : num_queues_(num_queues) + , num_high_priority_queues_(num_queues) + , thread_queue_init_() + , affinity_data_(affinity_data) + , description_(description) + { + } + + std::size_t num_queues_; + std::size_t num_high_priority_queues_; + thread_queue_init_parameters thread_queue_init_; + detail::affinity_data const& affinity_data_; + char const* description_; + }; + using init_parameter_type = init_parameter; + + private: + //////////////////////////////////////////////////////////////////////// + struct task_data + { + explicit HPX_HOST_DEVICE_CONSTEXPR task_data( + std::uint16_t num_thread = std::uint16_t(-1)) noexcept + : num_thread_(num_thread) + { + } + + // core number this task data originated from + std::uint16_t num_thread_; + hpx::detail::small_vector tasks_; + }; + + //////////////////////////////////////////////////////////////////////// + using task_channel = lcos::local::channel_spsc; + + //////////////////////////////////////////////////////////////////////// + struct steal_request + { + enum class state : std::uint16_t + { + working = 0, + idle = 2, + failed = 4 + }; + + steal_request() noexcept + : victims_() + { + } + + steal_request(std::size_t num_thread, task_channel* channel, + mask_cref_type victims, bool idle, bool stealhalf) + : channel_(channel) + , victims_(victims) + , num_thread_(static_cast(num_thread)) + , attempt_(static_cast(count(victims) - 1)) + , state_(idle ? state::idle : state::working) + , stealhalf_(stealhalf) + { + } + + task_channel* channel_ = nullptr; + mask_type victims_; + std::uint16_t num_thread_ = static_cast(-1); + std::uint16_t attempt_ = 0; + state state_ = state::failed; + // true ? attempt steal-half : attempt steal-one + bool stealhalf_ = false; + }; + + //////////////////////////////////////////////////////////////////////// + using steal_request_channel = + lcos::local::base_channel_mpsc; + + //////////////////////////////////////////////////////////////////////// + struct scheduler_data + { + scheduler_data() noexcept + : victims_() + { + } + + ~scheduler_data() + { + delete queue_; + delete high_priority_queue_; + delete bound_queue_; + delete requests_; + delete tasks_; + } + + // interval at which we re-decide on whether we should steal just + // one task or half of what's available + static constexpr std::uint16_t num_steal_adaptive_interval_ = 25; + + void init(std::size_t num_thread, std::size_t size, + thread_queue_init_parameters const& queue_init, + bool need_high_priority_queue) + { + if (queue_ == nullptr) + { + num_thread_ = static_cast(num_thread); + + // initialize queues + queue_ = new thread_queue_type(queue_init); + if (need_high_priority_queue) + { + high_priority_queue_ = + new thread_queue_type(queue_init); + } + bound_queue_ = new thread_queue_type(queue_init); + + // initialize channels needed for work stealing + requests_ = new steal_request_channel(size); + tasks_ = new task_channel(1); + } + } + + // initial affinity mask for this core + mask_type victims_; + + // queues for threads scheduled on this core + thread_queue_type* queue_ = nullptr; + thread_queue_type* high_priority_queue_ = nullptr; + thread_queue_type* bound_queue_ = nullptr; + + // channel for posting steal requests to this core (use + // hpx::spinlock) + steal_request_channel* requests_ = nullptr; + + // one channel per steal request per core + task_channel* tasks_ = nullptr; + + // the number of outstanding steal requests + std::uint16_t requested_ = 0; + + // core number this scheduler data instance refers to + std::uint16_t num_thread_ = static_cast(-1); + + // adaptive stealing + std::uint16_t num_recent_steals_ = 0; + std::uint16_t num_recent_tasks_executed_ = 0; + bool stealhalf_ = false; + +#if defined(HPX_HAVE_WORKREQUESTING_LAST_VICTIM) + // core number the last stolen tasks originated from + std::uint16_t last_victim_ = static_cast(-1); +#endif +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + // collect some statistics + std::uint32_t steal_requests_sent_ = 0; + std::uint32_t steal_requests_received_ = 0; + std::uint32_t steal_requests_discarded_ = 0; +#endif + }; + + public: + static unsigned int random_seed() noexcept + { + static std::random_device rd; + return rd(); + } + + explicit local_workrequesting_scheduler(init_parameter_type const& init, + bool deferred_initialization = true) + : scheduler_base(init.num_queues_, init.description_, + init.thread_queue_init_, + policies::scheduler_mode::fast_idle_mode) + , data_(init.num_queues_) + , low_priority_queue_(thread_queue_init_) + , curr_queue_(0) + , gen_(random_seed()) + , affinity_data_(init.affinity_data_) + , num_queues_(init.num_queues_) + , num_high_priority_queues_(init.num_high_priority_queues_) + { + HPX_ASSERT(init.num_queues_ != 0); + HPX_ASSERT(num_high_priority_queues_ != 0); + HPX_ASSERT(num_high_priority_queues_ <= num_queues_); + + if (!deferred_initialization) + { + for (std::size_t i = 0; i != init.num_queues_; ++i) + { + data_[i].data_.init(i, init.num_queues_, + this->thread_queue_init_, + i < num_high_priority_queues_); + } + } + } + + ~local_workrequesting_scheduler() override = default; + + static std::string get_scheduler_name() + { + return "local_workrequesting_scheduler"; + } + +#ifdef HPX_HAVE_THREAD_CREATION_AND_CLEANUP_RATES + std::uint64_t get_creation_time(bool reset) override + { + std::uint64_t time = 0; + + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + time += d.high_priority_queue_->get_creation_time(reset); + } + time += d.queue_->get_creation_time(reset); + time += d.bound_queue_->get_creation_time(reset); + } + + return time + low_priority_queue_.get_creation_time(reset); + } + + std::uint64_t get_cleanup_time(bool reset) override + { + std::uint64_t time = 0; + + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + time += d.high_priority_queue_->get_cleanup_time(reset); + } + time += d.queue_->get_cleanup_time(reset); + time += d.bound_queue_->get_cleanup_time(reset); + } + + return time + low_priority_queue_.get_cleanup_time(reset); + } +#endif + +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + std::int64_t get_num_pending_misses( + std::size_t num_thread, bool reset) override + { + std::int64_t count = 0; + if (num_thread == std::size_t(-1)) + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += d.high_priority_queue_->get_num_pending_misses( + reset); + } + count += d.queue_->get_num_pending_misses(reset); + count += d.bound_queue_->get_num_pending_misses(reset); + } + + return count + + low_priority_queue_.get_num_pending_misses(reset); + } + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + count += d.high_priority_queue_->get_num_pending_misses(reset); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_num_pending_misses(reset); + } + count += d.queue_->get_num_pending_misses(reset); + return count + d.bound_queue_->get_num_pending_misses(reset); + } + + std::int64_t get_num_pending_accesses( + std::size_t num_thread, bool reset) override + { + std::int64_t count = 0; + if (num_thread == std::size_t(-1)) + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_pending_accesses( + reset); + } + count += d.queue_->get_num_pending_accesses(reset); + count += d.bound_queue_->get_num_pending_accesses(reset); + } + + return count + + low_priority_queue_.get_num_pending_accesses(reset); + } + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_pending_accesses(reset); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_num_pending_accesses(reset); + } + count += d.queue_->get_num_pending_accesses(reset); + return count + d.bound_queue_->get_num_pending_accesses(reset); + } + + std::int64_t get_num_stolen_from_pending( + std::size_t num_thread, bool reset) override + { + std::int64_t count = 0; + if (num_thread == std::size_t(-1)) + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_from_pending( + reset); + } + count += d.queue_->get_num_stolen_from_pending(reset); + count += d.bound_queue_->get_num_stolen_from_pending(reset); + } + + return count + + low_priority_queue_.get_num_stolen_from_pending(reset); + } + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_from_pending(reset); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_num_stolen_from_pending(reset); + } + count += d.queue_->get_num_stolen_from_pending(reset); + return count + d.bound_queue_->get_num_stolen_from_pending(reset); + } + + std::int64_t get_num_stolen_to_pending( + std::size_t num_thread, bool reset) override + { + std::int64_t count = 0; + if (num_thread == std::size_t(-1)) + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_to_pending( + reset); + } + count += d.queue_->get_num_stolen_to_pending(reset); + count += d.bound_queue_->get_num_stolen_to_pending(reset); + } + + return count + + low_priority_queue_.get_num_stolen_to_pending(reset); + } + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_to_pending(reset); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_num_stolen_to_pending(reset); + } + count += d.queue_->get_num_stolen_to_pending(reset); + return count + d.bound_queue_->get_num_stolen_to_pending(reset); + } + + std::int64_t get_num_stolen_from_staged( + std::size_t num_thread, bool reset) override + { + std::int64_t count = 0; + if (num_thread == std::size_t(-1)) + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_from_staged( + reset); + } + count += d.queue_->get_num_stolen_from_staged(reset); + count += d.bound_queue_->get_num_stolen_from_staged(reset); + } + + return count + + low_priority_queue_.get_num_stolen_from_staged(reset); + } + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_from_staged(reset); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_num_stolen_from_staged(reset); + } + count += d.queue_->get_num_stolen_from_staged(reset); + return count + d.bound_queue_->get_num_stolen_from_staged(reset); + } + + std::int64_t get_num_stolen_to_staged( + std::size_t num_thread, bool reset) override + { + std::int64_t count = 0; + if (num_thread == std::size_t(-1)) + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_to_staged( + reset); + } + count += d.queue_->get_num_stolen_to_staged(reset); + count += d.bound_queue_->get_num_stolen_to_staged(reset); + } + + return count + + low_priority_queue_.get_num_stolen_to_staged(reset); + } + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_num_stolen_to_staged(reset); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_num_stolen_to_staged(reset); + } + count += d.queue_->get_num_stolen_to_staged(reset); + return count + d.bound_queue_->get_num_stolen_to_staged(reset); + } +#endif + + /////////////////////////////////////////////////////////////////////// + void abort_all_suspended_threads() override + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + data_[i].data_.queue_->abort_all_suspended_threads(); + data_[i].data_.bound_queue_->abort_all_suspended_threads(); + } + } + + /////////////////////////////////////////////////////////////////////// + bool cleanup_terminated(bool delete_all) override + { + bool empty = true; + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + empty = d.high_priority_queue_->cleanup_terminated( + delete_all) && + empty; + } + empty = d.queue_->cleanup_terminated(delete_all) && empty; + empty = d.bound_queue_->cleanup_terminated(delete_all) && empty; + } + return low_priority_queue_.cleanup_terminated(delete_all) && empty; + } + + bool cleanup_terminated( + std::size_t num_thread, bool delete_all) override + { + auto& d = data_[num_thread].data_; + bool empty = d.queue_->cleanup_terminated(delete_all); + empty = d.queue_->cleanup_terminated(delete_all) && empty; + if (!delete_all) + return empty; + + if (num_thread < num_high_priority_queues_) + { + empty = + d.high_priority_queue_->cleanup_terminated(delete_all) && + empty; + } + + if (num_thread == num_queues_ - 1) + { + return low_priority_queue_.cleanup_terminated(delete_all) && + empty; + } + return empty; + } + + /////////////////////////////////////////////////////////////////////// + // create a new thread and schedule it if the initial state is equal to + // pending + void create_thread(thread_init_data& data, thread_id_ref_type* id, + error_code& ec) override + { + std::size_t num_thread = + data.schedulehint.mode == thread_schedule_hint_mode::thread ? + data.schedulehint.hint : + std::size_t(-1); + + if (std::size_t(-1) == num_thread) + { + num_thread = curr_queue_++ % num_queues_; + } + else if (num_thread >= num_queues_) + { + num_thread %= num_queues_; + } + + num_thread = select_active_pu(num_thread); + + data.schedulehint.mode = thread_schedule_hint_mode::thread; + data.schedulehint.hint = static_cast(num_thread); + + // now create the thread + switch (data.priority) + { + case thread_priority::high_recursive: + case thread_priority::high: + case thread_priority::boost: + { + if (data.priority == thread_priority::boost) + { + data.priority = thread_priority::normal; + } + + if (num_thread >= num_high_priority_queues_) + { + num_thread %= num_high_priority_queues_; + } + + // we never stage high priority threads, so there is no need to + // call wait_or_add_new for those. + data_[num_thread].data_.high_priority_queue_->create_thread( + data, id, ec); + break; + } + + case thread_priority::low: + low_priority_queue_.create_thread(data, id, ec); + break; + + case thread_priority::bound: + HPX_ASSERT(num_thread < num_queues_); + data_[num_thread].data_.bound_queue_->create_thread( + data, id, ec); + break; + + case thread_priority::default_: + case thread_priority::normal: + HPX_ASSERT(num_thread < num_queues_); + data_[num_thread].data_.queue_->create_thread(data, id, ec); + break; + + default: + case thread_priority::unknown: + { + HPX_THROW_EXCEPTION(hpx::error::bad_parameter, + "local_workrequesting_scheduler::create_thread", + "unknown thread priority value (thread_priority::unknown)"); + } + } + } + + // Retrieve the next viable steal request from our channel + bool try_receiving_steal_request( + scheduler_data& d, steal_request& req) noexcept + { + bool ret = d.requests_->get(&req); + while (ret && req.state_ == steal_request::state::failed) + { + // forget the received steal request + --data_[req.num_thread_].data_.requested_; + + // there should have been exactly one outstanding steal request + HPX_ASSERT(data_[req.num_thread_].data_.requested_ == 0); + + // try to retrieve next steal request + ret = d.requests_->get(&req); + } + + // No special treatment for other states + HPX_ASSERT( + (ret && req.state_ != steal_request::state::failed) || !ret); + + return ret; + } + + // Pass steal request on to another worker. Returns true if we have + // handled our own steal request. + bool decline_or_forward_steal_request( + scheduler_data& d, steal_request& req) noexcept + { + HPX_ASSERT(req.attempt_ < num_queues_); + + if (req.num_thread_ == d.num_thread_) + { + // Steal request was either returned by another worker or + // picked up by us. + + if (req.state_ == steal_request::state::idle || + d.queue_->get_pending_queue_length( + std::memory_order_relaxed) != 0) + { +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + ++d.steal_requests_discarded_; +#endif + // we have work now, drop this steal request + --d.requested_; + + // there should have been exactly one outstanding steal + // request + HPX_ASSERT(d.requested_ == 0); + } + else + { + // Continue circulating the steal request if it makes sense + req.state_ = steal_request::state::idle; + req.victims_ = d.victims_; + req.attempt_ = + static_cast(count(d.victims_) - 1); + + std::size_t victim = next_victim(d, req); + data_[victim].data_.requests_->set(HPX_MOVE(req)); +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + ++d.steal_requests_sent_; +#endif + } + + return true; + } + + // send this steal request on to the next (random) core + ++req.attempt_; + set(req.victims_, d.num_thread_); // don't ask a core twice + + HPX_ASSERT(req.attempt_ == count(req.victims_) - 1); + + std::size_t victim = next_victim(d, req); + data_[victim].data_.requests_->set(HPX_MOVE(req)); +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + ++d.steal_requests_sent_; +#endif + return false; + } + + // decline_or_forward_all_steal_requests is only called when a worker + // has nothing else to do but to relay steal requests, which means the + // worker is idle. + void decline_or_forward_all_steal_requests(scheduler_data& d) noexcept + { + steal_request req; + while (try_receiving_steal_request(d, req)) + { +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + ++d.steal_requests_received_; +#endif + decline_or_forward_steal_request(d, req); + } + } + + // Handle a steal request by sending tasks in return or passing it on to + // another worker. Returns true if the request was satisfied. + bool handle_steal_request( + scheduler_data& d, steal_request& req) noexcept + { +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + ++d.steal_requests_received_; +#endif + if (req.num_thread_ == d.num_thread_) + { + // got back our own steal request. + HPX_ASSERT(req.state_ != steal_request::state::failed); + + // Defer the decision to decline_steal_request + decline_or_forward_steal_request(d, req); + return false; + } + + // Send tasks from our queue to the requesting core, depending on + // what's requested, either one task or half of the available tasks + std::size_t max_num_to_steal = 1; + if (req.stealhalf_) + { + max_num_to_steal = d.queue_->get_pending_queue_length( + std::memory_order_relaxed) / + 2; + } + + if (max_num_to_steal != 0) + { + task_data thrds(d.num_thread_); + thrds.tasks_.reserve(max_num_to_steal); + + thread_id_ref_type thrd; + while (max_num_to_steal-- != 0 && + d.queue_->get_next_thread(thrd, false, true)) + { +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + d.queue_->increment_num_stolen_from_pending(); +#endif + thrds.tasks_.push_back(HPX_MOVE(thrd)); + } + + // we are ready to send at least one task + if (!thrds.tasks_.empty()) + { + // send these tasks to the core that has sent the steal + // request + req.channel_->set(HPX_MOVE(thrds)); + + // wake the thread up so that it can pick up the stolen + // tasks + do_some_work(req.num_thread_); + + return true; + } + } + + // There's nothing we can do with this steal request except pass + // it on to a different worker + decline_or_forward_steal_request(d, req); + return false; + } + + // Return the next thread to be executed, return false if none is + // available + bool get_next_thread(std::size_t num_thread, bool running, + thread_id_ref_type& thrd, bool enable_stealing) override + { + HPX_ASSERT(num_thread < num_queues_); + + auto& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + bool result = d.high_priority_queue_->get_next_thread(thrd); + +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + d.high_priority_queue_->increment_num_pending_accesses(); + if (result) + { + ++d.num_recent_tasks_executed_; + return true; + } + d.high_priority_queue_->increment_num_pending_misses(); +#else + if (result) + { + ++d.num_recent_tasks_executed_; + return true; + } +#endif + } + + bool result = false; + for (thread_queue_type* this_queue : {d.bound_queue_, d.queue_}) + { + result = this_queue->get_next_thread(thrd); + +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + this_queue->increment_num_pending_accesses(); +#endif + if (result) + break; +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + this_queue->increment_num_pending_misses(); +#endif + } + + if (enable_stealing && result) + { + // We found a task to run, however before running it we handle + // steal requests (assuming that there is more work left that + // could be used to satisfy steal requests). + + steal_request req; + while (try_receiving_steal_request(d, req)) + { + if (!handle_steal_request(d, req)) + break; + } + + ++d.num_recent_tasks_executed_; + return true; + } + + // Give up if we have work to convert. + if (d.queue_->get_staged_queue_length(std::memory_order_relaxed) != + 0 || + !running) + { + return false; + } + + if (low_priority_queue_.get_next_thread(thrd)) + { + ++d.num_recent_tasks_executed_; + return true; + } + + return false; + } + + // Schedule the passed thread + void schedule_thread(thread_id_ref_type thrd, + threads::thread_schedule_hint schedulehint, + bool allow_fallback = false, + thread_priority priority = thread_priority::normal) override + { + std::size_t num_thread = std::size_t(-1); + if (schedulehint.mode == thread_schedule_hint_mode::thread) + { + num_thread = schedulehint.hint; + } + else + { + allow_fallback = false; + } + + if (std::size_t(-1) == num_thread) + { + num_thread = curr_queue_++ % num_queues_; + } + else if (num_thread >= num_queues_) + { + num_thread %= num_queues_; + } + + num_thread = select_active_pu(num_thread, allow_fallback); + + HPX_ASSERT(get_thread_id_data(thrd)->get_scheduler_base() == this); + HPX_ASSERT(num_thread < num_queues_); + + switch (priority) + { + case thread_priority::high_recursive: + case thread_priority::high: + case thread_priority::boost: + { + std::size_t num = num_thread; + if (num >= num_high_priority_queues_) + { + num %= num_high_priority_queues_; + } + + data_[num].data_.high_priority_queue_->schedule_thread( + HPX_MOVE(thrd), true); + break; + } + + case thread_priority::low: + low_priority_queue_.schedule_thread(HPX_MOVE(thrd)); + break; + + case thread_priority::default_: + case thread_priority::normal: + data_[num_thread].data_.queue_->schedule_thread(HPX_MOVE(thrd)); + break; + + case thread_priority::bound: + data_[num_thread].data_.bound_queue_->schedule_thread( + HPX_MOVE(thrd)); + break; + + default: + case thread_priority::unknown: + { + HPX_THROW_EXCEPTION(hpx::error::bad_parameter, + "local_workrequesting_scheduler::schedule_thread", + "unknown thread priority value (thread_priority::unknown)"); + } + } + } + + void schedule_thread_last(thread_id_ref_type thrd, + threads::thread_schedule_hint schedulehint, + bool allow_fallback = false, + thread_priority priority = thread_priority::normal) override + { + std::size_t num_thread = std::size_t(-1); + if (schedulehint.mode == thread_schedule_hint_mode::thread) + { + num_thread = schedulehint.hint; + } + else + { + allow_fallback = false; + } + + if (std::size_t(-1) == num_thread) + { + num_thread = curr_queue_++ % num_queues_; + } + else if (num_thread >= num_queues_) + { + num_thread %= num_queues_; + } + + num_thread = select_active_pu(num_thread, allow_fallback); + + HPX_ASSERT(get_thread_id_data(thrd)->get_scheduler_base() == this); + HPX_ASSERT(num_thread < num_queues_); + + switch (priority) + { + case thread_priority::high_recursive: + case thread_priority::high: + case thread_priority::boost: + { + std::size_t num = num_thread; + if (num >= num_high_priority_queues_) + { + num %= num_high_priority_queues_; + } + + data_[num].data_.high_priority_queue_->schedule_thread( + HPX_MOVE(thrd), true); + break; + } + + case thread_priority::low: + low_priority_queue_.schedule_thread(HPX_MOVE(thrd), true); + break; + + default: + case thread_priority::default_: + case thread_priority::normal: + data_[num_thread].data_.queue_->schedule_thread( + HPX_MOVE(thrd), true); + break; + + case thread_priority::bound: + data_[num_thread].data_.bound_queue_->schedule_thread( + HPX_MOVE(thrd), true); + break; + } + } + + /// Destroy the passed thread as it has been terminated + void destroy_thread(threads::thread_data* thrd) override + { + HPX_ASSERT(thrd->get_scheduler_base() == this); + thrd->get_queue().destroy_thread(thrd); + } + + /////////////////////////////////////////////////////////////////////// + // This returns the current length of the queues (work items and new + // items) + std::int64_t get_queue_length( + std::size_t num_thread = std::size_t(-1)) const override + { + // Return queue length of one specific queue. + std::int64_t count = 0; + if (std::size_t(-1) != num_thread) + { + HPX_ASSERT(num_thread < num_queues_); + auto const& d = data_[num_thread].data_; + + if (num_thread < num_high_priority_queues_) + { + count += d.high_priority_queue_->get_queue_length(); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_queue_length(); + } + count += d.queue_->get_queue_length(); + return count + d.bound_queue_->get_queue_length(); + } + + // Cumulative queue lengths of all queues. + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += d.high_priority_queue_->get_queue_length(); + } + count += d.queue_->get_queue_length(); + count += d.bound_queue_->get_queue_length(); + } + return count + low_priority_queue_.get_queue_length(); + } + + /////////////////////////////////////////////////////////////////////// + // Queries the current thread count of the queues. + std::int64_t get_thread_count( + thread_schedule_state state = thread_schedule_state::unknown, + thread_priority priority = thread_priority::default_, + std::size_t num_thread = std::size_t(-1), + bool /* reset */ = false) const override + { + // Return thread count of one specific queue. + std::int64_t count = 0; + if (std::size_t(-1) != num_thread) + { + HPX_ASSERT(num_thread < num_queues_); + + auto const& d = data_[num_thread].data_; + switch (priority) + { + case thread_priority::default_: + { + if (num_thread < num_high_priority_queues_) + { + count = d.high_priority_queue_->get_thread_count(state); + } + if (num_thread == num_queues_ - 1) + { + count += low_priority_queue_.get_thread_count(state); + } + count += d.queue_->get_thread_count(state); + return count + d.bound_queue_->get_thread_count(state); + } + + case thread_priority::low: + { + if (num_queues_ - 1 == num_thread) + return low_priority_queue_.get_thread_count(state); + break; + } + + case thread_priority::normal: + return d.queue_->get_thread_count(state); + + case thread_priority::bound: + return d.bound_queue_->get_thread_count(state); + + case thread_priority::boost: + case thread_priority::high: + case thread_priority::high_recursive: + { + if (num_thread < num_high_priority_queues_) + { + return d.high_priority_queue_->get_thread_count(state); + } + break; + } + + default: + case thread_priority::unknown: + { + HPX_THROW_EXCEPTION(hpx::error::bad_parameter, + "local_workrequesting_scheduler::get_thread_count", + "unknown thread priority value " + "(thread_priority::unknown)"); + return 0; + } + } + return 0; + } + + // Return the cumulative count for all queues. + switch (priority) + { + case thread_priority::default_: + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[i].data_; + if (i < num_high_priority_queues_) + { + count += + d.high_priority_queue_->get_thread_count(state); + } + count += d.queue_->get_thread_count(state); + count += d.bound_queue_->get_thread_count(state); + } + count += low_priority_queue_.get_thread_count(state); + break; + } + + case thread_priority::low: + return low_priority_queue_.get_thread_count(state); + + case thread_priority::normal: + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + count += data_[i].data_.queue_->get_thread_count(state); + } + break; + } + + case thread_priority::bound: + { + for (std::size_t i = 0; i != num_queues_; ++i) + { + count += + data_[i].data_.bound_queue_->get_thread_count(state); + } + break; + } + + case thread_priority::boost: + case thread_priority::high: + case thread_priority::high_recursive: + { + for (std::size_t i = 0; i != num_high_priority_queues_; ++i) + { + count += + data_[i].data_.high_priority_queue_->get_thread_count( + state); + } + break; + } + + default: + case thread_priority::unknown: + { + HPX_THROW_EXCEPTION(hpx::error::bad_parameter, + "local_workrequesting_scheduler::get_thread_count", + "unknown thread priority value " + "(thread_priority::unknown)"); + return 0; + } + } + return count; + } + + // Queries whether a given core is idle + bool is_core_idle(std::size_t num_thread) const override + { + if (num_thread < num_queues_) + { + for (thread_queue_type* this_queue : + {data_[num_thread].data_.bound_queue_, + data_[num_thread].data_.queue_}) + { + if (this_queue->get_queue_length() != 0) + { + return false; + } + } + } + + if (num_thread < num_high_priority_queues_ && + data_[num_thread] + .data_.high_priority_queue_->get_queue_length() != 0) + { + return false; + } + return true; + } + + /////////////////////////////////////////////////////////////////////// + // Enumerate matching threads from all queues + bool enumerate_threads(hpx::function const& f, + thread_schedule_state state = + thread_schedule_state::unknown) const override + { + bool result = true; + for (std::size_t i = 0; i != num_high_priority_queues_; ++i) + { + result = result && + data_[i].data_.high_priority_queue_->enumerate_threads( + f, state); + } + + result = result && low_priority_queue_.enumerate_threads(f, state); + + for (std::size_t i = 0; i != num_queues_; ++i) + { + result = result && + data_[i].data_.queue_->enumerate_threads(f, state); + result = result && + data_[i].data_.bound_queue_->enumerate_threads(f, state); + } + return result; + } + +#ifdef HPX_HAVE_THREAD_QUEUE_WAITTIME + /////////////////////////////////////////////////////////////////////// + // Queries the current average thread wait time of the queues. + std::int64_t get_average_thread_wait_time( + std::size_t num_thread = std::size_t(-1)) const override + { + // Return average thread wait time of one specific queue. + std::uint64_t wait_time = 0; + std::uint64_t count = 0; + if (std::size_t(-1) != num_thread) + { + HPX_ASSERT(num_thread < num_queues_); + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + wait_time = + d.high_priority_queue_->get_average_thread_wait_time(); + ++count; + } + + if (num_thread == num_queues_ - 1) + { + wait_time += + low_priority_queue_.get_average_thread_wait_time(); + ++count; + } + + wait_time += d.queue_->get_average_thread_wait_time(); + wait_time += d.bound_queue_->get_average_thread_wait_time(); + return wait_time / (count + 1); + } + + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + wait_time += + d.high_priority_queue_->get_average_thread_wait_time(); + } + wait_time += d.queue_->get_average_thread_wait_time(); + wait_time += d.bound_queue_->get_average_thread_wait_time(); + ++count; + } + + return (wait_time + + low_priority_queue_.get_average_thread_wait_time()) / + (count + 1); + } + + /////////////////////////////////////////////////////////////////////// + // Queries the current average task wait time of the queues. + std::int64_t get_average_task_wait_time( + std::size_t num_thread = std::size_t(-1)) const override + { + // Return average task wait time of one specific queue. + // Return average thread wait time of one specific queue. + std::uint64_t wait_time = 0; + std::uint64_t count = 0; + if (std::size_t(-1) != num_thread) + { + HPX_ASSERT(num_thread < num_queues_); + + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + wait_time = + d.high_priority_queue_->get_average_task_wait_time(); + ++count; + } + + if (num_thread == num_queues_ - 1) + { + wait_time += + low_priority_queue_.get_average_task_wait_time(); + ++count; + } + + wait_time += d.queue_->get_average_task_wait_time(); + wait_time += d.bound_queue_->get_average_task_wait_time(); + return wait_time / (count + 1); + } + + for (std::size_t i = 0; i != num_queues_; ++i) + { + auto const& d = data_[num_thread].data_; + if (num_thread < num_high_priority_queues_) + { + wait_time += + d.high_priority_queue_->get_average_task_wait_time(); + } + wait_time += d.queue_->get_average_task_wait_time(); + wait_time += d.bound_queue_->get_average_task_wait_time(); + ++count; + } + + return (wait_time + + low_priority_queue_.get_average_task_wait_time()) / + (count + 1); + } +#endif + + // return a random victim for the current stealing operation + std::size_t random_victim(steal_request const& req) noexcept + { + std::size_t result = 0; + + { + // generate at most 3 random numbers before resorting to more + // expensive algorithm + std::uniform_int_distribution uniform( + 0, std::int16_t(num_queues_ - 1)); + + int attempts = 0; + do + { + result = uniform(gen_); + if (result != req.num_thread_ && + !test(req.victims_, result)) + { + HPX_ASSERT(result < num_queues_); + return result; + } + } while (++attempts < 3); + } + + // to avoid infinite trials we randomly select one of the possible + // victims + std::uniform_int_distribution uniform( + 0, std::int16_t(num_queues_ - count(req.victims_) - 1)); + + // generate one more random number + std::size_t selected_victim = uniform(gen_); + for (std::size_t i = 0; i != num_queues_; ++i) + { + if (!test(req.victims_, i)) + { + if (selected_victim == 0) + { + result = i; + break; + } + --selected_victim; + } + } + + HPX_ASSERT(result < num_queues_ && result != req.num_thread_ && + !test(req.victims_, result)); + + return result; + } + + // return the number of the next victim core + std::size_t next_victim([[maybe_unused]] scheduler_data& d, + steal_request const& req) noexcept + { + std::size_t victim = std::size_t(-1); + + // return thief if max steal attempts has been reached or no more + // cores are available for stealing + if (req.attempt_ == num_queues_ - 1) + { + // Return steal request to thief + victim = req.num_thread_; + } + else + { + HPX_ASSERT( + req.num_thread_ == d.num_thread_ || req.attempt_ != 0); + +#if defined(HPX_HAVE_WORKREQUESTING_LAST_VICTIM) + if (d.last_victim_ != std::uint16_t(-1)) + { + victim = d.last_victim_; + } + else +#endif + { + victim = random_victim(req); + } + } + + // couldn't find victim, return steal request to thief + if (victim == std::size_t(-1)) + { + victim = req.num_thread_; + HPX_ASSERT(victim != d.num_thread_); + } + + HPX_ASSERT(victim < num_queues_); + HPX_ASSERT(req.attempt_ < num_queues_); + + return victim; + } + + // Every worker can have at most MAXSTEAL pending steal requests. A + // steal request with idle == false indicates that the requesting worker + // is still busy working on some tasks. A steal request with idle == + // true indicates that the requesting worker is in fact idle and has + // nothing to work on. + void send_steal_request(scheduler_data& d, bool idle = true) noexcept + { + if (d.requested_ == 0) + { + // Estimate work-stealing efficiency during the last interval; + // switch strategies if the value is below a threshold + if (d.num_recent_steals_ >= + scheduler_data::num_steal_adaptive_interval_) + { + double ratio = + static_cast(d.num_recent_tasks_executed_) / + d.num_steal_adaptive_interval_; + + d.num_recent_steals_ = 0; + d.num_recent_tasks_executed_ = 0; + + if (ratio >= 2.) + { + d.stealhalf_ = true; + } + else + { + if (d.stealhalf_) + { + d.stealhalf_ = false; + } + else if (ratio <= 1.) + { + d.stealhalf_ = true; + } + } + } + + steal_request req( + d.num_thread_, d.tasks_, d.victims_, idle, d.stealhalf_); + std::size_t victim = next_victim(d, req); + + ++d.requested_; + data_[victim].data_.requests_->set(HPX_MOVE(req)); +#if defined(HPX_HAVE_WORKREQUESTING_STEAL_STATISTICS) + ++d.steal_requests_sent_; +#endif + } + } + + // Try receiving tasks that are sent by another core as a response to + // one of our steal requests. This returns true if new tasks were + // received. + bool try_receiving_tasks(scheduler_data& d, std::size_t& added, + thread_id_ref_type* next_thrd) + { + task_data thrds{}; + if (d.tasks_->get(&thrds)) + { + // keep track of number of outstanding steal requests, there + // should have been at most one + --d.requested_; + HPX_ASSERT(d.requested_ == 0); + + // if at least one thrd was received + if (!thrds.tasks_.empty()) + { + // Schedule all but the first received task in reverse order + // to maintain the sequence of tasks as pulled from the + // victims queue. + for (std::size_t i = thrds.tasks_.size() - 1; i != 0; --i) + { + HPX_ASSERT(thrds.tasks_[i]); + d.queue_->schedule_thread( + HPX_MOVE(thrds.tasks_[i]), true); + +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + d.queue_->increment_num_stolen_to_pending(); +#endif + ++added; + } + +#if defined(HPX_HAVE_WORKREQUESTING_LAST_VICTIM) + // store the originating core for the next stealing + // operation + d.last_victim_ = thrds.num_thread_; + HPX_ASSERT(d.last_victim_ != d.num_thread_); +#endif + // the last of the received tasks will be either directly + // executed or normally scheduled + if (next_thrd != nullptr) + { + // directly return the last thread as it should be run + // immediately + ++d.num_recent_tasks_executed_; + *next_thrd = HPX_MOVE(thrds.tasks_.front()); + } + else + { + d.queue_->schedule_thread( + HPX_MOVE(thrds.tasks_.front()), true); + +#ifdef HPX_HAVE_THREAD_STEALING_COUNTS + d.queue_->increment_num_stolen_to_pending(); +#endif + ++added; + } + + ++d.num_recent_steals_; + return true; + } + } + return false; + } + + // This is a function which gets called periodically by the thread + // manager to allow for maintenance tasks to be executed in the + // scheduler. Returns true if the OS thread calling this function has to + // be terminated (i.e. no more work has to be done). + bool wait_or_add_new(std::size_t num_thread, bool running, + [[maybe_unused]] std::int64_t& idle_loop_count, + bool enable_stealing, std::size_t& added, + thread_id_ref_type* next_thrd = nullptr) override + { + HPX_ASSERT(num_thread < num_queues_); + + added = 0; + + auto& d = data_[num_thread].data_; + + // We don't need to call wait_or_add_new for high priority or bound + // threads as these threads are never created 'staged'. + + bool result = + d.queue_->wait_or_add_new(running, added, enable_stealing); + + // check if work was available + if (0 != added) + return result; + + if (num_thread == num_queues_ - 1) + { + result = low_priority_queue_.wait_or_add_new(running, added) && + result; + } + + // check if we have been disabled + if (!running) + return true; + + // return if no stealing is requested (or not possible) + if (num_queues_ == 1 || !enable_stealing) + return result; + + // attempt to steal more work + send_steal_request(d); + HPX_ASSERT(d.requested_ != 0); + + // now try to handle steal requests again if we have not received a + // task from some other core yet + if (!try_receiving_tasks(d, added, next_thrd)) + { + // decline or forward all pending steal requests + decline_or_forward_all_steal_requests(d); + } + +#ifdef HPX_HAVE_THREAD_MINIMAL_DEADLOCK_DETECTION + // no new work is available, are we deadlocked? + if (HPX_UNLIKELY(get_minimal_deadlock_detection_enabled() && + LHPX_ENABLED(error))) + { + bool suspended_only = true; + + for (std::size_t i = 0; suspended_only && i != num_queues_; ++i) + { + suspended_only = + data_[i].data_.queue_->dump_suspended_threads( + i, idle_loop_count, running); + } + + if (HPX_UNLIKELY(suspended_only)) + { + if (running) + { + LTM_(error) //-V128 + << "queue(" << num_thread << "): " + << "no new work available, are we " + "deadlocked?"; + } + else + { + LHPX_CONSOLE_( + hpx::util::logging::level::error) //-V128 + << " [TM] " //-V128 + << "queue(" << num_thread << "): " + << "no new work available, are we " + "deadlocked?\n"; + } + } + } +#endif + return result; + } + + /////////////////////////////////////////////////////////////////////// + void on_start_thread(std::size_t num_thread) override + { + hpx::threads::detail::set_local_thread_num_tss(num_thread); + hpx::threads::detail::set_thread_pool_num_tss( + parent_pool_->get_pool_id().index()); + + auto& d = data_[num_thread].data_; + d.init(num_thread, num_queues_, this->thread_queue_init_, + num_thread < num_high_priority_queues_); + + d.queue_->on_start_thread(num_thread); + d.bound_queue_->on_start_thread(num_thread); + if (num_thread < num_high_priority_queues_) + { + d.high_priority_queue_->on_start_thread(num_thread); + } + + if (num_thread == num_queues_ - 1) + { + low_priority_queue_.on_start_thread(num_thread); + } + + std::size_t num_threads = num_queues_; + //auto const& topo = create_topology(); + + // Initially set all bits, code below resets the bits corresponding + // to cores that can serve as a vistim for the current core. A set + // bit in this mask means 'do not steal from this core'. + resize(d.victims_, num_threads); + reset(d.victims_); + set(d.victims_, num_thread); + //for (std::size_t i = 0; i != num_threads; ++i) + //{ + // set(d.victims_, i); + //} + // + //// get NUMA domain masks of all queues... + //std::vector numa_masks(num_threads); + //std::vector numa_domains(num_threads); + //std::vector core_masks(num_threads); + //for (std::size_t i = 0; i != num_threads; ++i) + //{ + // std::size_t num_pu = affinity_data_.get_pu_num(i); + // numa_masks[i] = topo.get_numa_node_affinity_mask(num_pu); + // numa_domains[i] = static_cast( + // topo.get_numa_node_number(num_pu)); + // core_masks[i] = topo.get_core_affinity_mask(num_pu); + //} + // + //// iterate over the number of threads again to determine where to + //// steal from + //std::ptrdiff_t radius = + // std::lround(static_cast(num_threads) / 2.0); + // + //mask_cref_type numa_mask = numa_masks[num_thread]; + //mask_cref_type core_mask = core_masks[num_thread]; + // + //auto iterate = [&](auto&& f) { + // // check our neighbors in a radial fashion (left and right + // // alternating, increasing distance each iteration) + // std::ptrdiff_t i = 1; + // for (/**/; i < radius; ++i) + // { + // std::ptrdiff_t left = + // (static_cast(num_thread) - i) % + // static_cast(num_threads); + // if (left < 0) + // left = num_threads + left; + // + // if (f(std::size_t(left))) + // { + // unset(data_[num_thread].data_.victims_, + // static_cast(left)); + // } + // + // std::size_t right = (num_thread + i) % num_threads; + // if (f(right)) + // { + // unset(data_[num_thread].data_.victims_, right); + // } + // } + // if ((num_threads % 2) == 0) + // { + // std::size_t right = (num_thread + i) % num_threads; + // if (f(right)) + // { + // unset(data_[num_thread].data_.victims_, right); + // } + // } + //}; + // + //// check for threads that share the same core... + //iterate([&](std::size_t other_num_thread) { + // return any(core_mask & core_masks[other_num_thread]); + //}); + // + //// check for threads that share the same NUMA domain... + //iterate([&](std::size_t other_num_thread) { + // return !any(core_mask & core_masks[other_num_thread]) && + // any(numa_mask & numa_masks[other_num_thread]); + //}); + // + //// check for the rest and if we are NUMA aware + //if (has_scheduler_mode( + // policies::scheduler_mode::enable_stealing_numa)) + //{ + // iterate([&](std::size_t other_num_thread) { + // // allow stealing from neighboring NUMA domain only + // std::ptrdiff_t numa_distance = numa_domains[num_thread] - + // numa_domains[other_num_thread]; + // if (numa_distance > 1 || numa_distance < -1) + // return false; + // // steal from even cores from neighboring NUMA domains + // if (numa_distance == 1 || numa_distance == -1) + // return other_num_thread % 2 == 0; + // // cores from our domain are handled above + // return false; + // }); + //} + } + + void on_stop_thread(std::size_t num_thread) override + { + auto& d = data_[num_thread].data_; + + d.queue_->on_stop_thread(num_thread); + d.bound_queue_->on_stop_thread(num_thread); + if (num_thread < num_high_priority_queues_) + { + d.high_priority_queue_->on_stop_thread(num_thread); + } + + if (num_thread == num_queues_ - 1) + { + low_priority_queue_.on_stop_thread(num_thread); + } + } + + void on_error( + std::size_t num_thread, std::exception_ptr const& e) override + { + auto& d = data_[num_thread].data_; + + d.queue_->on_error(num_thread, e); + d.bound_queue_->on_error(num_thread, e); + if (num_thread < num_high_priority_queues_) + { + d.high_priority_queue_->on_error(num_thread, e); + } + + if (num_thread == num_queues_ - 1) + { + low_priority_queue_.on_error(num_thread, e); + } + } + + void reset_thread_distribution() override + { + curr_queue_.store(0, std::memory_order_release); + } + + void set_scheduler_mode(scheduler_mode mode) noexcept override + { + // we should not disable stealing for this scheduler, this would + // possibly lead to deadlocks + scheduler_base::set_scheduler_mode(mode | + policies::scheduler_mode::enable_stealing | + policies::scheduler_mode::enable_stealing_numa); + } + + protected: + std::vector> data_; + thread_queue_type low_priority_queue_; + + std::atomic curr_queue_; + + std::mt19937 gen_; + + detail::affinity_data const& affinity_data_; + std::size_t const num_queues_; + std::size_t const num_high_priority_queues_; + }; +} // namespace hpx::threads::policies + +#include diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 4c8524bad734..25260cc85ca4 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -457,7 +457,8 @@ namespace hpx::threads::policies { if (delete_all) { - // do not lock mutex while deleting all threads, do it piece-wise + // do not lock mutex while deleting all threads, do it + // piece-wise while (true) { std::unique_lock lk(mtx_, std::try_to_lock); diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index b06b8b8c6cd1..5cd224ff1497 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -63,6 +63,7 @@ namespace hpx::util { strm << R"( "executor" : ")" << std::get<1>(item.first) << "\",\n"; strm << R"( "series" : [)"; + double average = 0.0; int series = 0; for (auto const val : item.second) { @@ -70,8 +71,10 @@ namespace hpx::util { strm << ", "; strm << val; ++series; + average += val; } - strm << "]\n"; + strm << "],\n"; + strm << " \"average\" : " << average / series << "\n"; strm << " }"; ++outputs; } diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp index a84afdb668dd..86c18a3ba2b4 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #if defined(HPX_HAVE_APEX) #include @@ -178,6 +179,7 @@ namespace hpx::threads::detail { idle_collect_rate idle_rate(counters.tfunc_time_, counters.exec_time_); [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector(idle_rate); + HPX_UNUSED(tfunc_time_collector); // spin for some time after queues have become empty bool may_exit = false; @@ -235,6 +237,8 @@ namespace hpx::threads::detail { scheduler.get_next_thread( num_thread, running, thrd, enable_stealing))) { + HPX_UNUSED(tfunc_time_collector); + HPX_ASSERT(get_thread_id_data(thrd)->get_scheduler_base() == &scheduler); @@ -269,6 +273,7 @@ namespace hpx::threads::detail { [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector_inner(idle_rate); + HPX_UNUSED(tfunc_time_collector); // thread returns new required state store the // returned state in the thread @@ -289,7 +294,7 @@ namespace hpx::threads::detail { // and add to aggregate execution time. [[maybe_unused]] exec_time_wrapper exec_time_collector(idle_rate); - + HPX_UNUSED(exec_time_collector); #if defined(HPX_HAVE_APEX) // get the APEX data pointer, in case we are // resuming the thread and have to restore any @@ -464,8 +469,9 @@ namespace hpx::threads::detail { { ++idle_loop_count; - if (scheduler.wait_or_add_new(num_thread, running, - idle_loop_count, enable_stealing_staged, added, + next_thrd = nullptr; + if (scheduler.wait_or_add_new(num_thread, + running, idle_loop_count, enable_stealing_staged, added, &next_thrd)) { // Clean up terminated threads before trying to exit diff --git a/libs/core/thread_pools/src/scheduled_thread_pool.cpp b/libs/core/thread_pools/src/scheduled_thread_pool.cpp index cc2c684cae9f..308d8748f586 100644 --- a/libs/core/thread_pools/src/scheduled_thread_pool.cpp +++ b/libs/core/thread_pools/src/scheduled_thread_pool.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -66,3 +67,14 @@ template class HPX_CORE_EXPORT hpx::threads::policies::shared_priority_queue_scheduler<>; template class HPX_CORE_EXPORT hpx::threads::detail::scheduled_thread_pool< hpx::threads::policies::shared_priority_queue_scheduler<>>; + +template class HPX_CORE_EXPORT + hpx::threads::policies::local_workrequesting_scheduler<>; +template class HPX_CORE_EXPORT hpx::threads::detail::scheduled_thread_pool< + hpx::threads::policies::local_workrequesting_scheduler<>>; +template class HPX_CORE_EXPORT + hpx::threads::policies::local_workrequesting_scheduler; +template class HPX_CORE_EXPORT hpx::threads::detail::scheduled_thread_pool< + hpx::threads::policies::local_workrequesting_scheduler>; diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp index 131369ccdc6c..70f5ad7c5685 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp @@ -387,9 +387,6 @@ namespace hpx::threads { HPX_CORE_EXPORT bool add_thread_exit_callback(thread_id_type const& id, hpx::function const& f, error_code& ec = throws); - HPX_CORE_EXPORT void free_thread_exit_callbacks( - thread_id_type const& id, error_code& ec = throws); - /////////////////////////////////////////////////////////////////////////// HPX_CORE_EXPORT std::size_t get_thread_data( thread_id_type const& id, error_code& ec = throws); diff --git a/libs/core/threading_base/src/scheduler_base.cpp b/libs/core/threading_base/src/scheduler_base.cpp index 3bb4facc62b3..8ddd582abe4e 100644 --- a/libs/core/threading_base/src/scheduler_base.cpp +++ b/libs/core/threading_base/src/scheduler_base.cpp @@ -330,8 +330,7 @@ namespace hpx::threads::policies { void scheduler_base::add_scheduler_mode(scheduler_mode mode) noexcept { // distribute the same value across all cores - mode = static_cast(get_scheduler_mode() | mode); - set_scheduler_mode(mode); + set_scheduler_mode(get_scheduler_mode() | mode); } void scheduler_base::remove_scheduler_mode(scheduler_mode mode) noexcept diff --git a/libs/core/threading_base/src/thread_helpers.cpp b/libs/core/threading_base/src/thread_helpers.cpp index ecbd95b0c87d..6e3830f65430 100644 --- a/libs/core/threading_base/src/thread_helpers.cpp +++ b/libs/core/threading_base/src/thread_helpers.cpp @@ -344,22 +344,6 @@ namespace hpx::threads { return get_thread_id_data(id)->add_thread_exit_callback(f); } - void free_thread_exit_callbacks(thread_id_type const& id, error_code& ec) - { - if (HPX_UNLIKELY(!id)) - { - HPX_THROWS_IF(ec, hpx::error::null_thread_id, - "hpx::threads::add_thread_exit_callback", - "null thread id encountered"); - return; - } - - if (&ec != &throws) - ec = make_success_code(); - - get_thread_id_data(id)->free_thread_exit_callbacks(); - } - /////////////////////////////////////////////////////////////////////////// #ifdef HPX_HAVE_THREAD_FULLBACKTRACE_ON_SUSPENSION char const* get_thread_backtrace(thread_id_type const& id, error_code& ec) diff --git a/libs/core/threading_base/tests/regressions/thread_stacksize_current.cpp b/libs/core/threading_base/tests/regressions/thread_stacksize_current.cpp index 077df205ea9f..eab76a473acc 100644 --- a/libs/core/threading_base/tests/regressions/thread_stacksize_current.cpp +++ b/libs/core/threading_base/tests/regressions/thread_stacksize_current.cpp @@ -75,7 +75,9 @@ int main(int argc, char** argv) "abp-priority-fifo", "abp-priority-lifo", #endif - "shared-priority" + "shared-priority", + "local-workrequesting-fifo", + "local-workrequesting-lifo" }; // clang-format on for (auto const& scheduler : schedulers) diff --git a/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp b/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp index b21712446e79..6e5efe4e9693 100644 --- a/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp +++ b/libs/core/threadmanager/include/hpx/modules/threadmanager.hpp @@ -425,6 +425,12 @@ namespace hpx { namespace threads { void create_scheduler_shared_priority( thread_pool_init_parameters const&, policies::thread_queue_init_parameters const&, std::size_t); + void create_scheduler_local_workrequesting_fifo( + thread_pool_init_parameters const&, + policies::thread_queue_init_parameters const&, std::size_t); + void create_scheduler_local_workrequesting_lifo( + thread_pool_init_parameters const&, + policies::thread_queue_init_parameters const&, std::size_t); mutable mutex_type mtx_; // mutex protecting the members diff --git a/libs/core/threadmanager/src/threadmanager.cpp b/libs/core/threadmanager/src/threadmanager.cpp index 1b4c555a7eca..1fde4443d865 100644 --- a/libs/core/threadmanager/src/threadmanager.cpp +++ b/libs/core/threadmanager/src/threadmanager.cpp @@ -502,6 +502,85 @@ namespace hpx { namespace threads { pools_.push_back(HPX_MOVE(pool)); } + void threadmanager::create_scheduler_local_workrequesting_fifo( + thread_pool_init_parameters const& thread_pool_init, + policies::thread_queue_init_parameters const& thread_queue_init, + std::size_t numa_sensitive) + { + // set parameters for scheduler and pool instantiation and + // perform compatibility checks + std::size_t num_high_priority_queues = + hpx::util::get_entry_as(rtcfg_, + "hpx.thread_queue.high_priority_queues", + thread_pool_init.num_threads_); + detail::check_num_high_priority_queues( + thread_pool_init.num_threads_, num_high_priority_queues); + + // instantiate the scheduler + using local_sched_type = + hpx::threads::policies::local_workrequesting_scheduler<>; + + local_sched_type::init_parameter_type init( + thread_pool_init.num_threads_, thread_pool_init.affinity_data_, + num_high_priority_queues, thread_queue_init, + "core-local_workrequesting_scheduler"); + + std::unique_ptr sched(new local_sched_type(init)); + + // set the default scheduler flags + sched->set_scheduler_mode(thread_pool_init.mode_); + + // conditionally set/unset this flag + sched->update_scheduler_mode( + policies::scheduler_mode::enable_stealing_numa, !numa_sensitive); + + // instantiate the pool + std::unique_ptr pool = std::make_unique< + hpx::threads::detail::scheduled_thread_pool>( + HPX_MOVE(sched), thread_pool_init); + pools_.push_back(HPX_MOVE(pool)); + } + + void threadmanager::create_scheduler_local_workrequesting_lifo( + thread_pool_init_parameters const& thread_pool_init, + policies::thread_queue_init_parameters const& thread_queue_init, + std::size_t numa_sensitive) + { + // set parameters for scheduler and pool instantiation and + // perform compatibility checks + std::size_t num_high_priority_queues = + hpx::util::get_entry_as(rtcfg_, + "hpx.thread_queue.high_priority_queues", + thread_pool_init.num_threads_); + detail::check_num_high_priority_queues( + thread_pool_init.num_threads_, num_high_priority_queues); + + // instantiate the scheduler + using local_sched_type = + hpx::threads::policies::local_workrequesting_scheduler; + + local_sched_type::init_parameter_type init( + thread_pool_init.num_threads_, thread_pool_init.affinity_data_, + num_high_priority_queues, thread_queue_init, + "core-local_workrequesting_scheduler"); + + std::unique_ptr sched(new local_sched_type(init)); + + // set the default scheduler flags + sched->set_scheduler_mode(thread_pool_init.mode_); + + // conditionally set/unset this flag + sched->update_scheduler_mode( + policies::scheduler_mode::enable_stealing_numa, !numa_sensitive); + + // instantiate the pool + std::unique_ptr pool = std::make_unique< + hpx::threads::detail::scheduled_thread_pool>( + HPX_MOVE(sched), thread_pool_init); + pools_.push_back(HPX_MOVE(pool)); + } + void threadmanager::create_pools() { auto& rp = hpx::resource::get_partitioner(); @@ -640,6 +719,16 @@ namespace hpx { namespace threads { thread_pool_init, thread_queue_init, numa_sensitive); break; + case resource::scheduling_policy::local_workrequesting_fifo: + create_scheduler_local_workrequesting_fifo( + thread_pool_init, thread_queue_init, numa_sensitive); + break; + + case resource::scheduling_policy::local_workrequesting_lifo: + create_scheduler_local_workrequesting_lifo( + thread_pool_init, thread_queue_init, numa_sensitive); + break; + case resource::scheduling_policy::abp_priority_fifo: create_scheduler_abp_priority_fifo( thread_pool_init, thread_queue_init, numa_sensitive); diff --git a/tools/perftests_ci/perftest/plot.py b/tools/perftests_ci/perftest/plot.py index 2d7c4777fa27..aecbf6ab713e 100644 --- a/tools/perftests_ci/perftest/plot.py +++ b/tools/perftests_ci/perftest/plot.py @@ -38,8 +38,10 @@ def __str__(self): @classmethod def outputs_by_key(cls, data): def split_output(o): - return cls(**{k: v - for k, v in o.items() if k != 'series'}), o['series'] + return cls(**{ + k: v for k, v in o.items() \ + if k != 'series' and k != 'average' \ + }), o['series'] return dict(split_output(o) for o in data['outputs']) From c227cb26601087e1f0cdb2102a0a12a8e65eb788 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 17 Jan 2023 17:17:59 -0600 Subject: [PATCH 193/333] Fixing race during destruction of hpx::thread --- .../include/hpx/threading/thread.hpp | 6 +- libs/core/threading/src/thread.cpp | 12 +- .../hpx/threading_base/thread_data.hpp | 32 +++++- libs/core/threading_base/src/thread_data.cpp | 103 ++++++++++++++---- 4 files changed, 120 insertions(+), 33 deletions(-) diff --git a/libs/core/threading/include/hpx/threading/thread.hpp b/libs/core/threading/include/hpx/threading/thread.hpp index 77005ea665d8..cf7c3fac0a4d 100644 --- a/libs/core/threading/include/hpx/threading/thread.hpp +++ b/libs/core/threading/include/hpx/threading/thread.hpp @@ -168,9 +168,11 @@ namespace hpx { return threads::invalid_thread_id != id_; } - void detach_locked() + threads::thread_id_ref_type detach_locked() { - id_ = threads::invalid_thread_id; + threads::thread_id_ref_type id; + std::swap(id, id_); + return id; } void start_thread(threads::thread_pool_base* pool, diff --git a/libs/core/threading/src/thread.cpp b/libs/core/threading/src/thread.cpp index 7b135e96bb0f..55a72fb0b7fa 100644 --- a/libs/core/threading/src/thread.cpp +++ b/libs/core/threading/src/thread.cpp @@ -112,7 +112,6 @@ namespace hpx { "run_thread_exit_callbacks", "null thread id encountered"); } threads::run_thread_exit_callbacks(id); - threads::free_thread_exit_callbacks(id); } threads::thread_result_type thread::thread_function_nullary( @@ -214,17 +213,18 @@ namespace hpx { } this_thread::interruption_point(); + // invalidate this object + threads::thread_id_ref_type id = detach_locked(); + // register callback function to be called when thread exits - if (threads::add_thread_exit_callback(id_.noref(), - hpx::bind_front(&resume_thread, HPX_MOVE(this_id)))) + if (threads::add_thread_exit_callback( + id.noref(), hpx::bind_front(&resume_thread, HPX_MOVE(this_id)))) { // wait for thread to be terminated - unlock_guard ul(l); + l.unlock(); this_thread::suspend( threads::thread_schedule_state::suspended, "thread::join"); } - - detach_locked(); // invalidate this object } // extensions diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp index 51d80608cbae..745c1d72e798 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp @@ -478,7 +478,6 @@ namespace hpx::threads { bool add_thread_exit_callback(function const& f); void run_thread_exit_callbacks(); - void free_thread_exit_callbacks(); // no need to protect the variables related to scoped children as those // are supposed to be accessed by ourselves only @@ -508,7 +507,8 @@ namespace hpx::threads { void set_last_worker_thread_num( std::size_t last_worker_thread_num) noexcept { - last_worker_thread_num_ = last_worker_thread_num; + last_worker_thread_num_ = + static_cast(last_worker_thread_num); } constexpr std::ptrdiff_t get_stack_size() const noexcept @@ -602,6 +602,34 @@ namespace hpx::threads { private: mutable std::atomic current_state_; + /////////////////////////////////////////////////////////////////////// + thread_priority priority_; + thread_stacksize stacksize_enum_; + + bool requested_interrupt_; + bool enabled_interrupt_; + + enum class exit_func_state + { + none, + ready, + processed + }; + + std::atomic ran_exit_funcs_; + bool const is_stackless_; + + std::uint16_t last_worker_thread_num_; + + // reference to scheduler which created/manages this thread + policies::scheduler_base* scheduler_base_; + void* queue_; + + std::ptrdiff_t stacksize_; + + // Singly linked list (heap-allocated) + std::forward_list> exit_funcs_; + /////////////////////////////////////////////////////////////////////// // Debugging/logging information #ifdef HPX_HAVE_THREAD_DESCRIPTION diff --git a/libs/core/threading_base/src/thread_data.cpp b/libs/core/threading_base/src/thread_data.cpp index a884fbc33d52..d89c7033ebc9 100644 --- a/libs/core/threading_base/src/thread_data.cpp +++ b/libs/core/threading_base/src/thread_data.cpp @@ -52,6 +52,16 @@ namespace hpx::threads { : detail::thread_data_reference_counting(addref) , current_state_(thread_state( init_data.initial_state, thread_restart_state::signaled)) + , priority_(init_data.priority) + , stacksize_enum_(init_data.stacksize) + , requested_interrupt_(false) + , enabled_interrupt_(true) + , ran_exit_funcs_(exit_func_state::none) + , is_stackless_(is_stackless) + , last_worker_thread_num_(std::uint16_t(-1)) + , scheduler_base_(init_data.scheduler_base) + , queue_(queue) + , stacksize_(stacksize) #ifdef HPX_HAVE_THREAD_DESCRIPTION , description_(init_data.description) , lco_description_() @@ -107,7 +117,13 @@ namespace hpx::threads { thread_data::~thread_data() { LTM_(debug).format("thread_data::~thread_data({})", this); - free_thread_exit_callbacks(); + + // Exit functions should have been executed. + HPX_ASSERT(exit_funcs_.empty() || + ran_exit_funcs_.load(std::memory_order_relaxed) == + exit_func_state::none || + ran_exit_funcs_.load(std::memory_order_relaxed) == + exit_func_state::processed); } void thread_data::destroy_thread() @@ -121,20 +137,55 @@ namespace hpx::threads { void thread_data::run_thread_exit_callbacks() { - std::unique_lock l( - spinlock_pool::spinlock_for(this)); - - while (!exit_funcs_.empty()) + // when leaving this function the state must be 'processed' + while (true) { + exit_func_state expected = exit_func_state::ready; + if (ran_exit_funcs_.compare_exchange_strong( + expected, exit_func_state::processed)) { - hpx::unlock_guard> - ul(l); - if (!exit_funcs_.front().empty()) - exit_funcs_.front()(); + // run exit functions only if there are any (state is 'ready') + std::unique_lock l( + spinlock_pool::spinlock_for(this)); + + while (!exit_funcs_.empty()) + { + if (!exit_funcs_.front().empty()) + { + auto f = exit_funcs_.front(); + exit_funcs_.pop_front(); + + hpx::unlock_guard< + std::unique_lock> + ul(l); + f(); + } + else + { + exit_funcs_.pop_front(); + } + } + + // clear all exit functions now as they are not needed anymore + exit_funcs_.clear(); + return; + } + else if (expected == exit_func_state::none) + { + if (ran_exit_funcs_.compare_exchange_strong( + expected, exit_func_state::processed)) + { + return; + } + + // try again, state was set to ready or processed by now + } + else + { + HPX_ASSERT(expected == exit_func_state::processed); + return; } - exit_funcs_.pop_front(); } - ran_exit_funcs_ = true; } bool thread_data::add_thread_exit_callback(hpx::function const& f) @@ -149,20 +200,28 @@ namespace hpx::threads { return false; } - exit_funcs_.push_front(f); + // don't register any more exit callback if the thread has already + // exited + exit_func_state expected = exit_func_state::none; + if (!ran_exit_funcs_.compare_exchange_strong( + expected, exit_func_state::ready)) + { + // the state was not none (i.e. ready or processed), bail out if it + // was processed + if (expected == exit_func_state::processed) + { + return false; + } + } - return true; - } + HPX_ASSERT(ran_exit_funcs_.load(std::memory_order_relaxed) == + exit_func_state::ready); - void thread_data::free_thread_exit_callbacks() - { std::lock_guard l( spinlock_pool::spinlock_for(this)); - // Exit functions should have been executed. - HPX_ASSERT(exit_funcs_.empty() || ran_exit_funcs_); - - exit_funcs_.clear(); + exit_funcs_.push_front(f); + return true; } bool thread_data::interruption_point(bool throw_on_interrupt) @@ -196,8 +255,6 @@ namespace hpx::threads { "thread_data::rebind_base({}), description({}), phase({}), rebind", this, get_description(), get_thread_phase()); - free_thread_exit_callbacks(); - current_state_.store(thread_state( init_data.initial_state, thread_restart_state::signaled)); @@ -219,7 +276,7 @@ namespace hpx::threads { priority_ = init_data.priority; requested_interrupt_ = false; enabled_interrupt_ = true; - ran_exit_funcs_ = false; + ran_exit_funcs_.store(exit_func_state::none, std::memory_order_relaxed); runs_as_child_.store(init_data.schedulehint.runs_as_child_mode() == hpx::threads::thread_execution_hint::run_as_child, From c8d09edaf63604aaf282e649b1b0543473fcc5c1 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 27 Jan 2023 16:07:16 -0600 Subject: [PATCH 194/333] Fixing merge conflicts, revert changes to thread exit callbacks --- .../include/hpx/modules/schedulers.hpp | 2 + .../local_workrequesting_scheduler.hpp | 80 ++++++++------ .../include/hpx/schedulers/thread_queue.hpp | 3 +- .../hpx/thread_pools/scheduling_loop.hpp | 11 +- .../include/hpx/threading/thread.hpp | 6 +- libs/core/threading/src/thread.cpp | 12 +- .../hpx/threading_base/thread_data.hpp | 32 +----- .../hpx/threading_base/thread_helpers.hpp | 3 + libs/core/threading_base/src/thread_data.cpp | 103 ++++-------------- .../threading_base/src/thread_helpers.cpp | 16 +++ 10 files changed, 105 insertions(+), 163 deletions(-) diff --git a/libs/core/schedulers/include/hpx/modules/schedulers.hpp b/libs/core/schedulers/include/hpx/modules/schedulers.hpp index d6790d352d73..2de14cc98363 100644 --- a/libs/core/schedulers/include/hpx/modules/schedulers.hpp +++ b/libs/core/schedulers/include/hpx/modules/schedulers.hpp @@ -6,6 +6,8 @@ #pragma once +#include + #include #include #include diff --git a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp index bb025b80e987..ac6c1e060ee8 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -46,7 +47,7 @@ // different working principle if compared to the classic work-stealing. Instead // of actively pulling work from the work queues of neighboring cores it relies // on a push model. Cores that run out of work post steal requests that are -// handled by cores that have work awailable by actively sending tasks to the +// handled by cores that have work available by actively sending tasks to the // requesting core. // // When a worker runs out of tasks, it becomes a thief by sending steal requests @@ -115,18 +116,18 @@ namespace hpx::threads::policies { using thread_queue_type = thread_queue; - public: struct init_parameter { init_parameter(std::size_t num_queues, detail::affinity_data const& affinity_data, - std::size_t num_high_priority_queues = std::size_t(-1), + std::size_t num_high_priority_queues = static_cast( + -1), thread_queue_init_parameters const& thread_queue_init = thread_queue_init_parameters{}, char const* description = "local_workrequesting_scheduler") : num_queues_(num_queues) , num_high_priority_queues_( - num_high_priority_queues == std::size_t(-1) ? + num_high_priority_queues == static_cast(-1) ? num_queues : num_high_priority_queues) , thread_queue_init_(thread_queue_init) @@ -159,7 +160,8 @@ namespace hpx::threads::policies { struct task_data { explicit HPX_HOST_DEVICE_CONSTEXPR task_data( - std::uint16_t num_thread = std::uint16_t(-1)) noexcept + std::uint16_t num_thread = static_cast( + -1)) noexcept : num_thread_(num_thread) { } @@ -188,8 +190,8 @@ namespace hpx::threads::policies { { } - steal_request(std::size_t num_thread, task_channel* channel, - mask_cref_type victims, bool idle, bool stealhalf) + steal_request(std::size_t const num_thread, task_channel* channel, + mask_cref_type victims, bool idle, bool const stealhalf) : channel_(channel) , victims_(victims) , num_thread_(static_cast(num_thread)) @@ -221,6 +223,11 @@ namespace hpx::threads::policies { { } + scheduler_data(scheduler_data const&) = delete; + scheduler_data(scheduler_data&&) = delete; + scheduler_data& operator=(scheduler_data const&) = delete; + scheduler_data& operator=(scheduler_data&&) = delete; + ~scheduler_data() { delete queue_; @@ -330,9 +337,18 @@ namespace hpx::threads::policies { } } + local_workrequesting_scheduler( + local_workrequesting_scheduler const&) = delete; + local_workrequesting_scheduler( + local_workrequesting_scheduler&&) = delete; + local_workrequesting_scheduler& operator=( + local_workrequesting_scheduler const&) = delete; + local_workrequesting_scheduler& operator=( + local_workrequesting_scheduler&&) = delete; + ~local_workrequesting_scheduler() override = default; - static std::string get_scheduler_name() + static std::string_view get_scheduler_name() { return "local_workrequesting_scheduler"; } @@ -659,9 +675,9 @@ namespace hpx::threads::policies { std::size_t num_thread = data.schedulehint.mode == thread_schedule_hint_mode::thread ? data.schedulehint.hint : - std::size_t(-1); + static_cast(-1); - if (std::size_t(-1) == num_thread) + if (static_cast(-1) == num_thread) { num_thread = curr_queue_++ % num_queues_; } @@ -889,7 +905,7 @@ namespace hpx::threads::policies { // Return the next thread to be executed, return false if none is // available bool get_next_thread(std::size_t num_thread, bool running, - thread_id_ref_type& thrd, bool enable_stealing) override + thread_id_ref_type& thrd, bool enable_stealing) { HPX_ASSERT(num_thread < num_queues_); @@ -968,9 +984,9 @@ namespace hpx::threads::policies { void schedule_thread(thread_id_ref_type thrd, threads::thread_schedule_hint schedulehint, bool allow_fallback = false, - thread_priority priority = thread_priority::normal) override + thread_priority priority = thread_priority::default_) override { - std::size_t num_thread = std::size_t(-1); + std::size_t num_thread = static_cast(-1); if (schedulehint.mode == thread_schedule_hint_mode::thread) { num_thread = schedulehint.hint; @@ -980,7 +996,7 @@ namespace hpx::threads::policies { allow_fallback = false; } - if (std::size_t(-1) == num_thread) + if (static_cast(-1) == num_thread) { num_thread = curr_queue_++ % num_queues_; } @@ -1038,9 +1054,9 @@ namespace hpx::threads::policies { void schedule_thread_last(thread_id_ref_type thrd, threads::thread_schedule_hint schedulehint, bool allow_fallback = false, - thread_priority priority = thread_priority::normal) override + thread_priority priority = thread_priority::default_) override { - std::size_t num_thread = std::size_t(-1); + std::size_t num_thread = static_cast(-1); if (schedulehint.mode == thread_schedule_hint_mode::thread) { num_thread = schedulehint.hint; @@ -1050,7 +1066,7 @@ namespace hpx::threads::policies { allow_fallback = false; } - if (std::size_t(-1) == num_thread) + if (static_cast(-1) == num_thread) { num_thread = curr_queue_++ % num_queues_; } @@ -1109,12 +1125,11 @@ namespace hpx::threads::policies { /////////////////////////////////////////////////////////////////////// // This returns the current length of the queues (work items and new // items) - std::int64_t get_queue_length( - std::size_t num_thread = std::size_t(-1)) const override + std::int64_t get_queue_length(std::size_t num_thread) const override { // Return queue length of one specific queue. std::int64_t count = 0; - if (std::size_t(-1) != num_thread) + if (static_cast(-1) != num_thread) { HPX_ASSERT(num_thread < num_queues_); auto const& d = data_[num_thread].data_; @@ -1150,12 +1165,12 @@ namespace hpx::threads::policies { std::int64_t get_thread_count( thread_schedule_state state = thread_schedule_state::unknown, thread_priority priority = thread_priority::default_, - std::size_t num_thread = std::size_t(-1), + std::size_t num_thread = static_cast(-1), bool /* reset */ = false) const override { // Return thread count of one specific queue. std::int64_t count = 0; - if (std::size_t(-1) != num_thread) + if (static_cast(-1) != num_thread) { HPX_ASSERT(num_thread < num_queues_); @@ -1207,7 +1222,6 @@ namespace hpx::threads::policies { "local_workrequesting_scheduler::get_thread_count", "unknown thread priority value " "(thread_priority::unknown)"); - return 0; } } return 0; @@ -1275,7 +1289,6 @@ namespace hpx::threads::policies { "local_workrequesting_scheduler::get_thread_count", "unknown thread priority value " "(thread_priority::unknown)"); - return 0; } } return count; @@ -1444,7 +1457,7 @@ namespace hpx::threads::policies { // generate at most 3 random numbers before resorting to more // expensive algorithm std::uniform_int_distribution uniform( - 0, std::int16_t(num_queues_ - 1)); + 0, static_cast(num_queues_ - 1)); int attempts = 0; do @@ -1461,8 +1474,9 @@ namespace hpx::threads::policies { // to avoid infinite trials we randomly select one of the possible // victims - std::uniform_int_distribution uniform( - 0, std::int16_t(num_queues_ - count(req.victims_) - 1)); + std::uniform_int_distribution uniform(0, + static_cast( + num_queues_ - count(req.victims_) - 1)); // generate one more random number std::size_t selected_victim = uniform(gen_); @@ -1489,7 +1503,7 @@ namespace hpx::threads::policies { std::size_t next_victim([[maybe_unused]] scheduler_data& d, steal_request const& req) noexcept { - std::size_t victim = std::size_t(-1); + std::size_t victim; // return thief if max steal attempts has been reached or no more // cores are available for stealing @@ -1516,7 +1530,7 @@ namespace hpx::threads::policies { } // couldn't find victim, return steal request to thief - if (victim == std::size_t(-1)) + if (victim == static_cast(-1)) { victim = req.num_thread_; HPX_ASSERT(victim != d.num_thread_); @@ -1650,7 +1664,7 @@ namespace hpx::threads::policies { bool wait_or_add_new(std::size_t num_thread, bool running, [[maybe_unused]] std::int64_t& idle_loop_count, bool enable_stealing, std::size_t& added, - thread_id_ref_type* next_thrd = nullptr) override + thread_id_ref_type* next_thrd = nullptr) { HPX_ASSERT(num_thread < num_queues_); @@ -1755,11 +1769,11 @@ namespace hpx::threads::policies { low_priority_queue_.on_start_thread(num_thread); } - std::size_t num_threads = num_queues_; + std::size_t const num_threads = num_queues_; //auto const& topo = create_topology(); // Initially set all bits, code below resets the bits corresponding - // to cores that can serve as a vistim for the current core. A set + // to cores that can serve as a victim for the current core. A set // bit in this mask means 'do not steal from this core'. resize(d.victims_, num_threads); reset(d.victims_); @@ -1889,7 +1903,7 @@ namespace hpx::threads::policies { } } - void reset_thread_distribution() override + void reset_thread_distribution() noexcept override { curr_queue_.store(0, std::memory_order_release); } diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 25260cc85ca4..4c8524bad734 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -457,8 +457,7 @@ namespace hpx::threads::policies { if (delete_all) { - // do not lock mutex while deleting all threads, do it - // piece-wise + // do not lock mutex while deleting all threads, do it piece-wise while (true) { std::unique_lock lk(mtx_, std::try_to_lock); diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp index 86c18a3ba2b4..6a6dbb766d69 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp @@ -19,7 +19,6 @@ #include #include #include -#include #if defined(HPX_HAVE_APEX) #include @@ -179,7 +178,6 @@ namespace hpx::threads::detail { idle_collect_rate idle_rate(counters.tfunc_time_, counters.exec_time_); [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector(idle_rate); - HPX_UNUSED(tfunc_time_collector); // spin for some time after queues have become empty bool may_exit = false; @@ -237,8 +235,6 @@ namespace hpx::threads::detail { scheduler.get_next_thread( num_thread, running, thrd, enable_stealing))) { - HPX_UNUSED(tfunc_time_collector); - HPX_ASSERT(get_thread_id_data(thrd)->get_scheduler_base() == &scheduler); @@ -273,7 +269,6 @@ namespace hpx::threads::detail { [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector_inner(idle_rate); - HPX_UNUSED(tfunc_time_collector); // thread returns new required state store the // returned state in the thread @@ -294,7 +289,7 @@ namespace hpx::threads::detail { // and add to aggregate execution time. [[maybe_unused]] exec_time_wrapper exec_time_collector(idle_rate); - HPX_UNUSED(exec_time_collector); + #if defined(HPX_HAVE_APEX) // get the APEX data pointer, in case we are // resuming the thread and have to restore any @@ -470,8 +465,8 @@ namespace hpx::threads::detail { ++idle_loop_count; next_thrd = nullptr; - if (scheduler.wait_or_add_new(num_thread, - running, idle_loop_count, enable_stealing_staged, added, + if (scheduler.wait_or_add_new(num_thread, running, + idle_loop_count, enable_stealing_staged, added, &next_thrd)) { // Clean up terminated threads before trying to exit diff --git a/libs/core/threading/include/hpx/threading/thread.hpp b/libs/core/threading/include/hpx/threading/thread.hpp index cf7c3fac0a4d..77005ea665d8 100644 --- a/libs/core/threading/include/hpx/threading/thread.hpp +++ b/libs/core/threading/include/hpx/threading/thread.hpp @@ -168,11 +168,9 @@ namespace hpx { return threads::invalid_thread_id != id_; } - threads::thread_id_ref_type detach_locked() + void detach_locked() { - threads::thread_id_ref_type id; - std::swap(id, id_); - return id; + id_ = threads::invalid_thread_id; } void start_thread(threads::thread_pool_base* pool, diff --git a/libs/core/threading/src/thread.cpp b/libs/core/threading/src/thread.cpp index 55a72fb0b7fa..7b135e96bb0f 100644 --- a/libs/core/threading/src/thread.cpp +++ b/libs/core/threading/src/thread.cpp @@ -112,6 +112,7 @@ namespace hpx { "run_thread_exit_callbacks", "null thread id encountered"); } threads::run_thread_exit_callbacks(id); + threads::free_thread_exit_callbacks(id); } threads::thread_result_type thread::thread_function_nullary( @@ -213,18 +214,17 @@ namespace hpx { } this_thread::interruption_point(); - // invalidate this object - threads::thread_id_ref_type id = detach_locked(); - // register callback function to be called when thread exits - if (threads::add_thread_exit_callback( - id.noref(), hpx::bind_front(&resume_thread, HPX_MOVE(this_id)))) + if (threads::add_thread_exit_callback(id_.noref(), + hpx::bind_front(&resume_thread, HPX_MOVE(this_id)))) { // wait for thread to be terminated - l.unlock(); + unlock_guard ul(l); this_thread::suspend( threads::thread_schedule_state::suspended, "thread::join"); } + + detach_locked(); // invalidate this object } // extensions diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp index 745c1d72e798..51d80608cbae 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_data.hpp @@ -478,6 +478,7 @@ namespace hpx::threads { bool add_thread_exit_callback(function const& f); void run_thread_exit_callbacks(); + void free_thread_exit_callbacks(); // no need to protect the variables related to scoped children as those // are supposed to be accessed by ourselves only @@ -507,8 +508,7 @@ namespace hpx::threads { void set_last_worker_thread_num( std::size_t last_worker_thread_num) noexcept { - last_worker_thread_num_ = - static_cast(last_worker_thread_num); + last_worker_thread_num_ = last_worker_thread_num; } constexpr std::ptrdiff_t get_stack_size() const noexcept @@ -602,34 +602,6 @@ namespace hpx::threads { private: mutable std::atomic current_state_; - /////////////////////////////////////////////////////////////////////// - thread_priority priority_; - thread_stacksize stacksize_enum_; - - bool requested_interrupt_; - bool enabled_interrupt_; - - enum class exit_func_state - { - none, - ready, - processed - }; - - std::atomic ran_exit_funcs_; - bool const is_stackless_; - - std::uint16_t last_worker_thread_num_; - - // reference to scheduler which created/manages this thread - policies::scheduler_base* scheduler_base_; - void* queue_; - - std::ptrdiff_t stacksize_; - - // Singly linked list (heap-allocated) - std::forward_list> exit_funcs_; - /////////////////////////////////////////////////////////////////////// // Debugging/logging information #ifdef HPX_HAVE_THREAD_DESCRIPTION diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp index 70f5ad7c5685..131369ccdc6c 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp @@ -387,6 +387,9 @@ namespace hpx::threads { HPX_CORE_EXPORT bool add_thread_exit_callback(thread_id_type const& id, hpx::function const& f, error_code& ec = throws); + HPX_CORE_EXPORT void free_thread_exit_callbacks( + thread_id_type const& id, error_code& ec = throws); + /////////////////////////////////////////////////////////////////////////// HPX_CORE_EXPORT std::size_t get_thread_data( thread_id_type const& id, error_code& ec = throws); diff --git a/libs/core/threading_base/src/thread_data.cpp b/libs/core/threading_base/src/thread_data.cpp index d89c7033ebc9..a884fbc33d52 100644 --- a/libs/core/threading_base/src/thread_data.cpp +++ b/libs/core/threading_base/src/thread_data.cpp @@ -52,16 +52,6 @@ namespace hpx::threads { : detail::thread_data_reference_counting(addref) , current_state_(thread_state( init_data.initial_state, thread_restart_state::signaled)) - , priority_(init_data.priority) - , stacksize_enum_(init_data.stacksize) - , requested_interrupt_(false) - , enabled_interrupt_(true) - , ran_exit_funcs_(exit_func_state::none) - , is_stackless_(is_stackless) - , last_worker_thread_num_(std::uint16_t(-1)) - , scheduler_base_(init_data.scheduler_base) - , queue_(queue) - , stacksize_(stacksize) #ifdef HPX_HAVE_THREAD_DESCRIPTION , description_(init_data.description) , lco_description_() @@ -117,13 +107,7 @@ namespace hpx::threads { thread_data::~thread_data() { LTM_(debug).format("thread_data::~thread_data({})", this); - - // Exit functions should have been executed. - HPX_ASSERT(exit_funcs_.empty() || - ran_exit_funcs_.load(std::memory_order_relaxed) == - exit_func_state::none || - ran_exit_funcs_.load(std::memory_order_relaxed) == - exit_func_state::processed); + free_thread_exit_callbacks(); } void thread_data::destroy_thread() @@ -137,55 +121,20 @@ namespace hpx::threads { void thread_data::run_thread_exit_callbacks() { - // when leaving this function the state must be 'processed' - while (true) - { - exit_func_state expected = exit_func_state::ready; - if (ran_exit_funcs_.compare_exchange_strong( - expected, exit_func_state::processed)) - { - // run exit functions only if there are any (state is 'ready') - std::unique_lock l( - spinlock_pool::spinlock_for(this)); - - while (!exit_funcs_.empty()) - { - if (!exit_funcs_.front().empty()) - { - auto f = exit_funcs_.front(); - exit_funcs_.pop_front(); - - hpx::unlock_guard< - std::unique_lock> - ul(l); - f(); - } - else - { - exit_funcs_.pop_front(); - } - } - - // clear all exit functions now as they are not needed anymore - exit_funcs_.clear(); - return; - } - else if (expected == exit_func_state::none) - { - if (ran_exit_funcs_.compare_exchange_strong( - expected, exit_func_state::processed)) - { - return; - } + std::unique_lock l( + spinlock_pool::spinlock_for(this)); - // try again, state was set to ready or processed by now - } - else + while (!exit_funcs_.empty()) + { { - HPX_ASSERT(expected == exit_func_state::processed); - return; + hpx::unlock_guard> + ul(l); + if (!exit_funcs_.front().empty()) + exit_funcs_.front()(); } + exit_funcs_.pop_front(); } + ran_exit_funcs_ = true; } bool thread_data::add_thread_exit_callback(hpx::function const& f) @@ -200,28 +149,20 @@ namespace hpx::threads { return false; } - // don't register any more exit callback if the thread has already - // exited - exit_func_state expected = exit_func_state::none; - if (!ran_exit_funcs_.compare_exchange_strong( - expected, exit_func_state::ready)) - { - // the state was not none (i.e. ready or processed), bail out if it - // was processed - if (expected == exit_func_state::processed) - { - return false; - } - } + exit_funcs_.push_front(f); - HPX_ASSERT(ran_exit_funcs_.load(std::memory_order_relaxed) == - exit_func_state::ready); + return true; + } + void thread_data::free_thread_exit_callbacks() + { std::lock_guard l( spinlock_pool::spinlock_for(this)); - exit_funcs_.push_front(f); - return true; + // Exit functions should have been executed. + HPX_ASSERT(exit_funcs_.empty() || ran_exit_funcs_); + + exit_funcs_.clear(); } bool thread_data::interruption_point(bool throw_on_interrupt) @@ -255,6 +196,8 @@ namespace hpx::threads { "thread_data::rebind_base({}), description({}), phase({}), rebind", this, get_description(), get_thread_phase()); + free_thread_exit_callbacks(); + current_state_.store(thread_state( init_data.initial_state, thread_restart_state::signaled)); @@ -276,7 +219,7 @@ namespace hpx::threads { priority_ = init_data.priority; requested_interrupt_ = false; enabled_interrupt_ = true; - ran_exit_funcs_.store(exit_func_state::none, std::memory_order_relaxed); + ran_exit_funcs_ = false; runs_as_child_.store(init_data.schedulehint.runs_as_child_mode() == hpx::threads::thread_execution_hint::run_as_child, diff --git a/libs/core/threading_base/src/thread_helpers.cpp b/libs/core/threading_base/src/thread_helpers.cpp index 6e3830f65430..ecbd95b0c87d 100644 --- a/libs/core/threading_base/src/thread_helpers.cpp +++ b/libs/core/threading_base/src/thread_helpers.cpp @@ -344,6 +344,22 @@ namespace hpx::threads { return get_thread_id_data(id)->add_thread_exit_callback(f); } + void free_thread_exit_callbacks(thread_id_type const& id, error_code& ec) + { + if (HPX_UNLIKELY(!id)) + { + HPX_THROWS_IF(ec, hpx::error::null_thread_id, + "hpx::threads::add_thread_exit_callback", + "null thread id encountered"); + return; + } + + if (&ec != &throws) + ec = make_success_code(); + + get_thread_id_data(id)->free_thread_exit_callbacks(); + } + /////////////////////////////////////////////////////////////////////////// #ifdef HPX_HAVE_THREAD_FULLBACKTRACE_ON_SUSPENSION char const* get_thread_backtrace(thread_id_type const& id, error_code& ec) From 3ecb7bdfb075bb7ad3a51ff28db39a4492946db1 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 21 Aug 2023 14:08:03 -0500 Subject: [PATCH 195/333] Fixing use of sliding semaphore --- examples/1d_stencil/1d_stencil_4.cpp | 8 ++++---- examples/1d_stencil/1d_stencil_4_throttle.cpp | 8 ++++---- examples/1d_stencil/1d_stencil_7.cpp | 9 +++++---- examples/1d_stencil/1d_stencil_8.cpp | 9 +++++---- .../tests/performance/replay/1d_stencil.cpp | 10 +++++----- .../tests/performance/replay/1d_stencil_checksum.cpp | 10 +++++----- .../tests/performance/replay/1d_stencil_replay.cpp | 10 +++++----- .../tests/performance/replay/dataflow_replay.cpp | 11 ++++++----- .../performance/replay/dataflow_replay_validate.cpp | 11 ++++++----- .../tests/performance/replay/pure_dataflow.cpp | 10 +++++----- .../performance/replicate/1d_stencil_replicate.cpp | 10 +++++----- .../replicate/1d_stencil_replicate_checksum.cpp | 10 +++++----- .../synchronization/tests/unit/sliding_semaphore.cpp | 9 +++++---- .../checkpoint/examples/1d_stencil_4_checkpoint.cpp | 8 ++++---- tests/performance/local/future_overhead.cpp | 11 ++++++----- 15 files changed, 75 insertions(+), 69 deletions(-) diff --git a/examples/1d_stencil/1d_stencil_4.cpp b/examples/1d_stencil/1d_stencil_4.cpp index de59844927af..c39fe45e4a48 100644 --- a/examples/1d_stencil/1d_stencil_4.cpp +++ b/examples/1d_stencil/1d_stencil_4.cpp @@ -163,7 +163,7 @@ struct stepper }); // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); auto Op = unwrapping(&stepper::heat_part); @@ -184,15 +184,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'nt'. diff --git a/examples/1d_stencil/1d_stencil_4_throttle.cpp b/examples/1d_stencil/1d_stencil_4_throttle.cpp index 87e8222c2712..5358c3eff3f1 100644 --- a/examples/1d_stencil/1d_stencil_4_throttle.cpp +++ b/examples/1d_stencil/1d_stencil_4_throttle.cpp @@ -240,7 +240,7 @@ struct stepper }); // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); auto Op = unwrapping(&stepper::heat_part); @@ -261,15 +261,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'nt'. diff --git a/examples/1d_stencil/1d_stencil_7.cpp b/examples/1d_stencil/1d_stencil_7.cpp index 99258d307a08..71971fb927ef 100644 --- a/examples/1d_stencil/1d_stencil_7.cpp +++ b/examples/1d_stencil/1d_stencil_7.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -359,7 +360,7 @@ stepper::space stepper::do_work(std::size_t np, std::size_t nx, std::size_t nt) // limit depth of dependency tree std::size_t nd = 3; - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); heat_part_action act; for (std::size_t t = 0; t != nt; ++t) @@ -377,15 +378,15 @@ stepper::space stepper::do_work(std::size_t np, std::size_t nx, std::size_t nt) if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'nt'. diff --git a/examples/1d_stencil/1d_stencil_8.cpp b/examples/1d_stencil/1d_stencil_8.cpp index 644452b5f820..d01c227c3724 100644 --- a/examples/1d_stencil/1d_stencil_8.cpp +++ b/examples/1d_stencil/1d_stencil_8.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -575,7 +576,7 @@ stepper_server::space stepper_server::do_work( } // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); for (std::size_t t = 0; t != nt; ++t) { @@ -626,15 +627,15 @@ stepper_server::space stepper_server::do_work( // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } return U_[nt % 2]; diff --git a/libs/core/resiliency/tests/performance/replay/1d_stencil.cpp b/libs/core/resiliency/tests/performance/replay/1d_stencil.cpp index 6e37da5b482b..de5de3981c9b 100644 --- a/libs/core/resiliency/tests/performance/replay/1d_stencil.cpp +++ b/libs/core/resiliency/tests/performance/replay/1d_stencil.cpp @@ -150,7 +150,7 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::size_t sti, - std::uint64_t nd, hpx::sliding_semaphore& sem) + std::uint64_t nd, std::shared_ptr sem) { using hpx::dataflow; using hpx::unwrapping; @@ -186,15 +186,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -227,7 +227,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work(subdomains, subdomain_width, iterations, sti, nd, sem); diff --git a/libs/core/resiliency/tests/performance/replay/1d_stencil_checksum.cpp b/libs/core/resiliency/tests/performance/replay/1d_stencil_checksum.cpp index c67de2cf4d64..cfeff02ab118 100644 --- a/libs/core/resiliency/tests/performance/replay/1d_stencil_checksum.cpp +++ b/libs/core/resiliency/tests/performance/replay/1d_stencil_checksum.cpp @@ -236,7 +236,7 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::size_t sti, std::uint64_t nd, std::uint64_t n_value, double error, - hpx::sliding_semaphore& sem) + std::shared_ptr sem) { using hpx::unwrapping; using hpx::resiliency::experimental::dataflow_replay_validate; @@ -275,15 +275,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -328,7 +328,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work(subdomains, subdomain_width, iterations, sti, nd, n_value, error, sem); diff --git a/libs/core/resiliency/tests/performance/replay/1d_stencil_replay.cpp b/libs/core/resiliency/tests/performance/replay/1d_stencil_replay.cpp index 48438b63079b..a42824677cea 100644 --- a/libs/core/resiliency/tests/performance/replay/1d_stencil_replay.cpp +++ b/libs/core/resiliency/tests/performance/replay/1d_stencil_replay.cpp @@ -193,7 +193,7 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::size_t sti, std::uint64_t nd, std::uint64_t n_value, double error, - hpx::sliding_semaphore& sem) + std::shared_ptr sem) { using hpx::unwrapping; using hpx::resiliency::experimental::dataflow_replay; @@ -231,15 +231,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -275,7 +275,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work(subdomains, subdomain_width, iterations, sti, nd, n_value, error, sem); diff --git a/libs/core/resiliency/tests/performance/replay/dataflow_replay.cpp b/libs/core/resiliency/tests/performance/replay/dataflow_replay.cpp index b489745bc4d5..93cd2e11d8a6 100644 --- a/libs/core/resiliency/tests/performance/replay/dataflow_replay.cpp +++ b/libs/core/resiliency/tests/performance/replay/dataflow_replay.cpp @@ -237,7 +237,8 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::uint64_t nd, - std::uint64_t n_value, double error, hpx::sliding_semaphore& sem) + std::uint64_t n_value, double error, + std::shared_ptr sem) { using hpx::unwrapping; using hpx::resiliency::experimental::dataflow_replay; @@ -273,15 +274,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -314,7 +315,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work( subdomains, subdomain_width, iterations, nd, n_value, error, sem); diff --git a/libs/core/resiliency/tests/performance/replay/dataflow_replay_validate.cpp b/libs/core/resiliency/tests/performance/replay/dataflow_replay_validate.cpp index a8746218ed18..b09400bb34e9 100644 --- a/libs/core/resiliency/tests/performance/replay/dataflow_replay_validate.cpp +++ b/libs/core/resiliency/tests/performance/replay/dataflow_replay_validate.cpp @@ -229,7 +229,8 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::uint64_t nd, - std::uint64_t n_value, double error, hpx::sliding_semaphore& sem) + std::uint64_t n_value, double error, + std::shared_ptr sem) { using hpx::unwrapping; using hpx::resiliency::experimental::dataflow_replay_validate; @@ -268,15 +269,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -318,7 +319,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work( subdomains, subdomain_width, iterations, nd, n_value, error, sem); diff --git a/libs/core/resiliency/tests/performance/replay/pure_dataflow.cpp b/libs/core/resiliency/tests/performance/replay/pure_dataflow.cpp index 8c59a48645f2..d049c042f88b 100644 --- a/libs/core/resiliency/tests/performance/replay/pure_dataflow.cpp +++ b/libs/core/resiliency/tests/performance/replay/pure_dataflow.cpp @@ -163,7 +163,7 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::uint64_t nd, - hpx::sliding_semaphore& sem) + std::shared_ptr sem) { using hpx::dataflow; using hpx::unwrapping; @@ -199,15 +199,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -237,7 +237,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work(subdomains, subdomain_width, iterations, nd, sem); diff --git a/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate.cpp b/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate.cpp index a3633a478c69..33d71caf3757 100644 --- a/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate.cpp +++ b/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate.cpp @@ -194,7 +194,7 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::size_t sti, std::uint64_t nd, std::uint64_t n_value, double error, - hpx::sliding_semaphore& sem) + std::shared_ptr sem) { using hpx::unwrapping; using hpx::resiliency::experimental::dataflow_replicate; @@ -230,15 +230,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -274,7 +274,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work(subdomains, subdomain_width, iterations, sti, nd, n_value, error, sem); diff --git a/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate_checksum.cpp b/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate_checksum.cpp index 76e17b837f59..8b33c2b44edc 100644 --- a/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate_checksum.cpp +++ b/libs/core/resiliency/tests/performance/replicate/1d_stencil_replicate_checksum.cpp @@ -235,7 +235,7 @@ struct stepper hpx::future do_work(std::size_t subdomains, std::size_t subdomain_width, std::size_t iterations, std::size_t sti, std::uint64_t nd, std::uint64_t n_value, double error, - hpx::sliding_semaphore& sem) + std::shared_ptr sem) { using hpx::unwrapping; using hpx::resiliency::experimental::dataflow_replicate_validate; @@ -271,15 +271,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Return the solution at time-step 'iterations'. @@ -324,7 +324,7 @@ int hpx_main(hpx::program_options::variables_map& vm) { // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); hpx::future result = step.do_work(subdomains, subdomain_width, iterations, sti, nd, n_value, error, sem); diff --git a/libs/core/synchronization/tests/unit/sliding_semaphore.cpp b/libs/core/synchronization/tests/unit/sliding_semaphore.cpp index fd6876fa9b9a..185b65756da7 100644 --- a/libs/core/synchronization/tests/unit/sliding_semaphore.cpp +++ b/libs/core/synchronization/tests/unit/sliding_semaphore.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -23,9 +24,9 @@ int const initial_count = 42; int const num_tasks = 139; std::atomic completed_tasks(0); -void worker(hpx::sliding_semaphore& sem) +void worker(std::shared_ptr sem) { - sem.signal(++count); // signal main thread + sem->signal(++count); // signal main thread } /////////////////////////////////////////////////////////////////////////////// @@ -34,14 +35,14 @@ int hpx_main() std::vector> futures; futures.reserve(num_tasks); - hpx::sliding_semaphore sem(initial_count); + auto sem = std::make_shared(initial_count); for (std::size_t i = 0; i != num_tasks; ++i) { futures.emplace_back(hpx::async(&worker, std::ref(sem))); } - sem.wait(initial_count + num_tasks); + sem->wait(initial_count + num_tasks); HPX_TEST_EQ(count, num_tasks); diff --git a/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp b/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp index 9a8fcf5f933d..c25d206a50df 100644 --- a/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp +++ b/libs/full/checkpoint/examples/1d_stencil_4_checkpoint.cpp @@ -317,7 +317,7 @@ struct stepper print(U); // limit depth of dependency tree - hpx::sliding_semaphore sem(nd); + auto sem = std::make_shared(nd); auto Op = unwrapping(&stepper::heat_part); @@ -367,15 +367,15 @@ struct stepper // trigger the semaphore once computation has reached this point if ((t % nd) == 0) { - next[0].then([&sem, t](partition&&) { + next[0].then([sem, t](partition&&) { // inform semaphore about new lower limit - sem.signal(t); + sem->signal(t); }); } // suspend if the tree has become too deep, the continuation above // will resume this thread once the computation has caught up - sem.wait(t); + sem->wait(t); } // Wait on Checkpoint Printing diff --git a/tests/performance/local/future_overhead.cpp b/tests/performance/local/future_overhead.cpp index 2ea3701c48d0..4bf154750913 100644 --- a/tests/performance/local/future_overhead.cpp +++ b/tests/performance/local/future_overhead.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -257,16 +258,16 @@ void measure_function_futures_sliding_semaphore( // start the clock high_resolution_timer walltime; const int sem_count = 5000; - hpx::sliding_semaphore sem(sem_count); + auto sem = std::make_shared(sem_count); for (std::uint64_t i = 0; i < count; ++i) { - hpx::async(exec, [i, &sem]() { + hpx::async(exec, [i, sem]() { null_function(); - sem.signal(i); + sem->signal(i); }); - sem.wait(i); + sem->wait(i); } - sem.wait(count + sem_count - 1); + sem->wait(count + sem_count - 1); // stop the clock const double duration = walltime.elapsed(); From 53db9433fb6f36b18916d6689e4b2bbec98c1041 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 21 Aug 2023 14:22:21 -0500 Subject: [PATCH 196/333] Cleaning up implementation --- .../local_workrequesting_scheduler.hpp | 136 +++--------------- 1 file changed, 20 insertions(+), 116 deletions(-) diff --git a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp index ac6c1e060ee8..57b737fbae0e 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp @@ -141,7 +141,6 @@ namespace hpx::threads::policies { char const* description) : num_queues_(num_queues) , num_high_priority_queues_(num_queues) - , thread_queue_init_() , affinity_data_(affinity_data) , description_(description) { @@ -185,10 +184,7 @@ namespace hpx::threads::policies { failed = 4 }; - steal_request() noexcept - : victims_() - { - } + steal_request() = default; steal_request(std::size_t const num_thread, task_channel* channel, mask_cref_type victims, bool idle, bool const stealhalf) @@ -218,10 +214,7 @@ namespace hpx::threads::policies { //////////////////////////////////////////////////////////////////////// struct scheduler_data { - scheduler_data() noexcept - : victims_() - { - } + scheduler_data() = default; scheduler_data(scheduler_data const&) = delete; scheduler_data(scheduler_data&&) = delete; @@ -731,7 +724,6 @@ namespace hpx::threads::policies { data_[num_thread].data_.queue_->create_thread(data, id, ec); break; - default: case thread_priority::unknown: { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, @@ -879,6 +871,7 @@ namespace hpx::threads::policies { d.queue_->increment_num_stolen_from_pending(); #endif thrds.tasks_.push_back(HPX_MOVE(thrd)); + thrd = thread_id_ref_type{}; } // we are ready to send at least one task @@ -905,7 +898,7 @@ namespace hpx::threads::policies { // Return the next thread to be executed, return false if none is // available bool get_next_thread(std::size_t num_thread, bool running, - thread_id_ref_type& thrd, bool enable_stealing) + thread_id_ref_type& thrd, bool allow_stealing) { HPX_ASSERT(num_thread < num_queues_); @@ -946,7 +939,7 @@ namespace hpx::threads::policies { #endif } - if (enable_stealing && result) + if (allow_stealing && result) { // We found a task to run, however before running it we handle // steal requests (assuming that there is more work left that @@ -986,7 +979,7 @@ namespace hpx::threads::policies { bool allow_fallback = false, thread_priority priority = thread_priority::default_) override { - std::size_t num_thread = static_cast(-1); + auto num_thread = static_cast(-1); if (schedulehint.mode == thread_schedule_hint_mode::thread) { num_thread = schedulehint.hint; @@ -1041,7 +1034,6 @@ namespace hpx::threads::policies { HPX_MOVE(thrd)); break; - default: case thread_priority::unknown: { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, @@ -1056,7 +1048,7 @@ namespace hpx::threads::policies { bool allow_fallback = false, thread_priority priority = thread_priority::default_) override { - std::size_t num_thread = static_cast(-1); + auto num_thread = static_cast(-1); if (schedulehint.mode == thread_schedule_hint_mode::thread) { num_thread = schedulehint.hint; @@ -1101,7 +1093,6 @@ namespace hpx::threads::policies { low_priority_queue_.schedule_thread(HPX_MOVE(thrd), true); break; - default: case thread_priority::default_: case thread_priority::normal: data_[num_thread].data_.queue_->schedule_thread( @@ -1112,6 +1103,13 @@ namespace hpx::threads::policies { data_[num_thread].data_.bound_queue_->schedule_thread( HPX_MOVE(thrd), true); break; + + case thread_priority::unknown: + { + HPX_THROW_EXCEPTION(hpx::error::bad_parameter, + "local_workrequesting_scheduler::schedule_thread_last", + "unknown thread priority value (thread_priority::unknown)"); + } } } @@ -1215,7 +1213,6 @@ namespace hpx::threads::policies { break; } - default: case thread_priority::unknown: { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, @@ -1282,7 +1279,6 @@ namespace hpx::threads::policies { break; } - default: case thread_priority::unknown: { HPX_THROW_EXCEPTION(hpx::error::bad_parameter, @@ -1556,7 +1552,7 @@ namespace hpx::threads::policies { if (d.num_recent_steals_ >= scheduler_data::num_steal_adaptive_interval_) { - double ratio = + double const ratio = static_cast(d.num_recent_tasks_executed_) / d.num_steal_adaptive_interval_; @@ -1662,9 +1658,8 @@ namespace hpx::threads::policies { // scheduler. Returns true if the OS thread calling this function has to // be terminated (i.e. no more work has to be done). bool wait_or_add_new(std::size_t num_thread, bool running, - [[maybe_unused]] std::int64_t& idle_loop_count, - bool enable_stealing, std::size_t& added, - thread_id_ref_type* next_thrd = nullptr) + [[maybe_unused]] std::int64_t& idle_loop_count, bool allow_stealing, + std::size_t& added, thread_id_ref_type* next_thrd = nullptr) { HPX_ASSERT(num_thread < num_queues_); @@ -1676,7 +1671,7 @@ namespace hpx::threads::policies { // threads as these threads are never created 'staged'. bool result = - d.queue_->wait_or_add_new(running, added, enable_stealing); + d.queue_->wait_or_add_new(running, added, allow_stealing); // check if work was available if (0 != added) @@ -1693,7 +1688,7 @@ namespace hpx::threads::policies { return true; // return if no stealing is requested (or not possible) - if (num_queues_ == 1 || !enable_stealing) + if (num_queues_ == 1 || !allow_stealing) return result; // attempt to steal more work @@ -1769,103 +1764,12 @@ namespace hpx::threads::policies { low_priority_queue_.on_start_thread(num_thread); } - std::size_t const num_threads = num_queues_; - //auto const& topo = create_topology(); - // Initially set all bits, code below resets the bits corresponding // to cores that can serve as a victim for the current core. A set // bit in this mask means 'do not steal from this core'. - resize(d.victims_, num_threads); + resize(d.victims_, num_queues_); reset(d.victims_); set(d.victims_, num_thread); - //for (std::size_t i = 0; i != num_threads; ++i) - //{ - // set(d.victims_, i); - //} - // - //// get NUMA domain masks of all queues... - //std::vector numa_masks(num_threads); - //std::vector numa_domains(num_threads); - //std::vector core_masks(num_threads); - //for (std::size_t i = 0; i != num_threads; ++i) - //{ - // std::size_t num_pu = affinity_data_.get_pu_num(i); - // numa_masks[i] = topo.get_numa_node_affinity_mask(num_pu); - // numa_domains[i] = static_cast( - // topo.get_numa_node_number(num_pu)); - // core_masks[i] = topo.get_core_affinity_mask(num_pu); - //} - // - //// iterate over the number of threads again to determine where to - //// steal from - //std::ptrdiff_t radius = - // std::lround(static_cast(num_threads) / 2.0); - // - //mask_cref_type numa_mask = numa_masks[num_thread]; - //mask_cref_type core_mask = core_masks[num_thread]; - // - //auto iterate = [&](auto&& f) { - // // check our neighbors in a radial fashion (left and right - // // alternating, increasing distance each iteration) - // std::ptrdiff_t i = 1; - // for (/**/; i < radius; ++i) - // { - // std::ptrdiff_t left = - // (static_cast(num_thread) - i) % - // static_cast(num_threads); - // if (left < 0) - // left = num_threads + left; - // - // if (f(std::size_t(left))) - // { - // unset(data_[num_thread].data_.victims_, - // static_cast(left)); - // } - // - // std::size_t right = (num_thread + i) % num_threads; - // if (f(right)) - // { - // unset(data_[num_thread].data_.victims_, right); - // } - // } - // if ((num_threads % 2) == 0) - // { - // std::size_t right = (num_thread + i) % num_threads; - // if (f(right)) - // { - // unset(data_[num_thread].data_.victims_, right); - // } - // } - //}; - // - //// check for threads that share the same core... - //iterate([&](std::size_t other_num_thread) { - // return any(core_mask & core_masks[other_num_thread]); - //}); - // - //// check for threads that share the same NUMA domain... - //iterate([&](std::size_t other_num_thread) { - // return !any(core_mask & core_masks[other_num_thread]) && - // any(numa_mask & numa_masks[other_num_thread]); - //}); - // - //// check for the rest and if we are NUMA aware - //if (has_scheduler_mode( - // policies::scheduler_mode::enable_stealing_numa)) - //{ - // iterate([&](std::size_t other_num_thread) { - // // allow stealing from neighboring NUMA domain only - // std::ptrdiff_t numa_distance = numa_domains[num_thread] - - // numa_domains[other_num_thread]; - // if (numa_distance > 1 || numa_distance < -1) - // return false; - // // steal from even cores from neighboring NUMA domains - // if (numa_distance == 1 || numa_distance == -1) - // return other_num_thread % 2 == 0; - // // cores from our domain are handled above - // return false; - // }); - //} } void on_stop_thread(std::size_t num_thread) override From 083048caa3cd593fbedcaf1c04059d4d729b1f0f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 28 Aug 2023 10:10:30 -0500 Subject: [PATCH 197/333] Reapply necessary fixes to scheduler --- .../local_workrequesting_scheduler.hpp | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp index 57b737fbae0e..bd9909fb9472 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp @@ -1675,7 +1675,9 @@ namespace hpx::threads::policies { // check if work was available if (0 != added) + { return result; + } if (num_thread == num_queues_ - 1) { @@ -1683,26 +1685,29 @@ namespace hpx::threads::policies { result; } - // check if we have been disabled - if (!running) - return true; - - // return if no stealing is requested (or not possible) - if (num_queues_ == 1 || !allow_stealing) - return result; + // check if we have been disabled or if no stealing is requested (or + // not possible) + if (!running || num_queues_ == 1) + { + return !running; + } // attempt to steal more work - send_steal_request(d); - HPX_ASSERT(d.requested_ != 0); + if (allow_stealing) + { + send_steal_request(d); + HPX_ASSERT(d.requested_ != 0); + } - // now try to handle steal requests again if we have not received a - // task from some other core yet - if (!try_receiving_tasks(d, added, next_thrd)) + if (try_receiving_tasks(d, added, next_thrd)) { - // decline or forward all pending steal requests - decline_or_forward_all_steal_requests(d); + return false; } + // if we did not receive any new task, decline or forward all + // pending steal requests + decline_or_forward_all_steal_requests(d); + #ifdef HPX_HAVE_THREAD_MINIMAL_DEADLOCK_DETECTION // no new work is available, are we deadlocked? if (HPX_UNLIKELY(get_minimal_deadlock_detection_enabled() && From 5d01676c3940add8dc355bc99a3e120672392b24 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Thu, 31 Aug 2023 14:08:43 +0200 Subject: [PATCH 198/333] Remove aurianer, biddisco, and msimberg from codeowners --- .github/CODEOWNERS | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4e1c30a2eaf8..4dd353e8627a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,10 +4,4 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -* @aurianer @biddisco @hkaiser @msimberg - -/docs/ @msimberg -/libs/**/*.txt @aurianer @msimberg -/libs/**/*.rst @aurianer @msimberg -/libs/**/*.py @aurianer @msimberg -/libs/**/*.sh @aurianer @msimberg +* @hkaiser From 75c3299745d17f0069f4f09b23b76b66a7484be3 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Fri, 1 Sep 2023 15:12:04 -0500 Subject: [PATCH 199/333] Fix shared_future_continuation_order regression test --- .../shared_future_continuation_order.cpp | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/libs/core/futures/tests/regressions/shared_future_continuation_order.cpp b/libs/core/futures/tests/regressions/shared_future_continuation_order.cpp index 1fe2b34c27f6..a9b90950d309 100644 --- a/libs/core/futures/tests/regressions/shared_future_continuation_order.cpp +++ b/libs/core/futures/tests/regressions/shared_future_continuation_order.cpp @@ -21,15 +21,17 @@ int hpx_main() hpx::promise p; hpx::shared_future f1 = p.get_future(); - hpx::future f2 = f1.then([](hpx::shared_future&& f) { - HPX_TEST_EQ(f.get(), 42); - return ++invocation_count; - }); - - hpx::future f3 = f1.then([](hpx::shared_future&& f) { - HPX_TEST_EQ(f.get(), 42); - return ++invocation_count; - }); + hpx::future f2 = + f1.then(hpx::launch::sync, [](hpx::shared_future&& f) { + HPX_TEST_EQ(f.get(), 42); + return ++invocation_count; + }); + + hpx::future f3 = + f1.then(hpx::launch::sync, [](hpx::shared_future&& f) { + HPX_TEST_EQ(f.get(), 42); + return ++invocation_count; + }); p.set_value(42); From d796e172dd2edb10507f3520762020f2bec955b2 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 31 Aug 2023 09:44:48 -0500 Subject: [PATCH 200/333] Fixing HPX_CURRENT_SOURCE_LOCATION when std::source_location exists --- libs/core/assertion/include/hpx/assertion/source_location.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/assertion/include/hpx/assertion/source_location.hpp b/libs/core/assertion/include/hpx/assertion/source_location.hpp index 5c98f9276ddc..58a67b1841ee 100644 --- a/libs/core/assertion/include/hpx/assertion/source_location.hpp +++ b/libs/core/assertion/include/hpx/assertion/source_location.hpp @@ -90,7 +90,7 @@ namespace hpx::assertion { } #if defined(HPX_HAVE_CXX20_SOURCE_LOCATION) -#define HPX_CURRENT_SOURCE_LOCATION() ::hpx::source_location::current() +#define HPX_CURRENT_SOURCE_LOCATION() std::source_location::current() #else #define HPX_CURRENT_SOURCE_LOCATION() \ ::hpx::source_location \ From 84a341145d2ad11b2abed02b151726b559cb655d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:59:57 +0000 Subject: [PATCH 201/333] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cff-validator.yml | 2 +- .github/workflows/linux_bors.yml | 2 +- .github/workflows/linux_crosscompile_arm_eve_sve_release.yml | 2 +- .github/workflows/linux_crosscompile_arm_sve_release.yml | 2 +- .github/workflows/linux_debug.yml | 2 +- .github/workflows/linux_hip.yml | 2 +- .github/workflows/linux_release_fetchcontent.yml | 2 +- .github/workflows/linux_release_static.yml | 2 +- .github/workflows/linux_sanitizers.yml | 2 +- .github/workflows/macos_debug.yml | 2 +- .github/workflows/windows_clang_debug.yml | 2 +- .github/workflows/windows_clang_release.yml | 2 +- .github/workflows/windows_debug_vs2019.yml | 2 +- .github/workflows/windows_debug_vs2022.yml | 2 +- .github/workflows/windows_release_2019.yml | 2 +- .github/workflows/windows_release_2022.yml | 2 +- .github/workflows/windows_release_gcc_mingw.yml | 2 +- .github/workflows/windows_release_static.yml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cff-validator.yml b/.github/workflows/cff-validator.yml index 542f55d7cd17..9f960c23867b 100644 --- a/.github/workflows/cff-validator.yml +++ b/.github/workflows/cff-validator.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # This is needed for workflows running on # ubuntu-20.04 or later diff --git a/.github/workflows/linux_bors.yml b/.github/workflows/linux_bors.yml index b16f85c8a393..65ae0101db40 100644 --- a/.github/workflows/linux_bors.yml +++ b/.github/workflows/linux_bors.yml @@ -19,7 +19,7 @@ jobs: container: stellargroup/build_env:14 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_crosscompile_arm_eve_sve_release.yml b/.github/workflows/linux_crosscompile_arm_eve_sve_release.yml index 491f1896d966..164e8b988863 100644 --- a/.github/workflows/linux_crosscompile_arm_eve_sve_release.yml +++ b/.github/workflows/linux_crosscompile_arm_eve_sve_release.yml @@ -14,7 +14,7 @@ jobs: container: stellargroup/crosscompile_arm_sve_build_env:1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_crosscompile_arm_sve_release.yml b/.github/workflows/linux_crosscompile_arm_sve_release.yml index 7ef23c6d876d..d5b265a2777c 100644 --- a/.github/workflows/linux_crosscompile_arm_sve_release.yml +++ b/.github/workflows/linux_crosscompile_arm_sve_release.yml @@ -14,7 +14,7 @@ jobs: container: stellargroup/crosscompile_arm_sve_build_env:1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_debug.yml b/.github/workflows/linux_debug.yml index 06af41b74798..c6c1685f0250 100644 --- a/.github/workflows/linux_debug.yml +++ b/.github/workflows/linux_debug.yml @@ -14,7 +14,7 @@ jobs: container: stellargroup/build_env:14 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_hip.yml b/.github/workflows/linux_hip.yml index 75eae8342729..de9a3d6bb33d 100644 --- a/.github/workflows/linux_hip.yml +++ b/.github/workflows/linux_hip.yml @@ -14,7 +14,7 @@ jobs: container: stellargroup/hip_build_env:10 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_release_fetchcontent.yml b/.github/workflows/linux_release_fetchcontent.yml index f28966a54cdf..be9ad924ae56 100644 --- a/.github/workflows/linux_release_fetchcontent.yml +++ b/.github/workflows/linux_release_fetchcontent.yml @@ -14,7 +14,7 @@ jobs: container: stellargroup/build_env:14 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_release_static.yml b/.github/workflows/linux_release_static.yml index 6ff33a5fb9d3..0d4f42536371 100644 --- a/.github/workflows/linux_release_static.yml +++ b/.github/workflows/linux_release_static.yml @@ -15,7 +15,7 @@ jobs: container: stellargroup/build_env:16 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/linux_sanitizers.yml b/.github/workflows/linux_sanitizers.yml index b1d8aff097c9..27bfd611f597 100644 --- a/.github/workflows/linux_sanitizers.yml +++ b/.github/workflows/linux_sanitizers.yml @@ -14,7 +14,7 @@ jobs: container: stellargroup/build_env:14 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure shell: bash run: | diff --git a/.github/workflows/macos_debug.yml b/.github/workflows/macos_debug.yml index 1c28635d4976..3ddb09f960f0 100644 --- a/.github/workflows/macos_debug.yml +++ b/.github/workflows/macos_debug.yml @@ -13,7 +13,7 @@ jobs: runs-on: macos-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install dependencies run: | # Workaround for https://github.com/actions/virtual-environments/issues/2322 diff --git a/.github/workflows/windows_clang_debug.yml b/.github/workflows/windows_clang_debug.yml index 6e7b5a64c7ab..8fe001689b3f 100644 --- a/.github/workflows/windows_clang_debug.yml +++ b/.github/workflows/windows_clang_debug.yml @@ -14,7 +14,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_clang_release.yml b/.github/workflows/windows_clang_release.yml index 5549002e8b86..3ef92e4facbf 100644 --- a/.github/workflows/windows_clang_release.yml +++ b/.github/workflows/windows_clang_release.yml @@ -14,7 +14,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_debug_vs2019.yml b/.github/workflows/windows_debug_vs2019.yml index 1e87dbd25c3b..a62663506f9e 100644 --- a/.github/workflows/windows_debug_vs2019.yml +++ b/.github/workflows/windows_debug_vs2019.yml @@ -13,7 +13,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_debug_vs2022.yml b/.github/workflows/windows_debug_vs2022.yml index f5c13a11d2e0..eeda51abfd8c 100644 --- a/.github/workflows/windows_debug_vs2022.yml +++ b/.github/workflows/windows_debug_vs2022.yml @@ -13,7 +13,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_release_2019.yml b/.github/workflows/windows_release_2019.yml index 8fa287234a62..785ab87afa4b 100644 --- a/.github/workflows/windows_release_2019.yml +++ b/.github/workflows/windows_release_2019.yml @@ -13,7 +13,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_release_2022.yml b/.github/workflows/windows_release_2022.yml index 3b3e23af3285..d89d9c973f41 100644 --- a/.github/workflows/windows_release_2022.yml +++ b/.github/workflows/windows_release_2022.yml @@ -13,7 +13,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_release_gcc_mingw.yml b/.github/workflows/windows_release_gcc_mingw.yml index dbc4419ef9a3..e3100cbc1060 100644 --- a/.github/workflows/windows_release_gcc_mingw.yml +++ b/.github/workflows/windows_release_gcc_mingw.yml @@ -13,7 +13,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' diff --git a/.github/workflows/windows_release_static.yml b/.github/workflows/windows_release_static.yml index 0367c8fd8fcd..c5318bf9d115 100644 --- a/.github/workflows/windows_release_static.yml +++ b/.github/workflows/windows_release_static.yml @@ -14,7 +14,7 @@ jobs: runs-on: windows-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: '3.22.x' From 15f3f7e10c15da9b9b248bfbf9d38d101a25814e Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:09:36 +0300 Subject: [PATCH 202/333] similar type detection --- .../type_support/is_trivially_relocatable.hpp | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp index 5ece0827cf8d..ad9d74cbf115 100644 --- a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp @@ -42,6 +42,60 @@ namespace hpx { { }; + // Constness, Volatility, References, Arrays are ignored + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable + : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable + : is_trivially_relocatable + { + }; + + template + struct is_trivially_relocatable + : is_trivially_relocatable + { + }; + template inline constexpr bool is_trivially_relocatable_v = is_trivially_relocatable::value; From 0b5aa311d0a3ab5230e5c08613b04436fe345d1b Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:09:56 +0300 Subject: [PATCH 203/333] is_relocatable_from --- .../include/hpx/type_support/is_relocatable.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp index d31a92dd31a2..05e881bfc69c 100644 --- a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp @@ -15,6 +15,19 @@ namespace hpx { { }; + // ToTp(FromTp&&) must be well-formed + template + struct is_relocatable_from + : std::bool_constant< + std::is_constructible_v, FromTp> && + std::is_same_v, std::decay_t>> + { + }; + template inline constexpr bool is_relocatable_v = is_relocatable::value; + + template + inline constexpr bool is_relocatable_from_v = + is_relocatable_from::value; } // namespace hpx From 8166924b5915a1653f9d3a27e89308213de997e5 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:10:10 +0300 Subject: [PATCH 204/333] relocate_at algorithm --- .../include/hpx/type_support/relocate_at.hpp | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 libs/core/type_support/include/hpx/type_support/relocate_at.hpp diff --git a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp new file mode 100644 index 000000000000..036c4324910a --- /dev/null +++ b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp @@ -0,0 +1,119 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include + +#include +#include + +#if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) +#include +#endif + +namespace hpx { + + namespace detail { + template + struct destroy_guard + { + T* t; + explicit destroy_guard(T* t) + : t(t) + { + } + ~destroy_guard() + { + std::destroy_at(t); + } + }; + } // namespace detail + +#if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) + using std::relocate; + using std::relocate_at; +#else + + namespace detail { + + /* + The condition to use memmove is: + hpx::is_trivially_relocatable_v && + !std::is_volatile_v + + The reason for the volatile check is that memmove is not allowed to + change the value of a volatile object. + */ + + template + constexpr bool relocate_using_memmove = + hpx::is_trivially_relocatable_v && !std::is_volatile_v; + + template , int> = 0> + T* relocate_at_helper(T* src, T* dst) noexcept + { + void* dst_void = const_cast(static_cast(dst)); + void* src_void = const_cast(static_cast(src)); + + std::memmove(dst_void, src_void, sizeof(T)); + + return std::launder(dst); + }; + + // this is move and destroy + template , int> = 0> + T* relocate_at_helper(T* src, T* dst) noexcept( + // has non-throwing move constructor + std::is_nothrow_move_constructible_v) + { + destroy_guard g(src); + return hpx::construct_at(dst, HPX_MOVE(*src)); + }; + } // namespace detail + + template + T* relocate_at(T* src, T* dst) noexcept( + // noexcept if the memmove path is taken or if the move path is noexcept + noexcept(detail::relocate_at_helper(src, dst))) + { + static_assert(hpx::is_relocatable_v, + "new (dst) T(std::move(*src)) must be well-formed"); + + return detail::relocate_at_helper(src, dst); + } + + template + T relocate(T* src) noexcept(std::is_nothrow_move_constructible_v) + { + static_assert( + hpx::is_relocatable_v, "T(std::move(*src)) must be well-formed"); + + detail::destroy_guard g(src); + return HPX_MOVE(*src); + } + + /* + Memmove codegen. This part relies on UB, so it's not used. It's here for + reference. More info on this: + + https://quuxplusone.github.io/blog/2022/05/18/std-relocate/ + + template + T relocate(T* source) + { + auto magic = (T(*)(void*, size_t)) memcpy; + return magic(source, sizeof(T)); + } + */ + +#endif // defined(HPX_HAVE_P1144_STD_RELOCATE_AT) + +} // namespace hpx From de953f87ca5e3154487dbfcad7455513f065fe16 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:10:31 +0300 Subject: [PATCH 205/333] uninitialized_relocate algorithm --- libs/core/type_support/CMakeLists.txt | 2 + .../type_support/uninitialized_relocate.hpp | 179 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp diff --git a/libs/core/type_support/CMakeLists.txt b/libs/core/type_support/CMakeLists.txt index 74ef2795d0d2..1e931219e277 100644 --- a/libs/core/type_support/CMakeLists.txt +++ b/libs/core/type_support/CMakeLists.txt @@ -22,7 +22,9 @@ set(type_support_headers hpx/type_support/lazy_enable_if.hpp hpx/type_support/pack.hpp hpx/type_support/meta.hpp + hpx/type_support/relocate_at.hpp hpx/type_support/static.hpp + hpx/type_support/uninitialized_relocate.hpp hpx/type_support/unwrap_ref.hpp hpx/type_support/unused.hpp hpx/type_support/void_guard.hpp diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp new file mode 100644 index 000000000000..38242c3efead --- /dev/null +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -0,0 +1,179 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include + +#include +#include + +#if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) +#include +#endif + +namespace hpx { + +#if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) + using std::uninitialized_relocate; +#else + + namespace detail { + + enum struct relocate_strategy + { + buffer_memcpy = 0, + for_loop_nothrow, + for_loop_try_catch + }; + + template + struct choose_uninitialized_relocate_helper + { + // using in_type = typename std::remove_reference_t< + // decltype(*std::declval())>; + // using out_type = typename std::remove_reference_t< + // decltype(*std::declval())>; + + using in_type = typename std::iterator_traits::value_type; + using out_type = typename std::iterator_traits::value_type; + + constexpr static bool valid_relocation = + hpx::is_relocatable_from_v; + + constexpr static bool is_buffer_memcpyable = + hpx::is_trivially_relocatable_v && + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The important check + std::is_same_v, std::decay_t> && + // can only relocate between same types + !std::is_volatile_v && !std::is_volatile_v && + // volatile types are not memcpyable + std::is_pointer_v && std::is_pointer_v; + // ^^ the best we can do to check for contiguous iterators + + constexpr static bool can_move_construct_nothrow = + std::is_nothrow_constructible_v>; + // Checks if the move constructor is noexcept to skip + // the try-catch block + + // Using an enum to distinguish implementations + constexpr static relocate_strategy value = is_buffer_memcpyable ? + relocate_strategy::buffer_memcpy : + can_move_construct_nothrow ? + relocate_strategy::for_loop_nothrow : + relocate_strategy::for_loop_try_catch; + }; + + template ::value == + relocate_strategy::buffer_memcpy, + int> = 0> + FwdIter uninitialized_relocate_helper( + InIter first, InIter last, FwdIter dst) noexcept + { + auto n_objects = std::distance(first, last); + + if (n_objects != 0) + { + void* first_void = const_cast( + static_cast(std::addressof(*first))); + void* last_void = const_cast( + static_cast(std::addressof(*last))); + + void* dst_void = const_cast( + static_cast(std::addressof(*dst))); + + std::byte* first_byte = + reinterpret_cast(first_void); + std::byte* last_byte = reinterpret_cast(last_void); + + auto n_bytes = std::distance(first_byte, last_byte); + + // Ideally we would want to convey to the compiler + // That the new buffer actually contains objects + // within their lifetime. But this is not possible + // with current language features. + std::memmove(dst_void, first_void, n_bytes); + + dst += n_objects; + } + + return dst; + } + + template ::value == + relocate_strategy::for_loop_nothrow, + int> = 0> + FwdIter uninitialized_relocate_helper( + InIter first, InIter last, FwdIter dst) noexcept + { + for (; first != last; ++first, ++dst) + { + // the move + destroy version will be used + hpx::relocate_at(std::addressof(*first), std::addressof(*dst)); + } + + return dst; + } + + template ::value == + relocate_strategy::for_loop_try_catch, + int> = 0> + FwdIter uninitialized_relocate_helper( + InIter first, InIter last, FwdIter dst) + { + FwdIter original_dst = dst; + + for (; first != last; ++first, ++dst) + { + try + { + // the move + destroy version will be used + hpx::relocate_at( + std::addressof(*first), std::addressof(*dst)); + } + catch (...) + { + // destroy all objects other that the one + // that caused the exception + + // destroy all objects constructed so far + std::destroy(original_dst, dst); + // destroy all the objects not relocated yet + std::destroy(++first, last); + + throw; + } + } + + return dst; + } + + } // namespace detail + + template + FwdIter + uninitialized_relocate(InIter first, InIter last, FwdIter dst) noexcept( + detail::choose_uninitialized_relocate_helper::value != + detail::relocate_strategy::for_loop_try_catch) + { + static_assert(detail::choose_uninitialized_relocate_helper::valid_relocation, + "uninitialized_move(first, last, dst) must be well-formed"); + return detail::uninitialized_relocate_helper(first, last, dst); + } + +#endif // defined(HPX_HAVE_P1144_STD_RELOCATE_AT) + +} // namespace hpx From 0faba806e81f95337beea19d3c04838c97cabe27 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:10:49 +0300 Subject: [PATCH 206/333] tests --- .../type_support/tests/unit/CMakeLists.txt | 30 +- .../tests/unit/fail_relocate_at.cpp | 20 ++ .../unit/fail_uninitialized_relocate.cpp | 20 ++ .../tests/unit/is_trivially_relocatable.cpp | 35 +++ .../type_support/tests/unit/relocate_at.cpp | 134 +++++++++ .../tests/unit/uninitialized_relocate.cpp | 275 ++++++++++++++++++ 6 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 libs/core/type_support/tests/unit/fail_relocate_at.cpp create mode 100644 libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp create mode 100644 libs/core/type_support/tests/unit/relocate_at.cpp create mode 100644 libs/core/type_support/tests/unit/uninitialized_relocate.cpp diff --git a/libs/core/type_support/tests/unit/CMakeLists.txt b/libs/core/type_support/tests/unit/CMakeLists.txt index 29519a4d3852..a84d995b157e 100644 --- a/libs/core/type_support/tests/unit/CMakeLists.txt +++ b/libs/core/type_support/tests/unit/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -set(tests is_relocatable is_trivially_relocatable) +set(tests relocate_at uninitialized_relocate) if(HPX_WITH_CXX20_COROUTINES) set(tests ${tests} generator) @@ -29,3 +29,31 @@ foreach(test ${tests}) add_hpx_unit_test("modules.type_support" ${test} ${${test}_PARAMETERS}) endforeach() + +if(HPX_WITH_COMPILE_ONLY_TESTS) + # add compile time tests + set(compile_tests is_relocatable is_trivially_relocatable) + + if(HPX_WITH_FAIL_COMPILE_TESTS) + set(fail_compile_tests fail_relocate_at fail_uninitialized_relocate) + foreach(fail_compile_test ${fail_compile_tests}) + set(${fail_compile_test}_FLAGS FAILURE_EXPECTED) + endforeach() + + set(compile_tests ${compile_tests} ${fail_compile_tests}) + endif() + + foreach(compile_test ${compile_tests}) + set(sources ${compile_test}.cpp) + + source_group("Source Files" FILES ${sources}) + + add_hpx_unit_compile_test( + "modules.type_support" ${compile_test} + SOURCES ${sources} ${${compile_test}_FLAGS} + FOLDER "Tests/Unit/Modules/Core/TypeSupport/CompileOnly" + ) + + endforeach() + +endif() diff --git a/libs/core/type_support/tests/unit/fail_relocate_at.cpp b/libs/core/type_support/tests/unit/fail_relocate_at.cpp new file mode 100644 index 000000000000..95a5dffa1aab --- /dev/null +++ b/libs/core/type_support/tests/unit/fail_relocate_at.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + int a[10]; + int b[10]; + + int(*p)[10] = &a; + int(*q)[10] = &b; + + hpx::relocate_at(p, q); +} diff --git a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp new file mode 100644 index 000000000000..4f3b12ec3104 --- /dev/null +++ b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + int a[10]; + int b[10]; + + int(*p)[10] = &a; + int(*q)[10] = &b; + + hpx::uninitialized_relocate(p, p + 1, q); +} diff --git a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp index 050d4f5bdb2d..8609dc4136d6 100644 --- a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp +++ b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp @@ -125,6 +125,41 @@ static_assert( static_assert( hpx::is_trivially_relocatable_v); +// c-v-ref-array qualified versions of explicitly declared trivially relocatable +// types are trivially relocatable + +static_assert( + hpx::is_trivially_relocatable_v); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 volatile>); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 const volatile>); +static_assert( + hpx::is_trivially_relocatable_v); +static_assert( + hpx::is_trivially_relocatable_v); +static_assert( + hpx::is_trivially_relocatable_v); +static_assert( + hpx::is_trivially_relocatable_v); + +// Chain of c-v-ref-array qualifiers are supported +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1[10][10]>); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 const[10]>); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 volatile[10]>); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 const volatile[10]>); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 (&)[10]>); +static_assert( + hpx::is_trivially_relocatable_v); +static_assert(hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 const volatile&>); + // Trivial relocatability is not inherited struct derived_from_explicitly_trivially_relocatable : explicitly_trivially_relocatable_1 diff --git a/libs/core/type_support/tests/unit/relocate_at.cpp b/libs/core/type_support/tests/unit/relocate_at.cpp new file mode 100644 index 000000000000..a4791c5f97b0 --- /dev/null +++ b/libs/core/type_support/tests/unit/relocate_at.cpp @@ -0,0 +1,134 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include + +struct non_trivially_relocatable_struct +{ + static int count; + int data; + + explicit non_trivially_relocatable_struct(int data) + : data(data) + { + count++; + } + non_trivially_relocatable_struct(non_trivially_relocatable_struct&& other) + : data(other.data) + { + count++; + } + ~non_trivially_relocatable_struct() + { + count--; + } + + // making sure the address is never directly accessed + friend void operator&(non_trivially_relocatable_struct) = delete; +}; +int non_trivially_relocatable_struct::count = 0; + +static_assert( + !hpx::is_trivially_relocatable_v); + +struct trivially_relocatable_struct +{ + static int count; + int data; + + explicit trivially_relocatable_struct(int data) + : data(data) + { + count++; + } + trivially_relocatable_struct(trivially_relocatable_struct&& other) + : data(other.data) + { + count++; + } + ~trivially_relocatable_struct() + { + count--; + } + + friend void operator&(trivially_relocatable_struct) = delete; +}; +int trivially_relocatable_struct::count = 0; + +HPX_DECLARE_TRIVIALLY_RELOCATABLE(trivially_relocatable_struct); +static_assert(hpx::is_trivially_relocatable_v); + +int hpx_main() +{ + { + void* mem1 = std::malloc(sizeof(non_trivially_relocatable_struct)); + void* mem2 = std::malloc(sizeof(non_trivially_relocatable_struct)); + + HPX_TEST(mem1 && mem2); + + HPX_TEST(non_trivially_relocatable_struct::count == 0); + + non_trivially_relocatable_struct* ptr1 = hpx::construct_at( + static_cast(mem1), 1234); + + non_trivially_relocatable_struct* ptr2 = + static_cast(mem2); + + // a single object was constructed + HPX_TEST(non_trivially_relocatable_struct::count == 1); + + hpx::relocate_at(ptr1, ptr2); + + // count = 1 + 1 (from the move construction) - 1 (from the destruction) + HPX_TEST(non_trivially_relocatable_struct::count == 1); + HPX_TEST(ptr2->data == 1234); + + std::destroy_at(ptr2); + + std::free(mem1); + std::free(mem2); + } + { + void* mem1 = std::malloc(sizeof(trivially_relocatable_struct)); + void* mem2 = std::malloc(sizeof(trivially_relocatable_struct)); + + HPX_TEST(mem1 && mem2); + + HPX_TEST(trivially_relocatable_struct::count == 0); + + trivially_relocatable_struct* ptr1 = hpx::construct_at( + static_cast(mem1), 1234); + + trivially_relocatable_struct* ptr2 = + static_cast(mem2); + + // a single object was constructed + HPX_TEST(trivially_relocatable_struct::count == 1); + + hpx::relocate_at(ptr1, ptr2); + + // count = 1 + 0 (relocation on trivially relocatable + // objects does not trigger move constructors + // or destructors); no object is destroyed or created + HPX_TEST(trivially_relocatable_struct::count == 1); + HPX_TEST(ptr2->data == 1234); + + std::destroy_at(ptr2); + + std::free(mem1); + std::free(mem2); + } + return hpx::local::finalize(); +} + +int main(int argc, char* argv[]) +{ + hpx::local::init(hpx_main, argc, argv); + return hpx::util::report_errors(); +} diff --git a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp new file mode 100644 index 000000000000..bd1b7103e248 --- /dev/null +++ b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp @@ -0,0 +1,275 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include + +#define N 50 +#define K 10 + +struct trivially_relocatable_struct +{ + static int count; + static int move_count; + static int dtor_count; + int data; + + explicit trivially_relocatable_struct(int data) + : data(data) + { + count++; + } + trivially_relocatable_struct(trivially_relocatable_struct&& other) + : data(other.data) + { + move_count++; + count++; + } + ~trivially_relocatable_struct() + { + dtor_count++; + count--; + } + + // making sure the address is never directly accessed + friend void operator&(trivially_relocatable_struct) = delete; +}; +int trivially_relocatable_struct::count = 0; +int trivially_relocatable_struct::move_count = 0; +int trivially_relocatable_struct::dtor_count = 0; + +HPX_DECLARE_TRIVIALLY_RELOCATABLE(trivially_relocatable_struct); +static_assert(hpx::is_trivially_relocatable_v); + +struct non_trivially_relocatable_struct +{ + static int count; + static int move_count; + static int dtor_count; + int data; + + explicit non_trivially_relocatable_struct(int data) + : data(data) + { + count++; + } + // mark as noexcept to enter simpler relocation path + non_trivially_relocatable_struct( + non_trivially_relocatable_struct&& other) noexcept + : data(other.data) + { + move_count++; + count++; + } + ~non_trivially_relocatable_struct() + { + dtor_count++; + count--; + } + + // making sure the address is never directly accessed + friend void operator&(non_trivially_relocatable_struct) = delete; +}; +int non_trivially_relocatable_struct::count = 0; +int non_trivially_relocatable_struct::move_count = 0; +int non_trivially_relocatable_struct::dtor_count = 0; + +static_assert( + !hpx::is_trivially_relocatable_v); + +struct non_trivially_relocatable_struct_throwing +{ + static int count; + static int move_count; + static int dtor_count; + + int data; + + explicit non_trivially_relocatable_struct_throwing(int data) + : data(data) + { + count++; + } + // do not mark as noexcept to enter try-catch relocation path + non_trivially_relocatable_struct_throwing( + non_trivially_relocatable_struct_throwing&& other) + : data(other.data) + { + if (move_count == K) + { + throw 42; + } + move_count++; + + count++; + } + ~non_trivially_relocatable_struct_throwing() + { + dtor_count++; + count--; + } + + // making sure the address is never directly accessed + friend void operator&(non_trivially_relocatable_struct_throwing) = delete; +}; + +int non_trivially_relocatable_struct_throwing::count = 0; +int non_trivially_relocatable_struct_throwing::move_count = 0; +int non_trivially_relocatable_struct_throwing::dtor_count = 0; + +static_assert(!hpx::is_trivially_relocatable_v< + non_trivially_relocatable_struct_throwing>); + +int hpx_main() +{ + { + void* mem1 = std::malloc(N * sizeof(trivially_relocatable_struct)); + void* mem2 = std::malloc(N * sizeof(trivially_relocatable_struct)); + + HPX_TEST(mem1 && mem2); + + trivially_relocatable_struct* ptr1 = + static_cast(mem1); + trivially_relocatable_struct* ptr2 = + static_cast(mem2); + + HPX_TEST(trivially_relocatable_struct::count == 0); + HPX_TEST(trivially_relocatable_struct::move_count == 0); + HPX_TEST(trivially_relocatable_struct::dtor_count == 0); + + for (int i = 0; i < N; i++) + { + hpx::construct_at(ptr1 + i, 1234); + } + + // N objects constructed + HPX_TEST(trivially_relocatable_struct::count == N); + + // relocate them to ptr2 + hpx::uninitialized_relocate(ptr1, ptr1 + N, ptr2); + + // All creations - destructions balance out + HPX_TEST(trivially_relocatable_struct::count == N); + + // No move constructor or destructor should be called + HPX_TEST(trivially_relocatable_struct::move_count == 0); + HPX_TEST(trivially_relocatable_struct::dtor_count == 0); + + for (int i = 0; i < N; i++) + { + HPX_TEST(ptr2[i].data == 1234); + } + + std::destroy(ptr2, ptr2 + N); + + std::free(mem1); + std::free(mem2); + } + { + void* mem1 = std::malloc(N * sizeof(non_trivially_relocatable_struct)); + void* mem2 = std::malloc(N * sizeof(non_trivially_relocatable_struct)); + + HPX_TEST(mem1 && mem2); + + non_trivially_relocatable_struct* ptr1 = + static_cast(mem1); + non_trivially_relocatable_struct* ptr2 = + static_cast(mem2); + + HPX_TEST(non_trivially_relocatable_struct::count == 0); + HPX_TEST(non_trivially_relocatable_struct::move_count == 0); + HPX_TEST(non_trivially_relocatable_struct::dtor_count == 0); + + for (int i = 0; i < N; i++) + { + hpx::construct_at(ptr1 + i, 1234); + } + + // N objects constructed + HPX_TEST(non_trivially_relocatable_struct::count == N); + + // relocate them to ptr2 + hpx::uninitialized_relocate(ptr1, ptr1 + N, ptr2); + + // All creations - destructions balance out + HPX_TEST(non_trivially_relocatable_struct::count == N); + + // Every object was moved from and then destroyed + HPX_TEST(non_trivially_relocatable_struct::move_count == N); + HPX_TEST(non_trivially_relocatable_struct::dtor_count == N); + + for (int i = 0; i < N; i++) + { + HPX_TEST(ptr2[i].data == 1234); + } + + std::destroy(ptr2, ptr2 + N); + + std::free(mem1); + std::free(mem2); + } + { + void* mem1 = + std::malloc(N * sizeof(non_trivially_relocatable_struct_throwing)); + void* mem2 = + std::malloc(N * sizeof(non_trivially_relocatable_struct_throwing)); + + HPX_TEST(mem1 && mem2); + + non_trivially_relocatable_struct_throwing* ptr1 = + static_cast(mem1); + non_trivially_relocatable_struct_throwing* ptr2 = + static_cast(mem2); + + HPX_TEST(non_trivially_relocatable_struct_throwing::count == 0); + HPX_TEST(non_trivially_relocatable_struct_throwing::move_count == 0); + HPX_TEST(non_trivially_relocatable_struct_throwing::dtor_count == 0); + + for (int i = 0; i < N; i++) + { + hpx::construct_at(ptr1 + i, 1234); + } + + // N objects constructed + HPX_TEST(non_trivially_relocatable_struct_throwing::count == N); + + // relocate them to ptr2 + try + { + hpx::uninitialized_relocate(ptr1, ptr1 + N, ptr2); + HPX_TEST(false); // should never reach this + } + catch (int forty_two) + { + HPX_TEST(forty_two == 42); + } + + // K move constructors were called + HPX_TEST(non_trivially_relocatable_struct_throwing::move_count == K); + + // K - 1 destructors were called to balance out the move constructors + // (- 1 because the last move constructor throws) + // and then N + 1 destructors were called: K on the old range and + // N - (K - 1) = N - K + 1 on the new range + HPX_TEST( + non_trivially_relocatable_struct_throwing::dtor_count == N + K); + + // It stops at K, so K-1 move-destruct pairs have been executed + // after this N - (K - 1) destructs will be done on the old range + // and K - 1 on the new range. giving 2*N total destructs + + std::free(mem1); + std::free(mem2); + } + return hpx::local::finalize(); +} + +int main(int argc, char* argv[]) +{ + hpx::local::init(hpx_main, argc, argv); + return hpx::util::report_errors(); +} From e2b5e8012d92b025301c400316d599da3ad8e06a Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:55:07 +0300 Subject: [PATCH 207/333] remove commented code --- .../include/hpx/type_support/uninitialized_relocate.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp index 38242c3efead..7c8821d7ca09 100644 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -35,11 +35,6 @@ namespace hpx { template struct choose_uninitialized_relocate_helper { - // using in_type = typename std::remove_reference_t< - // decltype(*std::declval())>; - // using out_type = typename std::remove_reference_t< - // decltype(*std::declval())>; - using in_type = typename std::iterator_traits::value_type; using out_type = typename std::iterator_traits::value_type; From 42126fe3afe00199db0126e0a127899331aca9cb Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 21:18:35 +0300 Subject: [PATCH 208/333] reject failing destroy_at types --- .../type_support/include/hpx/type_support/relocate_at.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp index 036c4324910a..0af625db38d1 100644 --- a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp +++ b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp @@ -20,7 +20,13 @@ namespace hpx { namespace detail { - template +#if __cplusplus <= 201703L +// pre c++17 std::destroy_at can be used only on non-array types + template && !std::is_array_v)> +#else +// c++17 std::destroy_at can be used on array types, destructing each element + template )> +#endif struct destroy_guard { T* t; From 228e48c5b7509d0b639377bc20d327fc53bbf804 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 22:26:56 +0300 Subject: [PATCH 209/333] move to namespace experimental --- .../include/hpx/type_support/relocate_at.hpp | 54 ++++++++++--------- .../type_support/uninitialized_relocate.hpp | 6 +-- .../tests/unit/fail_relocate_at.cpp | 2 +- .../unit/fail_uninitialized_relocate.cpp | 2 +- .../type_support/tests/unit/relocate_at.cpp | 4 +- .../tests/unit/uninitialized_relocate.cpp | 6 +-- 6 files changed, 40 insertions(+), 34 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp index 0af625db38d1..aed949fd5642 100644 --- a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp +++ b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -17,35 +18,36 @@ #include #endif -namespace hpx { - - namespace detail { +namespace hpx::detail { #if __cplusplus <= 201703L -// pre c++17 std::destroy_at can be used only on non-array types - template && !std::is_array_v)> + // pre c++17 std::destroy_at can be used only on non-array types + template && !std::is_array_v)> #else -// c++17 std::destroy_at can be used on array types, destructing each element - template )> + // c++17 std::destroy_at can be used on array types, destructing each element + template )> #endif - struct destroy_guard + struct destroy_guard + { + T* t; + explicit destroy_guard(T* t) + : t(t) { - T* t; - explicit destroy_guard(T* t) - : t(t) - { - } - ~destroy_guard() - { - std::destroy_at(t); - } - }; - } // namespace detail + } + ~destroy_guard() + { + std::destroy_at(t); + } + }; +} // namespace hpx::detail #if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) - using std::relocate; - using std::relocate_at; +using std::relocate; +using std::relocate_at; #else +namespace hpx::experimental { + namespace detail { /* @@ -80,6 +82,8 @@ namespace hpx { // has non-throwing move constructor std::is_nothrow_move_constructible_v) { + using hpx::detail::destroy_guard; + destroy_guard g(src); return hpx::construct_at(dst, HPX_MOVE(*src)); }; @@ -102,7 +106,9 @@ namespace hpx { static_assert( hpx::is_relocatable_v, "T(std::move(*src)) must be well-formed"); - detail::destroy_guard g(src); + using hpx::detail::destroy_guard; + + destroy_guard g(src); return HPX_MOVE(*src); } @@ -120,6 +126,6 @@ namespace hpx { } */ -#endif // defined(HPX_HAVE_P1144_STD_RELOCATE_AT) +#endif // !defined(HPX_HAVE_P1144_STD_RELOCATE_AT) -} // namespace hpx +} // namespace hpx::experimental diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp index 7c8821d7ca09..af00b73926b0 100644 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -17,7 +17,7 @@ #include #endif -namespace hpx { +namespace hpx::experimental { #if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) using std::uninitialized_relocate; @@ -114,7 +114,7 @@ namespace hpx { for (; first != last; ++first, ++dst) { // the move + destroy version will be used - hpx::relocate_at(std::addressof(*first), std::addressof(*dst)); + hpx::experimental::relocate_at(std::addressof(*first), std::addressof(*dst)); } return dst; @@ -135,7 +135,7 @@ namespace hpx { try { // the move + destroy version will be used - hpx::relocate_at( + hpx::experimental::relocate_at( std::addressof(*first), std::addressof(*dst)); } catch (...) diff --git a/libs/core/type_support/tests/unit/fail_relocate_at.cpp b/libs/core/type_support/tests/unit/fail_relocate_at.cpp index 95a5dffa1aab..875443d1e705 100644 --- a/libs/core/type_support/tests/unit/fail_relocate_at.cpp +++ b/libs/core/type_support/tests/unit/fail_relocate_at.cpp @@ -16,5 +16,5 @@ int main(int argc, char* argv[]) int(*p)[10] = &a; int(*q)[10] = &b; - hpx::relocate_at(p, q); + hpx::experimental::relocate_at(p, q); } diff --git a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp index 4f3b12ec3104..ea0c533f1cbe 100644 --- a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp +++ b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp @@ -16,5 +16,5 @@ int main(int argc, char* argv[]) int(*p)[10] = &a; int(*q)[10] = &b; - hpx::uninitialized_relocate(p, p + 1, q); + hpx::experimental::uninitialized_relocate(p, p + 1, q); } diff --git a/libs/core/type_support/tests/unit/relocate_at.cpp b/libs/core/type_support/tests/unit/relocate_at.cpp index a4791c5f97b0..7e99db32ec5f 100644 --- a/libs/core/type_support/tests/unit/relocate_at.cpp +++ b/libs/core/type_support/tests/unit/relocate_at.cpp @@ -83,7 +83,7 @@ int hpx_main() // a single object was constructed HPX_TEST(non_trivially_relocatable_struct::count == 1); - hpx::relocate_at(ptr1, ptr2); + hpx::experimental::relocate_at(ptr1, ptr2); // count = 1 + 1 (from the move construction) - 1 (from the destruction) HPX_TEST(non_trivially_relocatable_struct::count == 1); @@ -111,7 +111,7 @@ int hpx_main() // a single object was constructed HPX_TEST(trivially_relocatable_struct::count == 1); - hpx::relocate_at(ptr1, ptr2); + hpx::experimental::relocate_at(ptr1, ptr2); // count = 1 + 0 (relocation on trivially relocatable // objects does not trigger move constructors diff --git a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp index bd1b7103e248..fec880557ff5 100644 --- a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp +++ b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp @@ -150,7 +150,7 @@ int hpx_main() HPX_TEST(trivially_relocatable_struct::count == N); // relocate them to ptr2 - hpx::uninitialized_relocate(ptr1, ptr1 + N, ptr2); + hpx::experimental::uninitialized_relocate(ptr1, ptr1 + N, ptr2); // All creations - destructions balance out HPX_TEST(trivially_relocatable_struct::count == N); @@ -193,7 +193,7 @@ int hpx_main() HPX_TEST(non_trivially_relocatable_struct::count == N); // relocate them to ptr2 - hpx::uninitialized_relocate(ptr1, ptr1 + N, ptr2); + hpx::experimental::uninitialized_relocate(ptr1, ptr1 + N, ptr2); // All creations - destructions balance out HPX_TEST(non_trivially_relocatable_struct::count == N); @@ -240,7 +240,7 @@ int hpx_main() // relocate them to ptr2 try { - hpx::uninitialized_relocate(ptr1, ptr1 + N, ptr2); + hpx::experimental::uninitialized_relocate(ptr1, ptr1 + N, ptr2); HPX_TEST(false); // should never reach this } catch (int forty_two) From 8e2e89cfc2e5225ad85278bffa1b74af9cd4921e Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Sat, 19 Aug 2023 17:10:58 +0300 Subject: [PATCH 210/333] make relocate's format like relocate_at --- .../include/hpx/type_support/relocate_at.hpp | 56 +++++++++++++------ 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp index aed949fd5642..baa7696ccda9 100644 --- a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp +++ b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp @@ -87,6 +87,43 @@ namespace hpx::experimental { destroy_guard g(src); return hpx::construct_at(dst, HPX_MOVE(*src)); }; + + template + T relocate_helper(T* src) noexcept( + std::is_nothrow_move_constructible_v) + { + using hpx::detail::destroy_guard; + + destroy_guard g(src); + return HPX_MOVE(*src); + } + + /* + P1144 also proposes a version of relocate that does not call the + move constructor and instead memmoves the bytes of src to dest. + + Giving an interface like: + + T dest = relocate(std::addressof(src)); + + That results in a valid T object (dest) without calling any + constructor or destructor. + + This is not possible to do with the current C++ standard. + + One of the proposed ways to implement this uses a hypothetical + attribute "do_not_construct" and NRVO. + + Implementation: + + template , int> = 0> + T relocate(T* source) + { + __attribute__((do_not_construct)) T t; // hypothetical attribute + std::memmove(std::addressof(t), source, sizeof(T)); + return t; // NRVO + } + */ } // namespace detail template @@ -106,25 +143,8 @@ namespace hpx::experimental { static_assert( hpx::is_relocatable_v, "T(std::move(*src)) must be well-formed"); - using hpx::detail::destroy_guard; - - destroy_guard g(src); - return HPX_MOVE(*src); - } - - /* - Memmove codegen. This part relies on UB, so it's not used. It's here for - reference. More info on this: - - https://quuxplusone.github.io/blog/2022/05/18/std-relocate/ - - template - T relocate(T* source) - { - auto magic = (T(*)(void*, size_t)) memcpy; - return magic(source, sizeof(T)); + return detail::relocate_helper(src); } - */ #endif // !defined(HPX_HAVE_P1144_STD_RELOCATE_AT) From bbe9dc8ba3483d14ac10a207a40b4cb11f0ee6f6 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Sat, 19 Aug 2023 17:13:18 +0300 Subject: [PATCH 211/333] relocate test --- .../type_support/tests/unit/CMakeLists.txt | 2 +- .../core/type_support/tests/unit/relocate.cpp | 74 +++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 libs/core/type_support/tests/unit/relocate.cpp diff --git a/libs/core/type_support/tests/unit/CMakeLists.txt b/libs/core/type_support/tests/unit/CMakeLists.txt index a84d995b157e..83886f941586 100644 --- a/libs/core/type_support/tests/unit/CMakeLists.txt +++ b/libs/core/type_support/tests/unit/CMakeLists.txt @@ -4,7 +4,7 @@ # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -set(tests relocate_at uninitialized_relocate) +set(tests relocate relocate_at uninitialized_relocate) if(HPX_WITH_CXX20_COROUTINES) set(tests ${tests} generator) diff --git a/libs/core/type_support/tests/unit/relocate.cpp b/libs/core/type_support/tests/unit/relocate.cpp new file mode 100644 index 000000000000..3871598b40e2 --- /dev/null +++ b/libs/core/type_support/tests/unit/relocate.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2023 Isidoros Tsaousis-Seiras +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +/* + This test checks that the relocate_at function works correctly + for non-trivially relocatable types. + + The trivially relocatable optimization can not be implemented yet + so it is not tested separately. +*/ + +#include +#include +#include +#include + +struct non_trivially_relocatable_struct +{ + static int count; + int data; + + explicit non_trivially_relocatable_struct(int data) + : data(data) + { + count++; + } + non_trivially_relocatable_struct(non_trivially_relocatable_struct&& other) + : data(other.data) + { + count++; + } + ~non_trivially_relocatable_struct() + { + count--; + } + + // making sure the address is never directly accessed + friend void operator&(non_trivially_relocatable_struct) = delete; +}; +int non_trivially_relocatable_struct::count = 0; + +int hpx_main() +{ + void* mem1 = std::malloc(sizeof(non_trivially_relocatable_struct)); + + HPX_TEST(mem1); + + HPX_TEST(non_trivially_relocatable_struct::count == 0); + + non_trivially_relocatable_struct* ptr1 = hpx::construct_at( + static_cast(mem1), 1234); + + // a single object was constructed + HPX_TEST(non_trivially_relocatable_struct::count == 1); + + non_trivially_relocatable_struct obj2 = hpx::experimental::relocate(ptr1); + + // count = 1 + 1 (from the move construction) - 1 (from the destruction) + HPX_TEST(non_trivially_relocatable_struct::count == 1); + HPX_TEST(obj2.data == 1234); + + std::free(mem1); + + return hpx::local::finalize(); +} + +int main(int argc, char* argv[]) +{ + hpx::local::init(hpx_main, argc, argv); + return hpx::util::report_errors(); +} From 9d496a536b391460ae6f4196217f3e06394fd012 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Sat, 19 Aug 2023 17:14:13 +0300 Subject: [PATCH 212/333] clang-format and comments --- .../include/hpx/type_support/uninitialized_relocate.hpp | 5 +++-- libs/core/type_support/tests/unit/fail_relocate_at.cpp | 2 ++ .../type_support/tests/unit/fail_uninitialized_relocate.cpp | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp index af00b73926b0..7e6d6044d4b4 100644 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -114,7 +114,8 @@ namespace hpx::experimental { for (; first != last; ++first, ++dst) { // the move + destroy version will be used - hpx::experimental::relocate_at(std::addressof(*first), std::addressof(*dst)); + hpx::experimental::relocate_at( + std::addressof(*first), std::addressof(*dst)); } return dst; @@ -171,4 +172,4 @@ namespace hpx::experimental { #endif // defined(HPX_HAVE_P1144_STD_RELOCATE_AT) -} // namespace hpx +} // namespace hpx::experimental diff --git a/libs/core/type_support/tests/unit/fail_relocate_at.cpp b/libs/core/type_support/tests/unit/fail_relocate_at.cpp index 875443d1e705..f4d9881fc8cf 100644 --- a/libs/core/type_support/tests/unit/fail_relocate_at.cpp +++ b/libs/core/type_support/tests/unit/fail_relocate_at.cpp @@ -4,6 +4,8 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// This test should fail to compile + #include #include #include diff --git a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp index ea0c533f1cbe..9c4110838a28 100644 --- a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp +++ b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp @@ -4,6 +4,8 @@ // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// This test should fail to compile + #include #include #include From e31a9725967fe20c8e2abe70383567c7c55f651a Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Sat, 19 Aug 2023 18:22:59 +0300 Subject: [PATCH 213/333] remove whitespaces for inspect --- .../type_support/include/hpx/type_support/relocate_at.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp index baa7696ccda9..86da478e9f62 100644 --- a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp +++ b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp @@ -99,16 +99,16 @@ namespace hpx::experimental { } /* - P1144 also proposes a version of relocate that does not call the + P1144 also proposes a version of relocate that does not call the move constructor and instead memmoves the bytes of src to dest. Giving an interface like: T dest = relocate(std::addressof(src)); - That results in a valid T object (dest) without calling any + That results in a valid T object (dest) without calling any constructor or destructor. - + This is not possible to do with the current C++ standard. One of the proposed ways to implement this uses a hypothetical From df7cd7587853b0ee564d2d1f00e8fb067c4095b4 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 13:28:58 +0300 Subject: [PATCH 214/333] fix fail test --- libs/core/type_support/tests/unit/fail_relocate_at.cpp | 1 - .../core/type_support/tests/unit/fail_uninitialized_relocate.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/libs/core/type_support/tests/unit/fail_relocate_at.cpp b/libs/core/type_support/tests/unit/fail_relocate_at.cpp index f4d9881fc8cf..be88f03d36ee 100644 --- a/libs/core/type_support/tests/unit/fail_relocate_at.cpp +++ b/libs/core/type_support/tests/unit/fail_relocate_at.cpp @@ -8,7 +8,6 @@ #include #include -#include int main(int argc, char* argv[]) { diff --git a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp index 9c4110838a28..affcb672fd67 100644 --- a/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp +++ b/libs/core/type_support/tests/unit/fail_uninitialized_relocate.cpp @@ -8,7 +8,6 @@ #include #include -#include int main(int argc, char* argv[]) { From baf6170fa9ae87a2322c414369dbfba5c5c8eb0c Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 13:32:53 +0300 Subject: [PATCH 215/333] & and && are nor triv. rel. --- .../type_support/is_trivially_relocatable.hpp | 29 ++++++++++--------- .../tests/unit/is_trivially_relocatable.cpp | 27 ++++++++++------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp index ad9d74cbf115..a5587a963bfa 100644 --- a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp @@ -42,50 +42,53 @@ namespace hpx { { }; - // Constness, Volatility, References, Arrays are ignored + // References are not trivially relocatable template - struct is_trivially_relocatable : is_trivially_relocatable + struct is_trivially_relocatable : std::false_type { }; + // Temporary objects are not trivially relocatable template - struct is_trivially_relocatable : is_trivially_relocatable + struct is_trivially_relocatable : std::false_type { }; + // Constness, Volatility, Arrays are ignored template - struct is_trivially_relocatable - : is_trivially_relocatable + struct is_trivially_relocatable : is_trivially_relocatable { }; template - struct is_trivially_relocatable : is_trivially_relocatable + struct is_trivially_relocatable : is_trivially_relocatable { }; template - struct is_trivially_relocatable : is_trivially_relocatable + struct is_trivially_relocatable + : is_trivially_relocatable { }; + template struct is_trivially_relocatable : is_trivially_relocatable { }; - template - struct is_trivially_relocatable : is_trivially_relocatable + template + struct is_trivially_relocatable : is_trivially_relocatable { }; - template - struct is_trivially_relocatable : is_trivially_relocatable + template + struct is_trivially_relocatable : is_trivially_relocatable { }; - template - struct is_trivially_relocatable + template + struct is_trivially_relocatable : is_trivially_relocatable { }; diff --git a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp index 8609dc4136d6..76606e261878 100644 --- a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp +++ b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp @@ -134,16 +134,12 @@ static_assert(hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 volatile>); static_assert(hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 const volatile>); -static_assert( - hpx::is_trivially_relocatable_v); -static_assert( - hpx::is_trivially_relocatable_v); static_assert( hpx::is_trivially_relocatable_v); static_assert( hpx::is_trivially_relocatable_v); -// Chain of c-v-ref-array qualifiers are supported +// Chain of c-v-array qualifiers are supported static_assert(hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1[10][10]>); static_assert(hpx::is_trivially_relocatable_v< @@ -152,13 +148,19 @@ static_assert(hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 volatile[10]>); static_assert(hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 const volatile[10]>); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 (&)[10]>); + +// References and temporaries are not trivially relocatable +static_assert( + !hpx::is_trivially_relocatable_v); static_assert( - hpx::is_trivially_relocatable_v); +static_assert(!hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 (&)[10]>); +static_assert( + !hpx::is_trivially_relocatable_v); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 const volatile&>); +static_assert(!hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 const volatile&>); // Trivial relocatability is not inherited struct derived_from_explicitly_trivially_relocatable @@ -276,4 +278,9 @@ static_assert(hpx::is_trivially_relocatable_v, static_assert(hpx::is_trivially_relocatable_v, "char* should be Trivially Relocatable"); +// Void and function types are not trivially relocatable +static_assert(!hpx::is_trivially_relocatable_v); +static_assert(!hpx::is_trivially_relocatable_v); +static_assert(!hpx::is_trivially_relocatable_v); + int main(int, char*[]) {} From 9288e0c6ab4d64040fcd16abd6b0126e1f04c4f4 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 13:33:13 +0300 Subject: [PATCH 216/333] unneeded void cast --- .../type_support/uninitialized_relocate.hpp | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp index 7e6d6044d4b4..650cc255279d 100644 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -77,17 +77,13 @@ namespace hpx::experimental { if (n_objects != 0) { - void* first_void = const_cast( - static_cast(std::addressof(*first))); - void* last_void = const_cast( - static_cast(std::addressof(*last))); + std::byte const* first_byte = + reinterpret_cast(std::addressof(*first)); + std::byte const* last_byte = + reinterpret_cast(std::addressof(*last)); - void* dst_void = const_cast( - static_cast(std::addressof(*dst))); - - std::byte* first_byte = - reinterpret_cast(first_void); - std::byte* last_byte = reinterpret_cast(last_void); + std::byte* dst_byte = const_cast( + reinterpret_cast(std::addressof(*dst))); auto n_bytes = std::distance(first_byte, last_byte); @@ -95,7 +91,7 @@ namespace hpx::experimental { // That the new buffer actually contains objects // within their lifetime. But this is not possible // with current language features. - std::memmove(dst_void, first_void, n_bytes); + std::memmove(dst_byte, first_byte, n_bytes); dst += n_objects; } @@ -142,7 +138,8 @@ namespace hpx::experimental { catch (...) { // destroy all objects other that the one - // that caused the exception + // that caused the exception + // (relocate_at already destroyed that one) // destroy all objects constructed so far std::destroy(original_dst, dst); From a2106c639fe96e05650a4a8f73167062ad5db8ad Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 13:34:26 +0300 Subject: [PATCH 217/333] addition to a test --- libs/core/type_support/tests/unit/uninitialized_relocate.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp index fec880557ff5..c2d55b671b67 100644 --- a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp +++ b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp @@ -165,6 +165,7 @@ int hpx_main() } std::destroy(ptr2, ptr2 + N); + HPX_TEST(trivially_relocatable_struct::dtor_count == N); std::free(mem1); std::free(mem2); From 2dda850d10565273901d13e361eefdd2a8a6142f Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 17:12:15 +0300 Subject: [PATCH 218/333] only objects are relocatable --- .../hpx/type_support/is_relocatable.hpp | 6 +- .../tests/unit/is_relocatable.cpp | 62 ++++++++++++++++--- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp index 05e881bfc69c..2d0c6d919ccb 100644 --- a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp @@ -11,7 +11,9 @@ namespace hpx { template - struct is_relocatable : std::is_move_constructible + struct is_relocatable + : std::bool_constant && + std::is_object_v> { }; @@ -20,7 +22,7 @@ namespace hpx { struct is_relocatable_from : std::bool_constant< std::is_constructible_v, FromTp> && - std::is_same_v, std::decay_t>> + std::is_same_v, std::remove_cv_t>> { }; diff --git a/libs/core/type_support/tests/unit/is_relocatable.cpp b/libs/core/type_support/tests/unit/is_relocatable.cpp index 5b0c982e146a..f0a09884493c 100644 --- a/libs/core/type_support/tests/unit/is_relocatable.cpp +++ b/libs/core/type_support/tests/unit/is_relocatable.cpp @@ -8,18 +8,23 @@ #include +#include // for std::shared_ptr, std::unique_ptr #include // Integral types are relocatable static_assert(hpx::is_relocatable_v); static_assert(hpx::is_relocatable_v); + +// Pointer types are relocatable static_assert(hpx::is_relocatable_v); static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); +static_assert(hpx::is_relocatable_v); // Array types are not move-constructible and thus not relocatable static_assert(!hpx::is_relocatable_v); -static_assert(!hpx::is_relocatable_v); static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); static_assert(!hpx::is_relocatable_v); // Function types are not move-constructible and thus not relocatable @@ -56,13 +61,52 @@ struct not_copy_constructible static_assert(hpx::is_relocatable_v); -// reference types are relocatable -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); -static_assert(hpx::is_relocatable_v); +// reference types are not relocatable +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); +static_assert(!hpx::is_relocatable_v); + +/* + Tests for is_relocatable_from +*/ + +// clang-format off + +// Reference types are not relocatable +static_assert(!hpx::is_relocatable_from_v< + int (&)[], int (&)[4]>); + +// Array types are not move constructible +static_assert(!hpx::is_relocatable_from_v< + int[4], int[4]>); + +// This is a simple pointer +static_assert(hpx::is_relocatable_from_v< + int (*)[4], int (*)[4]>); + +// Can move from const shared_ptr +static_assert(hpx::is_relocatable_from_v< + std::shared_ptr, const std::shared_ptr>); + +// Can't move away from a const unique_ptr +static_assert(!hpx::is_relocatable_from_v< + std::unique_ptr, const std::unique_ptr>); + +// Can move away from a non-const unique_ptr +static_assert(hpx::is_relocatable_from_v< + std::unique_ptr, std::unique_ptr>); + +// Can move away from a non-const unique_ptr, the dest's constness does not matter +static_assert(hpx::is_relocatable_from_v< + const std::unique_ptr, std::unique_ptr>); + +// Can't move away from a const unique_ptr, the dest's constness does not matter +static_assert(!hpx::is_relocatable_from_v< + const std::unique_ptr, const std::unique_ptr>); +// clang-format on int main(int, char*[]) {} From 401f11654175f520258d66fb28bbc1ba5ad1b0f0 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 17:23:10 +0300 Subject: [PATCH 219/333] comment correction --- .../include/hpx/type_support/relocate_at.hpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp index 86da478e9f62..8aa1750097ff 100644 --- a/libs/core/type_support/include/hpx/type_support/relocate_at.hpp +++ b/libs/core/type_support/include/hpx/type_support/relocate_at.hpp @@ -19,12 +19,12 @@ #endif namespace hpx::detail { -#if __cplusplus <= 201703L - // pre c++17 std::destroy_at can be used only on non-array types +#if __cplusplus < 202002L + // until c++20 std::destroy_at can be used only on non-array types template && !std::is_array_v)> #else - // c++17 std::destroy_at can be used on array types, destructing each element + // since c++20 std::destroy_at can be used on array types, destructing each element template )> #endif struct destroy_guard @@ -79,12 +79,9 @@ namespace hpx::experimental { template , int> = 0> T* relocate_at_helper(T* src, T* dst) noexcept( - // has non-throwing move constructor std::is_nothrow_move_constructible_v) { - using hpx::detail::destroy_guard; - - destroy_guard g(src); + hpx::detail::destroy_guard g(src); return hpx::construct_at(dst, HPX_MOVE(*src)); }; @@ -92,9 +89,7 @@ namespace hpx::experimental { T relocate_helper(T* src) noexcept( std::is_nothrow_move_constructible_v) { - using hpx::detail::destroy_guard; - - destroy_guard g(src); + hpx::detail::destroy_guard g(src); return HPX_MOVE(*src); } @@ -138,7 +133,7 @@ namespace hpx::experimental { } template - T relocate(T* src) noexcept(std::is_nothrow_move_constructible_v) + T relocate(T* src) noexcept(noexcept(detail::relocate_helper(src))) { static_assert( hpx::is_relocatable_v, "T(std::move(*src)) must be well-formed"); From 4c323a0ee4942f2d9ecc3cd0b89a729bf8028b76 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 17:31:38 +0300 Subject: [PATCH 220/333] clang-format --- .../type_support/is_trivially_relocatable.hpp | 4 +--- .../hpx/type_support/uninitialized_relocate.hpp | 2 +- .../tests/unit/is_trivially_relocatable.cpp | 17 +++++++++-------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp index a5587a963bfa..a29546d2ae8e 100644 --- a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp @@ -71,7 +71,6 @@ namespace hpx { { }; - template struct is_trivially_relocatable : is_trivially_relocatable { @@ -88,8 +87,7 @@ namespace hpx { }; template - struct is_trivially_relocatable - : is_trivially_relocatable + struct is_trivially_relocatable : is_trivially_relocatable { }; diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp index 650cc255279d..babb9f26aa33 100644 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -138,7 +138,7 @@ namespace hpx::experimental { catch (...) { // destroy all objects other that the one - // that caused the exception + // that caused the exception // (relocate_at already destroyed that one) // destroy all objects constructed so far diff --git a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp index 76606e261878..c20bb642b208 100644 --- a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp +++ b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp @@ -150,17 +150,18 @@ static_assert(hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 const volatile[10]>); // References and temporaries are not trivially relocatable -static_assert( - !hpx::is_trivially_relocatable_v); -static_assert( - !hpx::is_trivially_relocatable_v); +// clang-format off static_assert(!hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 (&)[10]>); -static_assert( - !hpx::is_trivially_relocatable_v); + explicitly_trivially_relocatable_1&>); +static_assert(!hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1&&>); +static_assert(!hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 (&)[10]>); +static_assert(!hpx::is_trivially_relocatable_v< + explicitly_trivially_relocatable_1 (&&)[10]>); static_assert(!hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 const volatile&>); +// clang-format on // Trivial relocatability is not inherited struct derived_from_explicitly_trivially_relocatable From 76ca7305f9d44053cf60e2228143da5225c85fc4 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 17:32:34 +0300 Subject: [PATCH 221/333] template arg name size -> N --- .../include/hpx/type_support/is_trivially_relocatable.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp index a29546d2ae8e..1a777ed11986 100644 --- a/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_trivially_relocatable.hpp @@ -91,8 +91,8 @@ namespace hpx { { }; - template - struct is_trivially_relocatable + template + struct is_trivially_relocatable : is_trivially_relocatable { }; From 876c7194f343c5f7ff7da1bdbb34a2567864102b Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Thu, 24 Aug 2023 12:42:15 +0300 Subject: [PATCH 222/333] decay -> remove_cv --- .../hpx/type_support/uninitialized_relocate.hpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp index babb9f26aa33..750dc601d660 100644 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp @@ -44,7 +44,8 @@ namespace hpx::experimental { constexpr static bool is_buffer_memcpyable = hpx::is_trivially_relocatable_v && // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The important check - std::is_same_v, std::decay_t> && + std::is_same_v, + std::remove_cv_t> && // can only relocate between same types !std::is_volatile_v && !std::is_volatile_v && // volatile types are not memcpyable @@ -103,14 +104,17 @@ namespace hpx::experimental { std::enable_if_t< choose_uninitialized_relocate_helper::value == relocate_strategy::for_loop_nothrow, + // Either the buffer is not contiguous or the types are no-throw + // move constructible but not trivially relocatable int> = 0> FwdIter uninitialized_relocate_helper( InIter first, InIter last, FwdIter dst) noexcept { for (; first != last; ++first, ++dst) { - // the move + destroy version will be used - hpx::experimental::relocate_at( + // if the type is trivially relocatable this will be a memcpy + // otherwise it will be a move + destroy + relocate_at_helper( std::addressof(*first), std::addressof(*dst)); } @@ -132,7 +136,7 @@ namespace hpx::experimental { try { // the move + destroy version will be used - hpx::experimental::relocate_at( + relocate_at_helper( std::addressof(*first), std::addressof(*dst)); } catch (...) From a474cceb0c97262f0da386468cf52b50e6763fea Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:09:56 +0300 Subject: [PATCH 223/333] is_relocatable_from --- .../include/hpx/type_support/is_relocatable.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp index 2d0c6d919ccb..734d18fa298f 100644 --- a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp @@ -26,6 +26,15 @@ namespace hpx { { }; + // ToTp(FromTp&&) must be well-formed + template + struct is_relocatable_from + : std::bool_constant< + std::is_constructible_v, FromTp> && + std::is_same_v, std::decay_t>> + { + }; + template inline constexpr bool is_relocatable_v = is_relocatable::value; From 4c438f99897fa88c36f5c7d310825e515bbaad6b Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Wed, 23 Aug 2023 17:12:15 +0300 Subject: [PATCH 224/333] only objects are relocatable --- .../type_support/include/hpx/type_support/is_relocatable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp index 734d18fa298f..40974416e937 100644 --- a/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp +++ b/libs/core/type_support/include/hpx/type_support/is_relocatable.hpp @@ -31,7 +31,7 @@ namespace hpx { struct is_relocatable_from : std::bool_constant< std::is_constructible_v, FromTp> && - std::is_same_v, std::decay_t>> + std::is_same_v, std::remove_cv_t>> { }; From 01f0f36736723f837cae733131c66a8c692570e5 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 30 Aug 2023 16:17:17 +0200 Subject: [PATCH 225/333] Add MPI_Barrier docs --- docs/sphinx/manual/migration_guide.rst | 138 ++++++++++++++++++------- 1 file changed, 103 insertions(+), 35 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 7a6051009c83..66b0ec08005b 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1152,18 +1152,6 @@ To understand this example, let's focus on the `hpx_main()` function: Having said that, we conclude to the following table: -.. table:: |hpx| equivalent functions of |mpi| - - ========================= ============================================================== - |openmpi| function |hpx| equivalent - ========================= ============================================================== - MPI_Comm_size `hpx::get_num_localities` - MPI_Comm_rank `hpx::get_locality_id()` - MPI_Isend `hpx::collectives::set()` - MPI_Irecv `hpx::collectives::get()` - MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` - ========================= ============================================================== - MPI_Gather ---------- @@ -1293,17 +1281,6 @@ gather operation by sending data to the root locality. In more detail: - The `get()` member function of the `overall_result` future is used to wait for the gather operation to complete for this locality. - -.. table:: |hpx| equivalent functions of |mpi| - - ========================= ===================================================================== - |openmpi| function |hpx| equivalent - ========================= ===================================================================== - MPI_Comm_size `hpx::get_num_localities` - MPI_Comm_rank `hpx::get_locality_id()` - MPI_Gather `hpx::gather_here()` and `hpx::gather_there()` both used with `get()` - ========================= ===================================================================== - MPI_Scatter ----------- @@ -1431,17 +1408,6 @@ the data from the root locality. In more detail: - `HPX_TEST_EQ` is a macro provided by the |hpx| testing utilities to test the collected values. - -.. table:: |hpx| equivalent functions of |mpi| - - ========================= ============================================= - |openmpi| function |hpx| equivalent - ========================= ============================================= - MPI_Comm_size `hpx::get_num_localities` - MPI_Comm_rank `hpx::get_locality_id()` - MPI_Scatter `hpx::scatter_to()` and `hpx::scatter_from()` - ========================= ============================================= - MPI_Allgather ------------- @@ -1629,7 +1595,7 @@ detail: MPI_Alltoall ------------- -The following code cGathers data from and scatters data to all processes. +The following code gathers data from and scatters data to all processes. |mpi| code: @@ -1730,3 +1696,105 @@ detail: - The `get()` function waits until the result is available and then stores it in the variable `r`. +MPI_Barrier +----------- + +The following code shows how barrier is used to synchronize multiple processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + + int main(int argc, char **argv) { + MPI_Init(&argc, &argv); + + std::size_t iterations = 5; + + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + for (std::size_t i = 0; i != iterations; ++i) { + MPI_Barrier(MPI_COMM_WORLD); + if (rank == 0) { + std::cout << "Iteration " << i << " completed." << std::endl; + } + } + + MPI_Finalize(); + return 0; + } + +|hpx| equivalent: + +.. code-block:: c++ + + std::size_t iterations = 5; + std::uint32_t this_locality = hpx::get_locality_id(); + + char const* const barrier_test_name = "/test/barrier/multiple"; + + hpx::distributed::barrier b(barrier_test_name); + for (std::size_t i = 0; i != iterations; ++i) + { + b.wait(); + if (this_locality == 0) + { + std::cout << "Iteration " << i << " completed." << std::endl; + } + } + +This code will print the following message: + +.. code-block:: c++ + + Iteration 0 completed. + Iteration 1 completed. + Iteration 2 completed. + Iteration 3 completed. + Iteration 4 completed. + +|hpx| uses the function `barrier` to implement the functionality of `MPI_Barrier`. In more +detail: + +- After defining the number of iterations, we use + `hpx::get_locality_id()` to get the ID of the current locality. + +- `char const* const barrier_test_name = "/test/barrier/multiple"`: This line defines a constant + character array as the name of the barrier. This name is used to identify the barrier across + different localities. All participating threads that use this name will synchronize + at this barrier. + +- Using `hpx::distributed::barrier b(barrier_test_name)`, we create an instance of the distributed + barrier with the previously defined name. This barrier will be used to synchronize the execution + of threads across different localities. + +- Running for all the desired iterations, we use `b.wait()` to synchronize the threads. + Each thread waits until all other threads also reach this point before any of them can proceed + further. + + +List of |mpi|-|hpx| functions +----------------------------- + + .. table:: |hpx| equivalent functions of |mpi| + + ========================= ===================================================================== + |openmpi| function |hpx| equivalent + ========================= ===================================================================== + MPI_Allgather `hpx::distributed::all_gather` + MPI_Allreduce `hpx::distributed::all_reduce` + MPI_Alltoall `hpx::distributed::all_to_all` + MPI_Barrier `hpx::distributed::barrier` + MPI_Comm_size `hpx::get_num_localities` + MPI_Comm_rank `hpx::get_locality_id()` + MPI_Gather `hpx::gather_here()` and `hpx::gather_there()` both used with `get()` + MPI_Irecv `hpx::collectives::get()` + MPI_Isend `hpx::collectives::set()` + MPI_Scatter `hpx::scatter_to()` and `hpx::scatter_from()` + MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` + ========================= ===================================================================== From 3f5f871a37fb2cc7dfb85051fcb6f6521b03c824 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 30 Aug 2023 17:42:42 +0200 Subject: [PATCH 226/333] Add MPI_Bcast docs --- docs/sphinx/manual/migration_guide.rst | 117 ++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 3 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 66b0ec08005b..ff30533bd702 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1777,19 +1777,130 @@ detail: Each thread waits until all other threads also reach this point before any of them can proceed further. +MPI_Bcast +--------- + +The following code broadcasts data from one process to all other processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_localities; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + + int here; + MPI_Comm_rank(MPI_COMM_WORLD, &here); + + int value; + + for (int i = 0; i < 5; ++i) { + if (here == 0) { + value = i + 42; + } + + // Broadcast the value from process 0 to all other processes + MPI_Bcast(&value, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (here != 0) { + std::cout << "Locality " << here << " received " << value << std::endl; + } + + } + + MPI_Finalize(); + return 0; + } + + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + + std::uint32_t here = hpx::get_locality_id(); + + auto broadcast_direct_client = + create_communicator(broadcast_direct_basename, + num_sites_arg(num_localities), this_site_arg(here)); + + // test functionality based on immediate local result value + for (std::uint32_t i = 0; i != 5; ++i) + { + if (here == 0) + { + hpx::future result = + broadcast_to(broadcast_direct_client, i + 42); + + result.get(); + } + else + { + hpx::future result = + hpx::collectives::broadcast_from( + broadcast_direct_client); + + uint32_t r = result.get(); + + std::cout << "Locality " << here << " received " << r << std::endl; + } + } + +For num_localities = 2 this code will print the following message: + +.. code-block:: c++ + + Locality 1 received 42 + Locality 1 received 43 + Locality 1 received 44 + Locality 1 received 45 + Locality 1 received 46 + +|hpx| uses two functions to implement the functionality of `MPI_Bcast`: `hpx::broadcast_to` and +`hpx::broadcast_from`. `hpx::broadcast_to` is broadcasting the data from the root locality to all +other localities. `hpx::broadcast_from` allows non-root localities to collect the data sent by +the root locality. In more detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `create_communicator()` is used to create a communicator called + `broadcast_direct_client`. + +- If the current locality is the root (its ID is equal to 0): + + - The `hpx::broadcast_to` function is used to perform the broadcast operation using the communicator + `broadcast_direct_client`. This sends the data to other localities and + returns a future representing the result. + + - The `get()` member function of the `result` future is used to wait for and retrieve the result. + +- If the current locality is not the root: + + - The `hpx::scatter_from` function is used to collect the data by the root locality. + + - The `get()` member function of the `result` future is used to wait for the result. List of |mpi|-|hpx| functions ----------------------------- .. table:: |hpx| equivalent functions of |mpi| - ========================= ===================================================================== + ========================= ======================================================================= |openmpi| function |hpx| equivalent - ========================= ===================================================================== + ========================= ======================================================================= MPI_Allgather `hpx::distributed::all_gather` MPI_Allreduce `hpx::distributed::all_reduce` MPI_Alltoall `hpx::distributed::all_to_all` MPI_Barrier `hpx::distributed::barrier` + MPI_Bcast `hpx::broadcast_to()` and `hpx::broadcast_from()` both used with `get()` MPI_Comm_size `hpx::get_num_localities` MPI_Comm_rank `hpx::get_locality_id()` MPI_Gather `hpx::gather_here()` and `hpx::gather_there()` both used with `get()` @@ -1797,4 +1908,4 @@ List of |mpi|-|hpx| functions MPI_Isend `hpx::collectives::set()` MPI_Scatter `hpx::scatter_to()` and `hpx::scatter_from()` MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` - ========================= ===================================================================== + ========================= ======================================================================= From f73a588cf43b04971320e4b08d65c95b905f37ca Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 30 Aug 2023 18:36:03 +0200 Subject: [PATCH 227/333] Add MPI_Exscan docs --- docs/sphinx/manual/migration_guide.rst | 114 ++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 14 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index ff30533bd702..44b5e7f1412b 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1863,9 +1863,9 @@ For num_localities = 2 this code will print the following message: Locality 1 received 45 Locality 1 received 46 -|hpx| uses two functions to implement the functionality of `MPI_Bcast`: `hpx::broadcast_to` and -`hpx::broadcast_from`. `hpx::broadcast_to` is broadcasting the data from the root locality to all -other localities. `hpx::broadcast_from` allows non-root localities to collect the data sent by +|hpx| uses two functions to implement the functionality of `MPI_Bcast`: `broadcast_to` and +`broadcast_from`. `broadcast_to` is broadcasting the data from the root locality to all +other localities. `broadcast_from` allows non-root localities to collect the data sent by the root locality. In more detail: - `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while @@ -1876,7 +1876,7 @@ the root locality. In more detail: - If the current locality is the root (its ID is equal to 0): - - The `hpx::broadcast_to` function is used to perform the broadcast operation using the communicator + - The `broadcast_to` function is used to perform the broadcast operation using the communicator `broadcast_direct_client`. This sends the data to other localities and returns a future representing the result. @@ -1884,28 +1884,114 @@ the root locality. In more detail: - If the current locality is not the root: - - The `hpx::scatter_from` function is used to collect the data by the root locality. + - The `broadcast_from` function is used to collect the data by the root locality. - The `get()` member function of the `result` future is used to wait for the result. +MPI_Exscan +---------- + +The following code broadcasts data from one process to all other processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + #include + + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_localities; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + + int here; + MPI_Comm_rank(MPI_COMM_WORLD, &here); + + // Calculate the value for this locality (here) + int value = here; + + // Perform an exclusive scan + std::vector result(num_localities); + MPI_Exscan(&value, &result[0], 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + if (here != 0) { + int r = result[here - 1]; // Result is in the previous rank's slot + + std::cout << "Locality " << here << " has value " << r << std::endl; + } + + MPI_Finalize(); + return 0; + } + + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + std::uint32_t here = hpx::get_locality_id(); + + auto exclusive_scan_client = create_communicator(exclusive_scan_basename, + num_sites_arg(num_localities), this_site_arg(here)); + + // test functionality based on immediate local result value + std::uint32_t value = here; + + hpx::future overall_result = exclusive_scan( + exclusive_scan_client, value, std::plus{}); + + uint32_t r = overall_result.get(); + + if (here != 0) + { + std::cout << "Locality " << here << " has value " << r << std::endl; + } + +For num_localities = 2 this code will print the following message: + +.. code-block:: c++ + + Locality 1 has value 0 + +|hpx| uses the function `exclusive_scan` to implement `MPI_Exscan`. In more detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `create_communicator()` is used to create a communicator called + `exclusive_scan_client`. + +- The `exclusive_scan` function is used to perform the exclusive scan operation + using the communicator `exclusive_scan_client`. This sends the data to other localities and + returns a future representing the result. + +- The `get()` member function of the `overall_result` future is used to wait for the result. + + List of |mpi|-|hpx| functions ----------------------------- .. table:: |hpx| equivalent functions of |mpi| - ========================= ======================================================================= + ========================= ================================================================================================== |openmpi| function |hpx| equivalent - ========================= ======================================================================= - MPI_Allgather `hpx::distributed::all_gather` - MPI_Allreduce `hpx::distributed::all_reduce` - MPI_Alltoall `hpx::distributed::all_to_all` + ========================= ================================================================================================== + MPI_Allgather `hpx::collectives::all_gather` + MPI_Allreduce `hpx::collectives::all_reduce` + MPI_Alltoall `hpx::collectives::all_to_all` MPI_Barrier `hpx::distributed::barrier` - MPI_Bcast `hpx::broadcast_to()` and `hpx::broadcast_from()` both used with `get()` + MPI_Bcast `hpx::collectives::broadcast_to()` and `hpx::collectives::broadcast_from()` both used with `get()` MPI_Comm_size `hpx::get_num_localities` MPI_Comm_rank `hpx::get_locality_id()` - MPI_Gather `hpx::gather_here()` and `hpx::gather_there()` both used with `get()` + MPI_Exscan `hpx::collectives::exclusive_scan()` used with `get()` + MPI_Gather `hpx::collectives::gather_here()` and `hpx::collectives::gather_there()` both used with `get()` MPI_Irecv `hpx::collectives::get()` MPI_Isend `hpx::collectives::set()` - MPI_Scatter `hpx::scatter_to()` and `hpx::scatter_from()` + MPI_Scatter `hpx::collectives::scatter_to()` and `hpx::collectives::scatter_from()` MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` - ========================= ======================================================================= + ========================= ================================================================================================== From 3a8353a9ca56fdc18a34fb5b052bcaeac228f0b7 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 30 Aug 2023 18:54:02 +0200 Subject: [PATCH 228/333] Add MPI_Scan docs --- docs/sphinx/manual/migration_guide.rst | 88 +++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 44b5e7f1412b..3d27b7c9a318 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1891,7 +1891,8 @@ the root locality. In more detail: MPI_Exscan ---------- -The following code broadcasts data from one process to all other processes. +The following code computes the exclusive scan (partial reductions) of data on a +collection of processes. |mpi| code: @@ -1967,12 +1968,92 @@ For num_localities = 2 this code will print the following message: `exclusive_scan_client`. - The `exclusive_scan` function is used to perform the exclusive scan operation - using the communicator `exclusive_scan_client`. This sends the data to other localities and - returns a future representing the result. + using the communicator `exclusive_scan_client`. `std::plus{}` + specifies the binary associative operator to use for the scan. In this case, + it's addition for summing values. - The `get()` member function of the `overall_result` future is used to wait for the result. +MPI_Scan +-------- + +The following code Computes the inclusive scan (partial reductions) of data on a collection +of processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + #include + #include + + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_localities; + MPI_Comm_size(MPI_COMM_WORLD, &num_localities); + + int here; + MPI_Comm_rank(MPI_COMM_WORLD, &here); + + // Calculate the value for this locality (here) + int value = here; + + std::vector result(num_localities); + + MPI_Scan(&value, &result[0], 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + std::cout << "Locality " << here << " has value " << result[0] << std::endl; + + MPI_Finalize(); + return 0; + } + + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + std::uint32_t here = hpx::get_locality_id(); + + auto inclusive_scan_client = create_communicator(inclusive_scan_basename, + num_sites_arg(num_localities), this_site_arg(here)); + + std::uint32_t value = here; + + hpx::future overall_result = inclusive_scan( + inclusive_scan_client, value, std::plus{}); + + uint32_t r = overall_result.get(); + + std::cout << "Locality " << here << " has value " << r << std::endl; + +For num_localities = 2 this code will print the following message: + +.. code-block:: c++ + + Locality 0 has value 0 + Locality 1 has value 1 + +|hpx| uses the function `inclusive_scan` to implement `MPI_Scan`. In more detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `create_communicator()` is used to create a communicator called + `inclusive_scan_client`. + +- The `inclusive_scan` function is used to perform the exclusive scan operation + using the communicator `inclusive_scan_client`. `std::plus{}` + specifies the binary associative operator to use for the scan. In this case, + it's addition for summing values. + +- The `get()` member function of the `overall_result` future is used to wait for the result. + List of |mpi|-|hpx| functions ----------------------------- @@ -1990,6 +2071,7 @@ List of |mpi|-|hpx| functions MPI_Comm_rank `hpx::get_locality_id()` MPI_Exscan `hpx::collectives::exclusive_scan()` used with `get()` MPI_Gather `hpx::collectives::gather_here()` and `hpx::collectives::gather_there()` both used with `get()` + MPI_Scan `hpx::collectives::inclusive_scan()` used with `get()` MPI_Irecv `hpx::collectives::get()` MPI_Isend `hpx::collectives::set()` MPI_Scatter `hpx::collectives::scatter_to()` and `hpx::collectives::scatter_from()` From b3c92f51b1cf88c336b376c0ec9c8f6d56f57313 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 30 Aug 2023 19:13:46 +0200 Subject: [PATCH 229/333] Add MPI_Reduce docs --- docs/sphinx/manual/migration_guide.rst | 122 ++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 3d27b7c9a318..c75024735fb1 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1247,9 +1247,9 @@ This code will print 10 times the following message: Gathered data on the root: 42 43 -|hpx| uses two functions to implement the functionality of `MPI_Gather`: `hpx::gather_here` and -`hpx::gather_there`. `hpx::gather_here` is gathering data from all localities to the locality -with ID 0 (root locality). `hpx::gather_there` allows non-root localities to participate in the +|hpx| uses two functions to implement the functionality of `MPI_Gather`: `gather_here` and +`gather_there`. `gather_here` is gathering data from all localities to the locality +with ID 0 (root locality). `gather_there` allows non-root localities to participate in the gather operation by sending data to the root locality. In more detail: - `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while @@ -1260,7 +1260,7 @@ gather operation by sending data to the root locality. In more detail: - If the current locality is the root (its ID is equal to 0): - - the `hpx::gather_here` function is used to perform the gather operation. It collects data from all + - The `gather_here` function is used to perform the gather operation. It collects data from all other localities into the `overall_result` future object. The function arguments provide the necessary information, such as the base name for the gather operation (`gather_direct_basename`), the value to be gathered (`value`), the number of localities (`num_localities`), the current locality ID @@ -1274,7 +1274,7 @@ gather operation by sending data to the root locality. In more detail: - If the current locality is not the root: - - The `hpx::gather_there` function is used to participate in the gather operation initiated by + - The `gather_there` function is used to participate in the gather operation initiated by the root locality. It sends the data (in this case, the value `this_locality + 42`) to the root locality, indicating that it should be included in the gathering. @@ -2054,26 +2054,126 @@ For num_localities = 2 this code will print the following message: - The `get()` member function of the `overall_result` future is used to wait for the result. +MPI_Reduce +---------- + +The following code performs a global reduce operation across all processes. + +|mpi| code: + +.. code-block:: c++ + + #include + #include + + int main(int argc, char *argv[]) { + MPI_Init(&argc, &argv); + + int num_processes; + MPI_Comm_size(MPI_COMM_WORLD, &num_processes); + + int this_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &this_rank); + + int value = this_rank; + + int result = 0; + + // Perform the reduction operation + MPI_Reduce(&value, &result, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); + + // Print the result for the root process (process 0) + if (this_rank == 0) { + std::cout << "Locality " << this_rank << " has value " << result + << std::endl; + } + + MPI_Finalize(); + return 0; + } + +|hpx| equivalent: + +.. code-block:: c++ + + std::uint32_t num_localities = hpx::get_num_localities(hpx::launch::sync); + std::uint32_t this_locality = hpx::get_locality_id(); + + auto reduce_direct_client = create_communicator(reduce_direct_basename, + num_sites_arg(num_localities), this_site_arg(this_locality)); + + std::uint32_t value = hpx::get_locality_id(); + + if (this_locality == 0) + { + hpx::future overall_result = reduce_here( + reduce_direct_client, value, std::plus{}); + + uint32_t r = overall_result.get(); + + std::cout << "Locality " << this_locality << " has value " << r + << std::endl; + } + else + { + hpx::future overall_result = + reduce_there(reduce_direct_client, std::move(value)); + overall_result.get(); + } + +This code will print the following message: + +.. code-block:: c++ + + Locality 0 has value 1 + +|hpx| uses two functions to implement the functionality of `MPI_Reduce`: `reduce_here` and +`reduce_there`. `reduce_here` is gathering data from all localities to the locality +with ID 0 (root locality) and then performs the defined reduction operation. `reduce_there` +allows non-root localities to participate in the reduction operation by sending data to the +root locality. In more detail: + +- `hpx::get_num_localities(hpx::launch::sync)` retrieves the number of localities, while + `hpx::get_locality_id()` returns the ID of the current locality. + +- The function `create_communicator()` is used to create a communicator called + `reduce_direct_client`. + +- If the current locality is the root (its ID is equal to 0): + + - The `reduce_here` function initiates a reduction operation with addition (`std::plus`) as the + reduction operator. The result is stored in `overall_result`. + + - The `get()` member function of the `overall_result` future is used to wait for the result. + +- If the current locality is not the root: + + - The `reduce_there` initiates a remote reduction operation. + + - The `get()` member function of the `overall_result` future is used to wait for the remote + reduction operation to complete. This is done to ensure synchronization among localities. + List of |mpi|-|hpx| functions ----------------------------- .. table:: |hpx| equivalent functions of |mpi| - ========================= ================================================================================================== + ========================= ============================================================================================= |openmpi| function |hpx| equivalent - ========================= ================================================================================================== + ========================= ============================================================================================= MPI_Allgather `hpx::collectives::all_gather` MPI_Allreduce `hpx::collectives::all_reduce` MPI_Alltoall `hpx::collectives::all_to_all` MPI_Barrier `hpx::distributed::barrier` - MPI_Bcast `hpx::collectives::broadcast_to()` and `hpx::collectives::broadcast_from()` both used with `get()` + MPI_Bcast `hpx::collectives::broadcast_to()` and `hpx::collectives::broadcast_from()` used with `get()` MPI_Comm_size `hpx::get_num_localities` MPI_Comm_rank `hpx::get_locality_id()` MPI_Exscan `hpx::collectives::exclusive_scan()` used with `get()` - MPI_Gather `hpx::collectives::gather_here()` and `hpx::collectives::gather_there()` both used with `get()` - MPI_Scan `hpx::collectives::inclusive_scan()` used with `get()` + MPI_Gather `hpx::collectives::gather_here()` and `hpx::collectives::gather_there()` used with `get()` MPI_Irecv `hpx::collectives::get()` MPI_Isend `hpx::collectives::set()` + MPI_Reduce `hpx::collectives::reduce_here` and `hpx::collectives::reduce_there` used with `get()` + MPI_Scan `hpx::collectives::inclusive_scan()` used with `get()` MPI_Scatter `hpx::collectives::scatter_to()` and `hpx::collectives::scatter_from()` MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` - ========================= ================================================================================================== + ========================= ============================================================================================= From 543ce407c4bc4f4010cdde7f47ee0d07d3b658b8 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 7 Sep 2023 16:24:54 +0200 Subject: [PATCH 230/333] Update MPI-HPX table --- docs/sphinx/manual/migration_guide.rst | 78 +++++++++++++++++--------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index c75024735fb1..31c0622c74bf 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -42,6 +42,8 @@ Some general steps that can be used to migrate code to |hpx| code are the follow Compile the converted code with the |hpx| library and run it using the appropriate HPX runtime environment. +.. _openmp: + |openmp| ======== @@ -655,6 +657,8 @@ directive in |openmp|. This synchronization is achieved using :cpp:func:`hpx::wa If the `nowait` clause is used in the `sections` directive, then you can just remove the :cpp:func:`hpx::wait_all` function while keeping the rest of the code as it is. +.. _tbb: + |tbb| ===== @@ -965,7 +969,7 @@ parallel_scan [](const int& left, const int& right) { return left + right; }); :cpp:func:`hpx::inclusive_scan` with `hpx::execution::par` as execution policy - can be used to perform a prefix scan in parallel. The management of intermediate +can be used to perform a prefix scan in parallel. The management of intermediate results is seamlessly handled internally by |hpx|, eliminating the need for explicit consideration. `input.begin()` and `input.end()` refer to the beginning and end of the sequence of elements the algorithm will be applied to respectively. @@ -1045,12 +1049,41 @@ task_group feature. Therefore, utilizing :cpp:func:`hpx::experimental::task_group` provides an equivalent functionality to `tbb::task_group`. +.. _mpi: + |mpi| ===== |mpi| is a standardized communication protocol and library that allows multiple processes or nodes in a parallel computing system to exchange data and coordinate their execution. +List of |mpi|-|hpx| functions +----------------------------- + + .. table:: |hpx| equivalent functions of |mpi| + + ======================================== =================================================================================================================== + |mpi| function |hpx| equivalent + ======================================== =================================================================================================================== + :ref:`MPI_Allgather` :cpp:class:`hpx::collectives::all_gather` + :ref:`MPI_Allreduce` :cpp:class:`hpx::collectives::all_reduce` + :ref:`MPI_Alltoall` :cpp:class:`hpx::collectives::all_to_all` + :ref:`MPI_Barrier` :cpp:class:`hpx::distributed::barrier` + :ref:`MPI_Bcast` :cpp:class:`hpx::collectives::broadcast_to()` and :cpp:class:`hpx::collectives::broadcast_from()` used with :code:`get()` + :ref:`MPI_Comm_size ` :cpp:class:`hpx::get_num_localities` + :ref:`MPI_Comm_rank ` :cpp:class:`hpx::get_locality_id()` + :ref:`MPI_Exscan` :cpp:class:`hpx::collectives::exclusive_scan()` used with :code:`get()` + :ref:`MPI_Gather` :cpp:class:`hpx::collectives::gather_here()` and :cpp:class:`hpx::collectives::gather_there()` used with :code:`get()` + :ref:`MPI_Irecv ` :cpp:class:`hpx::collectives::get()` + :ref:`MPI_Isend ` :cpp:class:`hpx::collectives::set()` + :ref:`MPI_Reduce` :cpp:class:`hpx::collectives::reduce_here` and :cpp:class:`hpx::collectives::reduce_there` used with :code:`get()` + :ref:`MPI_Scan` :cpp:class:`hpx::collectives::inclusive_scan()` used with :code:`get()` + :ref:`MPI_Scatter` :cpp:class:`hpx::collectives::scatter_to()` and :cpp:class:`hpx::collectives::scatter_from()` + :ref:`MPI_Wait ` :cpp:class:`hpx::collectives::get()` used with a future i.e. :code:`setf.get()` + ======================================== =================================================================================================================== + +.. _MPI_Send_MPI_Recv: + MPI_Send & MPI_Recv ------------------- @@ -1152,6 +1185,8 @@ To understand this example, let's focus on the `hpx_main()` function: Having said that, we conclude to the following table: +.. _MPI_Gather: + MPI_Gather ---------- @@ -1281,6 +1316,8 @@ gather operation by sending data to the root locality. In more detail: - The `get()` member function of the `overall_result` future is used to wait for the gather operation to complete for this locality. +.. _MPI_Scatter: + MPI_Scatter ----------- @@ -1408,6 +1445,8 @@ the data from the root locality. In more detail: - `HPX_TEST_EQ` is a macro provided by the |hpx| testing utilities to test the collected values. +.. _MPI_Allgather: + MPI_Allgather ------------- @@ -1505,6 +1544,7 @@ detail: - The `get()` function waits until the result is available and then stores it in the vector called `r`. +.. _MPI_Allreduce: MPI_Allreduce ------------- @@ -1592,6 +1632,8 @@ detail: - The `get()` function waits until the result is available and then stores it in the variable `res`. +.. _MPI_Alltoall: + MPI_Alltoall ------------- @@ -1696,6 +1738,8 @@ detail: - The `get()` function waits until the result is available and then stores it in the variable `r`. +.. _MPI_Barrier: + MPI_Barrier ----------- @@ -1777,6 +1821,8 @@ detail: Each thread waits until all other threads also reach this point before any of them can proceed further. +.. _MPI_Bcast: + MPI_Bcast --------- @@ -1888,6 +1934,8 @@ the root locality. In more detail: - The `get()` member function of the `result` future is used to wait for the result. +.. _MPI_Exscan: + MPI_Exscan ---------- @@ -1974,6 +2022,7 @@ For num_localities = 2 this code will print the following message: - The `get()` member function of the `overall_result` future is used to wait for the result. +.. _MPI_Scan: MPI_Scan -------- @@ -2054,6 +2103,8 @@ For num_localities = 2 this code will print the following message: - The `get()` member function of the `overall_result` future is used to wait for the result. +.. _MPI_Reduce: + MPI_Reduce ---------- @@ -2152,28 +2203,3 @@ root locality. In more detail: - The `get()` member function of the `overall_result` future is used to wait for the remote reduction operation to complete. This is done to ensure synchronization among localities. - -List of |mpi|-|hpx| functions ------------------------------ - - .. table:: |hpx| equivalent functions of |mpi| - - ========================= ============================================================================================= - |openmpi| function |hpx| equivalent - ========================= ============================================================================================= - MPI_Allgather `hpx::collectives::all_gather` - MPI_Allreduce `hpx::collectives::all_reduce` - MPI_Alltoall `hpx::collectives::all_to_all` - MPI_Barrier `hpx::distributed::barrier` - MPI_Bcast `hpx::collectives::broadcast_to()` and `hpx::collectives::broadcast_from()` used with `get()` - MPI_Comm_size `hpx::get_num_localities` - MPI_Comm_rank `hpx::get_locality_id()` - MPI_Exscan `hpx::collectives::exclusive_scan()` used with `get()` - MPI_Gather `hpx::collectives::gather_here()` and `hpx::collectives::gather_there()` used with `get()` - MPI_Irecv `hpx::collectives::get()` - MPI_Isend `hpx::collectives::set()` - MPI_Reduce `hpx::collectives::reduce_here` and `hpx::collectives::reduce_there` used with `get()` - MPI_Scan `hpx::collectives::inclusive_scan()` used with `get()` - MPI_Scatter `hpx::collectives::scatter_to()` and `hpx::collectives::scatter_from()` - MPI_Wait `hpx::collectives::get()` used with a future i.e. `setf.get()` - ========================= ============================================================================================= From 169b8de5c60c4e1c4ea664ceb61391e88c22bdf0 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Fri, 1 Sep 2023 14:06:14 -0500 Subject: [PATCH 231/333] Log alive hpx threads on exit --- libs/core/runtime_local/src/runtime_local.cpp | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/libs/core/runtime_local/src/runtime_local.cpp b/libs/core/runtime_local/src/runtime_local.cpp index 0bc3eb1e47b8..5258e26031a3 100644 --- a/libs/core/runtime_local/src/runtime_local.cpp +++ b/libs/core/runtime_local/src/runtime_local.cpp @@ -57,6 +57,27 @@ #include #include +namespace hpx{ namespace detail { + void log_all_runtime_threads() + { + auto rt = hpx::get_runtime_ptr(); + if (rt) + { + rt->get_thread_manager().enumerate_threads( + [](hpx::threads::thread_id_type id) -> bool { + hpx::threads::thread_data* td = get_thread_id_data(id); + auto sched = td->get_scheduler_base(); + LTM_(debug).format( + "Logging all runtime threads: pool({}), scheduler({})," + "thread({}), description({}), state({})", + sched->get_parent_pool(), sched, id, + td->get_description(), td->get_state().state()); + return true; + }); + } + } +}}; // namespace hpx::detail + /////////////////////////////////////////////////////////////////////////////// // Make sure the system gets properly shut down while handling Ctrl-C and other // system signals @@ -91,6 +112,11 @@ namespace hpx { } #endif +#if defined(HPX_HAVE_LOGGING) + LRT_(debug).format("Terminating due to system signal({})", reason); + hpx::detail::log_all_runtime_threads(); +#endif + std::cerr << "{what}: " << (reason ? reason : "Unknown reason") << "\n"; } @@ -164,6 +190,11 @@ namespace hpx { } #endif +#if defined(HPX_HAVE_LOGGING) + LRT_(debug).format("Terminating due to system signal({})", signum); + hpx::detail::log_all_runtime_threads(); +#endif + std::cerr << "{what}: " << (reason ? reason : "Unknown reason") << "\n"; } From fe79f05453f94294252745300d4e0480f6b01fbd Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Sat, 2 Sep 2023 10:15:00 -0500 Subject: [PATCH 232/333] Fix for thread logging on exit --- libs/core/runtime_local/src/runtime_local.cpp | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/libs/core/runtime_local/src/runtime_local.cpp b/libs/core/runtime_local/src/runtime_local.cpp index 5258e26031a3..09b8c7710cfc 100644 --- a/libs/core/runtime_local/src/runtime_local.cpp +++ b/libs/core/runtime_local/src/runtime_local.cpp @@ -57,26 +57,35 @@ #include #include -namespace hpx{ namespace detail { - void log_all_runtime_threads() +#if defined(HPX_HAVE_LOGGING) +namespace hpx { namespace detail { + void try_log_runtime_threads() { - auto rt = hpx::get_runtime_ptr(); - if (rt) + // This may be used in non-valid runtime states, let it fail silently + try { + auto rt = hpx::get_runtime_ptr(); + if (rt == nullptr) + return; + rt->get_thread_manager().enumerate_threads( [](hpx::threads::thread_id_type id) -> bool { hpx::threads::thread_data* td = get_thread_id_data(id); auto sched = td->get_scheduler_base(); - LTM_(debug).format( - "Logging all runtime threads: pool({}), scheduler({})," - "thread({}), description({}), state({})", + LTM_(debug).format("Logging all runtime threads: pool({}), " + "scheduler({})," + "thread({}), description({}), state({})", sched->get_parent_pool(), sched, id, td->get_description(), td->get_state().state()); return true; }); } + catch (...) + { + } } }}; // namespace hpx::detail +#endif /////////////////////////////////////////////////////////////////////////////// // Make sure the system gets properly shut down while handling Ctrl-C and other @@ -114,7 +123,7 @@ namespace hpx { #if defined(HPX_HAVE_LOGGING) LRT_(debug).format("Terminating due to system signal({})", reason); - hpx::detail::log_all_runtime_threads(); + hpx::detail::try_log_runtime_threads(); #endif std::cerr << "{what}: " << (reason ? reason : "Unknown reason") @@ -192,7 +201,7 @@ namespace hpx { #if defined(HPX_HAVE_LOGGING) LRT_(debug).format("Terminating due to system signal({})", signum); - hpx::detail::log_all_runtime_threads(); + hpx::detail::try_log_runtime_threads(); #endif std::cerr << "{what}: " << (reason ? reason : "Unknown reason") From 2cbf38f639db5e42c05c47836013d62f2c5207f1 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 24 Aug 2023 08:46:00 -0500 Subject: [PATCH 233/333] Adding basic logging to collective operations - flyby: fixing JeMalloc integration for dependent projects --- .../include/hpx/collectives/all_gather.hpp | 6 ++++++ .../include/hpx/collectives/all_reduce.hpp | 7 +++++++ .../include/hpx/collectives/all_to_all.hpp | 7 +++++++ .../include/hpx/collectives/broadcast.hpp | 7 +++++++ .../hpx/collectives/create_communicator.hpp | 21 ++++++++----------- .../hpx/collectives/detail/communicator.hpp | 21 +++++++++++++++++++ .../hpx/collectives/exclusive_scan.hpp | 7 +++++++ .../include/hpx/collectives/gather.hpp | 6 ++++++ .../hpx/collectives/inclusive_scan.hpp | 6 ++++++ .../include/hpx/collectives/reduce.hpp | 6 ++++++ .../include/hpx/collectives/scatter.hpp | 6 ++++++ 11 files changed, 88 insertions(+), 12 deletions(-) diff --git a/libs/full/collectives/include/hpx/collectives/all_gather.hpp b/libs/full/collectives/include/hpx/collectives/all_gather.hpp index 3a25d64fa799..b245712ca359 100644 --- a/libs/full/collectives/include/hpx/collectives/all_gather.hpp +++ b/libs/full/collectives/include/hpx/collectives/all_gather.hpp @@ -131,6 +131,12 @@ namespace hpx::traits { namespace communication { struct all_gather_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "all_gather"; + } } // namespace communication /////////////////////////////////////////////////////////////////////////// diff --git a/libs/full/collectives/include/hpx/collectives/all_reduce.hpp b/libs/full/collectives/include/hpx/collectives/all_reduce.hpp index 145ef2d7a82e..867f43927ed6 100644 --- a/libs/full/collectives/include/hpx/collectives/all_reduce.hpp +++ b/libs/full/collectives/include/hpx/collectives/all_reduce.hpp @@ -136,7 +136,14 @@ namespace hpx { namespace collectives { namespace hpx::traits { namespace communication { + struct all_reduce_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "all_reduce"; + } } // namespace communication /////////////////////////////////////////////////////////////////////////// diff --git a/libs/full/collectives/include/hpx/collectives/all_to_all.hpp b/libs/full/collectives/include/hpx/collectives/all_to_all.hpp index b4f7b3648f0f..fbc341cb32e2 100644 --- a/libs/full/collectives/include/hpx/collectives/all_to_all.hpp +++ b/libs/full/collectives/include/hpx/collectives/all_to_all.hpp @@ -130,7 +130,14 @@ namespace hpx { namespace collectives { namespace hpx::traits { namespace communication { + struct all_to_all_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "all_to_all"; + } } // namespace communication /////////////////////////////////////////////////////////////////////////// diff --git a/libs/full/collectives/include/hpx/collectives/broadcast.hpp b/libs/full/collectives/include/hpx/collectives/broadcast.hpp index 64613a36c236..030505ef34eb 100644 --- a/libs/full/collectives/include/hpx/collectives/broadcast.hpp +++ b/libs/full/collectives/include/hpx/collectives/broadcast.hpp @@ -213,7 +213,14 @@ namespace hpx::traits { /////////////////////////////////////////////////////////////////////////// // support for broadcast namespace communication { + struct broadcast_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "broadcast"; + } } // namespace communication template diff --git a/libs/full/collectives/include/hpx/collectives/create_communicator.hpp b/libs/full/collectives/include/hpx/collectives/create_communicator.hpp index 6da5206d7602..85d5e19c1f35 100644 --- a/libs/full/collectives/include/hpx/collectives/create_communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/create_communicator.hpp @@ -102,18 +102,15 @@ namespace hpx::collectives::detail { }; } // namespace hpx::collectives::detail -namespace hpx::util { - - // This is explicitly instantiated to ensure that the id is stable across - // shared libraries. - template <> - struct extra_data_helper - { - HPX_EXPORT static extra_data_id_type id() noexcept; - static constexpr void reset( - collectives::detail::communicator_data*) noexcept {}; - }; -} // namespace hpx::util +// This is explicitly instantiated to ensure that the id is stable across +// shared libraries. +template <> +struct hpx::util::extra_data_helper +{ + HPX_EXPORT static extra_data_id_type id() noexcept; + static constexpr void reset( + collectives::detail::communicator_data*) noexcept {}; +}; // namespace hpx::util namespace hpx::collectives { diff --git a/libs/full/collectives/include/hpx/collectives/detail/communicator.hpp b/libs/full/collectives/include/hpx/collectives/detail/communicator.hpp index df9ee39f1c76..d020746c5d87 100644 --- a/libs/full/collectives/include/hpx/collectives/detail/communicator.hpp +++ b/libs/full/collectives/include/hpx/collectives/detail/communicator.hpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,16 @@ namespace hpx::traits { // This type can be specialized for a particular collective operation template struct communication_operation; + + namespace communication { + + // Retrieve name of the current communicator + template + constexpr char const* communicator_name() noexcept + { + return ""; + } + } // namespace communication } // namespace hpx::traits namespace hpx::collectives::detail { @@ -55,6 +66,11 @@ namespace hpx::collectives::detail { using collective_operation = traits::communication_operation; + LHPX_(info, " [COL] ") + .format("get({}): which({}), generation({})", + traits::communication::communicator_name(), + which, generation); + return collective_operation::template get( *this, which, generation, HPX_MOVE(args)...); } @@ -75,6 +91,11 @@ namespace hpx::collectives::detail { using collective_operation = traits::communication_operation; + LHPX_(info, " [COL] ") + .format("set({}): which({}), generation({})", + traits::communication::communicator_name(), + which, generation); + return collective_operation::template set( *this, which, generation, HPX_MOVE(args)...); } diff --git a/libs/full/collectives/include/hpx/collectives/exclusive_scan.hpp b/libs/full/collectives/include/hpx/collectives/exclusive_scan.hpp index 2a4ac6603007..ad21129d8022 100644 --- a/libs/full/collectives/include/hpx/collectives/exclusive_scan.hpp +++ b/libs/full/collectives/include/hpx/collectives/exclusive_scan.hpp @@ -148,7 +148,14 @@ namespace hpx { namespace collectives { namespace hpx::traits { namespace communication { + struct exclusive_scan_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "exclusive_scan"; + } } // namespace communication /////////////////////////////////////////////////////////////////////////// diff --git a/libs/full/collectives/include/hpx/collectives/gather.hpp b/libs/full/collectives/include/hpx/collectives/gather.hpp index 2794f63b26cd..c3def8c3cd37 100644 --- a/libs/full/collectives/include/hpx/collectives/gather.hpp +++ b/libs/full/collectives/include/hpx/collectives/gather.hpp @@ -234,6 +234,12 @@ namespace hpx::traits { namespace communication { struct gather_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "gather"; + } } // namespace communication template diff --git a/libs/full/collectives/include/hpx/collectives/inclusive_scan.hpp b/libs/full/collectives/include/hpx/collectives/inclusive_scan.hpp index e571949c80c0..3f2eb693da73 100644 --- a/libs/full/collectives/include/hpx/collectives/inclusive_scan.hpp +++ b/libs/full/collectives/include/hpx/collectives/inclusive_scan.hpp @@ -138,6 +138,12 @@ namespace hpx::traits { namespace communication { struct inclusive_scan_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "inclusive_scan"; + } } // namespace communication /////////////////////////////////////////////////////////////////////////// diff --git a/libs/full/collectives/include/hpx/collectives/reduce.hpp b/libs/full/collectives/include/hpx/collectives/reduce.hpp index 438ee9a41b07..ba473fd02950 100644 --- a/libs/full/collectives/include/hpx/collectives/reduce.hpp +++ b/libs/full/collectives/include/hpx/collectives/reduce.hpp @@ -235,6 +235,12 @@ namespace hpx::traits { namespace communication { struct reduce_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "reduce"; + } } // namespace communication /////////////////////////////////////////////////////////////////////////// diff --git a/libs/full/collectives/include/hpx/collectives/scatter.hpp b/libs/full/collectives/include/hpx/collectives/scatter.hpp index e308f3f14d85..3eeb6606c1d2 100644 --- a/libs/full/collectives/include/hpx/collectives/scatter.hpp +++ b/libs/full/collectives/include/hpx/collectives/scatter.hpp @@ -227,6 +227,12 @@ namespace hpx::traits { namespace communication { struct scatter_tag; + + template <> + constexpr char const* communicator_name() noexcept + { + return "scatter"; + } } // namespace communication template From c24319d899521cf4286657023a39f4680d8206a7 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Fri, 1 Sep 2023 13:21:23 -0500 Subject: [PATCH 234/333] Add coverage testing on Jenkins --- .jenkins/lsu-test-coverage/Jenkinsfile | 61 ++++++++++++++++ .jenkins/lsu-test-coverage/batch.sh | 53 ++++++++++++++ .jenkins/lsu-test-coverage/entry.sh | 69 +++++++++++++++++++ .../lsu-test-coverage/env-test-coverage.sh | 14 ++++ .../slurm-constraint-test-coverage.sh | 7 ++ 5 files changed, 204 insertions(+) create mode 100755 .jenkins/lsu-test-coverage/Jenkinsfile create mode 100755 .jenkins/lsu-test-coverage/batch.sh create mode 100755 .jenkins/lsu-test-coverage/entry.sh create mode 100755 .jenkins/lsu-test-coverage/env-test-coverage.sh create mode 100755 .jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh diff --git a/.jenkins/lsu-test-coverage/Jenkinsfile b/.jenkins/lsu-test-coverage/Jenkinsfile new file mode 100755 index 000000000000..1267b7684314 --- /dev/null +++ b/.jenkins/lsu-test-coverage/Jenkinsfile @@ -0,0 +1,61 @@ +#!groovy + +// Copyright (c) 2020 ETH Zurich +// Copyright (c) 2022 Hartmut Kaiser +// Copyright (c) 2023 Panos Syskakis +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +pipeline { + agent any + options { + buildDiscarder( + logRotator( + daysToKeepStr: "14", + numToKeepStr: "50", + artifactDaysToKeepStr: "14", + artifactNumToKeepStr: "50" + ) + ) + } + environment { + CODACY_TOKEN = credentials('CODACY_TOKEN_HPX') + + } + stages { + stage('checkout') { + steps { + dir('hpx') { + sh ''' + #!/bin/bash -l + rm -rf * + ''' + checkout scm + echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" + } + } + } + stage('test-coverage') { + environment{ + configuration_name = "test-coverage" + } + steps { + dir('hpx') { + sh ''' + #!/bin/bash -l + .jenkins/lsu-test-coverage/entry.sh + ''' + } + } + } + } + + post { + always { + archiveArtifacts artifacts: 'hpx/jenkins-hpx-*', fingerprint: true + archiveArtifacts artifacts: 'hpx/grcov-log.txt', fingerprint: true + } + } +} diff --git a/.jenkins/lsu-test-coverage/batch.sh b/.jenkins/lsu-test-coverage/batch.sh new file mode 100755 index 000000000000..65822a346eb1 --- /dev/null +++ b/.jenkins/lsu-test-coverage/batch.sh @@ -0,0 +1,53 @@ +#!/bin/bash -l + +# Copyright (c) 2023 Panos Syskakis +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +set -eux + +src_dir="$(pwd)" +build_dir="${src_dir}/build/" + +rm -rf "${build_dir}" + +source ${src_dir}/.jenkins/lsu-test-coverage/env-${configuration_name}.sh + +ulimit -l unlimited + +set +e + +# Configure +cmake \ + -S ${src_dir} \ + -B ${build_dir} \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DHPX_WITH_CXX_STANDARD=20 \ + -DHPX_WITH_MALLOC=system \ + -DHPX_WITH_FETCH_ASIO=ON \ + -DHPX_WITH_PARCELPORT_MPI=ON \ + -DHPX_WITH_PARCELPORT_LCI=ON \ + -DHPX_WITH_FETCH_LCI=ON \ + -DCMAKE_CXX_FLAGS="-O0 --coverage" \ + -DCMAKE_EXE_LINKER_FLAGS=--coverage + + +# Build +cmake --build ${build_dir} --target tests examples + +# Run tests +ctest --test-dir ${build_dir} --output-on-failure +ctest_status=$? + + +# Tests are finished; Collect coverage data +./grcov . -s ${src_dir} -o lcov.info -t lcov --log "grcov-log.txt" --ignore-not-existing --ignore "/*" + +# Upload to Codacy +bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r lcov.info --language CPP -t ${CODACY_TOKEN} --commit-uuid ${ghprbActualCommit} + +echo "${ctest_status}" > "jenkins-hpx-${configuration_name}-ctest-status.txt" +exit $ctest_status diff --git a/.jenkins/lsu-test-coverage/entry.sh b/.jenkins/lsu-test-coverage/entry.sh new file mode 100755 index 000000000000..77450cc69c70 --- /dev/null +++ b/.jenkins/lsu-test-coverage/entry.sh @@ -0,0 +1,69 @@ +#!/bin/bash -l + +# Copyright (c) 2020 ETH Zurich +# Copyright (c) 2022 Hartmut Kaiser +# Copyright (c) 2023 Panos Syskakis +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +# Make undefined variables errors, print each command +set -eux + +# Clean up old artifacts +rm -f ./jenkins-hpx* ./grcov-log.txt + +source .jenkins/lsu-test-coverage/slurm-constraint-${configuration_name}.sh + +if [[ -z "${ghprbPullId:-}" ]]; then + # Set name of branch if not building a pull request + export git_local_branch=$(echo ${GIT_BRANCH} | cut -f2 -d'/') + job_name="jenkins-hpx-${git_local_branch}-${configuration_name}" +else + job_name="jenkins-hpx-${ghprbPullId}-${configuration_name}" + + # Cancel currently running builds on the same branch, but only for pull + # requests + scancel --verbose --verbose --verbose --verbose --jobname="${job_name}" +fi + +# delay things for a random amount of time +sleep $[(RANDOM % 10) + 1].$[(RANDOM % 10)]s + +# Fetch grcov +wget https://github.com/mozilla/grcov/releases/download/v0.8.2/grcov-linux-x86_64.tar.bz2 -O grcov.tar.bz2 \ +&& echo "32e40a984cb7ed3a60760e26071618370f10fdce2186916e7321f1dd01a6d0fd grcov.tar.bz2" | sha256sum --check --status \ +&& tar -jxf grcov.tar.bz2 \ +&& rm grcov.tar.bz2 + +if [ ! -e "grcov" ]; then + echo "Error: Failed to fetch grcov." + exit 1 +fi + +# Start the actual build +set +e +sbatch \ + --verbose --verbose --verbose --verbose \ + --job-name="${job_name}" \ + --nodes="1" \ + --partition="${configuration_slurm_partition}" \ + --time="05:00:00" \ + --output="jenkins-hpx-${configuration_name}.out" \ + --error="jenkins-hpx-${configuration_name}.err" \ + --wait .jenkins/lsu-test-coverage/batch.sh + + +# Print slurm logs +echo "= stdout ==================================================" +cat jenkins-hpx-${configuration_name}.out + +echo "= stderr ==================================================" +cat jenkins-hpx-${configuration_name}.err + +# Get build status +status_file="jenkins-hpx-${configuration_name}-ctest-status.txt" + +set -e +exit $(cat ${status_file}) diff --git a/.jenkins/lsu-test-coverage/env-test-coverage.sh b/.jenkins/lsu-test-coverage/env-test-coverage.sh new file mode 100755 index 000000000000..ab12b28817f4 --- /dev/null +++ b/.jenkins/lsu-test-coverage/env-test-coverage.sh @@ -0,0 +1,14 @@ +# Copyright (c) 2023 Panos Syskakis +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +set -eu + +module avail +module purge +module load cmake +module load gcc/12 +module load boost/1.79.0-debug +module load hwloc +module load openmpi diff --git a/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh b/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh new file mode 100755 index 000000000000..008965ef474b --- /dev/null +++ b/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh @@ -0,0 +1,7 @@ +# Copyright (c) 2023 Panos Syskakis +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +configuration_slurm_partition="jenkins-compute" From 2319e4f47955d1625caa047163e358dbec369f9e Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Sat, 9 Sep 2023 16:46:58 -0500 Subject: [PATCH 235/333] Update coverage info in README --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 928941eb7038..ce4d1923db8b 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -|circleci_status| |codacy| |coveralls| |OpenSSF| |CFF| |JOSS| |zenodo_doi| +|circleci_status| |codacy_quality| |codacy_coverage| |OpenSSF| |CFF| |JOSS| |zenodo_doi| Documentation: `latest `_, @@ -98,8 +98,8 @@ continuous integration service additionally tracks the current build status for the master branch: |circleci_status|. We use `Codacy `_ to assess the code quality of this -project: |codacy|. For our coverage analysis we rely on -`Coveralls `_ to present the results: |coveralls|. +project: |codacy_quality|. For our coverage analysis, we also rely on +Codacy to present the results: |codacy_coverage|. If you can't find what you are looking for in the documentation or you suspect you've found a bug in HPX we very much encourage and appreciate any issue @@ -151,12 +151,12 @@ Past and current funding and support for HPX is listed `here :target: https://doi.org/10.5281/zenodo.598202 :alt: Latest software release of HPX -.. |codacy| image:: https://app.codacy.com/project/badge/Grade/0b8cd5a874914edaba67ce3bb711e688 +.. |codacy_quality| image:: https://app.codacy.com/project/badge/Grade/0b8cd5a874914edaba67ce3bb711e688 :target: https://app.codacy.com/gh/STEllAR-GROUP/hpx/dashboard :alt: HPX Code Quality Assessment -.. |coveralls| image:: https://coveralls.io/repos/github/STEllAR-GROUP/hpx/badge.svg - :target: https://coveralls.io/github/STEllAR-GROUP/hpx +.. |codacy_coverage| image:: https://app.codacy.com/project/badge/Coverage/0b8cd5a874914edaba67ce3bb711e688 + :target: https://app.codacy.com/gh/STEllAR-GROUP/hpx/dashboard :alt: HPX coverage report .. |JOSS| image:: https://joss.theoj.org/papers/022e5917b95517dff20cd3742ab95eca/status.svg From bdeff91a7597279012e538ebe0e4c853bd53d80b Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 10 Sep 2023 10:44:27 -0500 Subject: [PATCH 236/333] Fixing formatting --- cmake/HPX_UpdateGitDocs.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/HPX_UpdateGitDocs.cmake b/cmake/HPX_UpdateGitDocs.cmake index 6ecea2906076..092aa4f653dd 100644 --- a/cmake/HPX_UpdateGitDocs.cmake +++ b/cmake/HPX_UpdateGitDocs.cmake @@ -58,7 +58,9 @@ string(REGEX REPLACE " " ";" HPX_WITH_DOCUMENTATION_OUTPUT_FORMATS # If a branch name has been set, we copy files to a corresponding directory if(HPX_WITH_GIT_BRANCH) - message("Updating branch directory, " "HPX_WITH_GIT_BRANCH=\"${HPX_WITH_GIT_BRANCH}\"") + message("Updating branch directory, " + "HPX_WITH_GIT_BRANCH=\"${HPX_WITH_GIT_BRANCH}\"" + ) set(DOCS_BRANCH_DEST "${HPX_BINARY_DIR}/docs/gh-pages/branches/${HPX_WITH_GIT_BRANCH}" ) From 26d0df2599e62af6aa2de500fd2ceccf659301c2 Mon Sep 17 00:00:00 2001 From: Johan511 Date: Tue, 12 Sep 2023 00:04:33 +0530 Subject: [PATCH 237/333] changed chat link to discord Signed-off-by: Johan511 --- .github/SUPPORT.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md index 86d132b940f9..964e6db778c3 100644 --- a/.github/SUPPORT.md +++ b/.github/SUPPORT.md @@ -21,16 +21,11 @@ Please send us an [email](info@stellar-group.org) describing the issue. See our [security policy](https://github.com/STEllAR-GROUP/hpx/blob/master/.github/SECURITY.md) for details. -### Real-time Chat +### Chat -* #ste||ar on [Libera.Chat](https://libera.chat/): this is the best option +* #ste||ar on [Discord](https://discord.gg/Tn9QuzVjvy): this is the best option to get in contact with us as most of the developers are available on this - channel. Check out the [channel archives](http://irclog.cct.lsu.edu/ste~b~~b~ar) - for past conversations. The channel can also be accessed through - [Matrix](https://matrix.to/#/#ste||ar:libera.chat). -* Slack ([registration](https://cpplang.now.sh/)): The - [`#hpx`](https://cpplang.slack.com/messages/C68QLPZB3) channel is another - place where people usually offer support. + channel. ### Mailing Lists/Groups From 7e3cc679c2584940329b2f5d4e5342a0b93663f6 Mon Sep 17 00:00:00 2001 From: Johan511 Date: Tue, 12 Sep 2023 00:15:43 +0530 Subject: [PATCH 238/333] added cpp slack channel Signed-off-by: Johan511 --- .github/SUPPORT.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md index 964e6db778c3..46f50f619e67 100644 --- a/.github/SUPPORT.md +++ b/.github/SUPPORT.md @@ -26,6 +26,9 @@ for details. * #ste||ar on [Discord](https://discord.gg/Tn9QuzVjvy): this is the best option to get in contact with us as most of the developers are available on this channel. +* Slack ([registration](https://cpplang.now.sh/)): The + [`#hpx`](https://cpplang.slack.com/messages/C68QLPZB3) channel is another + place where people usually offer support. ### Mailing Lists/Groups From f0e8a37b99c8780e5a6c9b8bed45e87cc7d30647 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Sat, 16 Sep 2023 09:53:50 -0500 Subject: [PATCH 239/333] Make coverage work with Jenkins Github Branch Source plugin --- .jenkins/lsu-test-coverage/batch.sh | 2 +- .jenkins/lsu-test-coverage/entry.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.jenkins/lsu-test-coverage/batch.sh b/.jenkins/lsu-test-coverage/batch.sh index 65822a346eb1..07537741a0f7 100755 --- a/.jenkins/lsu-test-coverage/batch.sh +++ b/.jenkins/lsu-test-coverage/batch.sh @@ -47,7 +47,7 @@ ctest_status=$? ./grcov . -s ${src_dir} -o lcov.info -t lcov --log "grcov-log.txt" --ignore-not-existing --ignore "/*" # Upload to Codacy -bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r lcov.info --language CPP -t ${CODACY_TOKEN} --commit-uuid ${ghprbActualCommit} +bash <(curl -Ls https://coverage.codacy.com/get.sh) report -r lcov.info --language CPP -t ${CODACY_TOKEN} --commit-uuid ${GIT_COMMIT} echo "${ctest_status}" > "jenkins-hpx-${configuration_name}-ctest-status.txt" exit $ctest_status diff --git a/.jenkins/lsu-test-coverage/entry.sh b/.jenkins/lsu-test-coverage/entry.sh index 77450cc69c70..300fa0aaac80 100755 --- a/.jenkins/lsu-test-coverage/entry.sh +++ b/.jenkins/lsu-test-coverage/entry.sh @@ -16,12 +16,12 @@ rm -f ./jenkins-hpx* ./grcov-log.txt source .jenkins/lsu-test-coverage/slurm-constraint-${configuration_name}.sh -if [[ -z "${ghprbPullId:-}" ]]; then +if [[ -z "${CHANGE_ID:-}" ]]; then # Set name of branch if not building a pull request export git_local_branch=$(echo ${GIT_BRANCH} | cut -f2 -d'/') job_name="jenkins-hpx-${git_local_branch}-${configuration_name}" else - job_name="jenkins-hpx-${ghprbPullId}-${configuration_name}" + job_name="jenkins-hpx-${CHANGE_ID}-${configuration_name}" # Cancel currently running builds on the same branch, but only for pull # requests From 38b470702cbe3d50fc1fd70a75fcb06f6bd1d4a9 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Sat, 16 Sep 2023 11:54:01 -0500 Subject: [PATCH 240/333] Cleanup Jenkins scripts --- .jenkins/lsu-perftests/Jenkinsfile | 51 ++++++---------------- .jenkins/lsu-perftests/entry.sh | 3 -- .jenkins/lsu-perftests/launch_perftests.sh | 11 ----- .jenkins/lsu-test-coverage/Jenkinsfile | 31 ++++--------- .jenkins/lsu-test-coverage/entry.sh | 3 -- 5 files changed, 22 insertions(+), 77 deletions(-) diff --git a/.jenkins/lsu-perftests/Jenkinsfile b/.jenkins/lsu-perftests/Jenkinsfile index fdc1f585617f..0047ee63a43b 100644 --- a/.jenkins/lsu-perftests/Jenkinsfile +++ b/.jenkins/lsu-perftests/Jenkinsfile @@ -2,6 +2,7 @@ // Copyright (c) 2020 ETH Zurich // Copyright (c) 2022 Hartmut Kaiser +// Copyright (c) 2023 Panos Syskakis // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying @@ -19,51 +20,27 @@ pipeline { ) ) } - environment { - GITHUB_TOKEN = credentials('STELLARBOT_GITHUB_TOKEN') - } stages { - stage('checkout') { - steps { - dir('hpx') { - sh ''' - #!/bin/bash -l - rm -rf * - ''' - checkout scm - echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" - } - } - } stage('perftests') { - matrix { - axes { - axis { - name 'configuration_name' - values 'perftests' - } - } - stages { - stage('perftests') { - steps { - dir('hpx') { - sh ''' - #!/bin/bash -l - .jenkins/lsu-perftests/entry.sh - ''' - } - } - } - } + environment{ + GITHUB_TOKEN = credentials('STELLARBOT_GITHUB_TOKEN') + configuration_name = "perftests" + } + steps { + echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" + sh ''' + #!/bin/bash -l + .jenkins/lsu-perftests/entry.sh + ''' } } } post { always { - archiveArtifacts artifacts: 'hpx/jenkins-hpx-*', fingerprint: true - archiveArtifacts artifacts: 'hpx/*-Testing/**', fingerprint: true - archiveArtifacts artifacts: 'hpx/*-reports/**', fingerprint: true + archiveArtifacts artifacts: 'jenkins-hpx-*', fingerprint: true + archiveArtifacts artifacts: '*-Testing/**', fingerprint: true + archiveArtifacts artifacts: '*-reports/**', fingerprint: true } } } diff --git a/.jenkins/lsu-perftests/entry.sh b/.jenkins/lsu-perftests/entry.sh index f87d9ffd7cbc..fff1653b3c76 100755 --- a/.jenkins/lsu-perftests/entry.sh +++ b/.jenkins/lsu-perftests/entry.sh @@ -10,9 +10,6 @@ # Make undefined variables errors, print each command set -eux -# Clean up old artifacts -rm -f ./jenkins-hpx* ./*-Testing - source .jenkins/lsu-perftests/slurm-constraint-${configuration_name}.sh if [[ -z "${ghprbPullId:-}" ]]; then diff --git a/.jenkins/lsu-perftests/launch_perftests.sh b/.jenkins/lsu-perftests/launch_perftests.sh index ffe0f95f60f9..1c486d2912d9 100644 --- a/.jenkins/lsu-perftests/launch_perftests.sh +++ b/.jenkins/lsu-perftests/launch_perftests.sh @@ -24,17 +24,6 @@ hpx_test_options=( --vector_size=518176 --hpx:threads=4 --iterations=200 \ --warmup_iterations=20") - - # "--hpx:ini=hpx.thread_queue.init_threads_count=100 \ - # --hpx:threads=4 --vector_size=10000 --work_delay=1 \ - # --chunk_size=0 --test_count=5000" - # "--hpx:ini=hpx.thread_queue.init_threads_count=100 \ - # --hpx:queuing=local-priority --hpx:threads=4 --test-all \ - # --repetitions=100 --futures=500000" - # "--hpx:ini=hpx.thread_queue.init_threads_count=100 \ - # --vector_size=1048576 --hpx:threads=4 --iterations=5000 \ - # --warmup_iterations=500") - # Build binaries for performance tests ${perftests_dir}/driver.py -v -l $logfile build -b release -o build \ --source-dir ${src_dir} --build-dir ${build_dir} -e $envfile \ diff --git a/.jenkins/lsu-test-coverage/Jenkinsfile b/.jenkins/lsu-test-coverage/Jenkinsfile index 1267b7684314..47ebcee6d19b 100755 --- a/.jenkins/lsu-test-coverage/Jenkinsfile +++ b/.jenkins/lsu-test-coverage/Jenkinsfile @@ -20,42 +20,27 @@ pipeline { ) ) } - environment { - CODACY_TOKEN = credentials('CODACY_TOKEN_HPX') - } stages { - stage('checkout') { - steps { - dir('hpx') { - sh ''' - #!/bin/bash -l - rm -rf * - ''' - checkout scm - echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" - } - } - } stage('test-coverage') { environment{ + CODACY_TOKEN = credentials('CODACY_TOKEN_HPX') configuration_name = "test-coverage" } steps { - dir('hpx') { - sh ''' - #!/bin/bash -l - .jenkins/lsu-test-coverage/entry.sh - ''' - } + echo "Running ${env.BUILD_ID} on ${env.JENKINS_URL}" + sh ''' + #!/bin/bash -l + .jenkins/lsu-test-coverage/entry.sh + ''' } } } post { always { - archiveArtifacts artifacts: 'hpx/jenkins-hpx-*', fingerprint: true - archiveArtifacts artifacts: 'hpx/grcov-log.txt', fingerprint: true + archiveArtifacts artifacts: 'jenkins-hpx-*', fingerprint: true + archiveArtifacts artifacts: 'grcov-log.txt', fingerprint: true } } } diff --git a/.jenkins/lsu-test-coverage/entry.sh b/.jenkins/lsu-test-coverage/entry.sh index 300fa0aaac80..85d6c23c441e 100755 --- a/.jenkins/lsu-test-coverage/entry.sh +++ b/.jenkins/lsu-test-coverage/entry.sh @@ -11,9 +11,6 @@ # Make undefined variables errors, print each command set -eux -# Clean up old artifacts -rm -f ./jenkins-hpx* ./grcov-log.txt - source .jenkins/lsu-test-coverage/slurm-constraint-${configuration_name}.sh if [[ -z "${CHANGE_ID:-}" ]]; then From 2c89c1ca530de854bb82a3f15b0b3d7fe070e8ce Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 21 Sep 2023 19:43:44 +0200 Subject: [PATCH 241/333] Fix missing links for header files --- docs/sphinx/api/public_api.rst | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/sphinx/api/public_api.rst b/docs/sphinx/api/public_api.rst index e7a0a55784dd..c81e8e6cd581 100644 --- a/docs/sphinx/api/public_api.rst +++ b/docs/sphinx/api/public_api.rst @@ -23,7 +23,7 @@ sub-namespaces will eventually be removed. ``hpx/algorithm.hpp`` ===================== -The header :hpx-header:`libs/full/include,hpx/algorithm.hpp` corresponds to the +The header :hpx-header:`libs/full/include,include/hpx/algorithm.hpp` corresponds to the C++ standard library header :cppreference-header:`algorithm`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -191,7 +191,7 @@ Functions ``hpx/any.hpp`` =============== -The header :hpx-header:`libs/full/include,hpx/any.hpp` corresponds to the C++ +The header :hpx-header:`libs/core/include_local,include/hpx/any.hpp` corresponds to the C++ standard library header :cppreference-header:`any`. :cpp:type:`hpx::any` is compatible with ``std::any``. @@ -253,7 +253,7 @@ Macros ``hpx/barrier.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/barrier.hpp` corresponds to the +The header :hpx-header:`libs/full/include,include/hpx/barrier.hpp` corresponds to the C++ standard library header :cppreference-header:`barrier` and contains a distributed barrier implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -282,7 +282,7 @@ Classes ``hpx/channel.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/channel.hpp` contains a local and a +The header :hpx-header:`libs/full/include,include/hpx/channel.hpp` contains a local and a distributed channel implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -310,7 +310,7 @@ Classes ``hpx/chrono.hpp`` ================== -The header :hpx-header:`libs/full/include,hpx/chrono.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/chrono.hpp` corresponds to the C++ standard library header :cppreference-header:`chrono`. The following replacements and extensions are provided compared to :cppreference-header:`chrono`. @@ -332,7 +332,7 @@ Classes ``hpx/condition_variable.hpp`` ============================== -The header :hpx-header:`libs/full/include,hpx/condition_variable.hpp` corresponds to the C++ +The header :hpx-header:`libs/core/include_local,include/hpx/condition_variable.hpp` corresponds to the C++ standard library header :cppreference-header:`condition_variable`. Classes @@ -353,7 +353,7 @@ Classes ``hpx/exception.hpp`` ===================== -The header :hpx-header:`libs/full/include,hpx/exception.hpp` corresponds to +The header :hpx-header:`libs/core/include_local,include/hpx/exception.hpp` corresponds to the C++ standard library header :cppreference-header:`exception`. :cpp:class:`hpx::exception` extends ``std::exception`` and is the base class for all exceptions thrown in |hpx|. :c:macro:`HPX_THROW_EXCEPTION` can be used to throw |hpx| exceptions with file and line information @@ -380,7 +380,7 @@ Classes ``hpx/execution.hpp`` ===================== -The header :hpx-header:`libs/full/include,hpx/execution.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/execution.hpp` corresponds to the C++ standard library header :cppreference-header:`execution`. See :ref:`parallel`, :ref:`parallel_algorithms` and :ref:`executor_parameters` for more information about execution policies and executor parameters. @@ -430,7 +430,7 @@ Classes ``hpx/functional.hpp`` ====================== -The header :hpx-header:`libs/full/include,hpx/functional.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/functional.hpp` corresponds to the C++ standard library header :cppreference-header:`functional`. :cpp:class:`hpx::function` is a more efficient and serializable replacement for ``std::function``. @@ -493,7 +493,7 @@ Functions ``hpx/future.hpp`` ================== -The header :hpx-header:`libs/full/include,hpx/future.hpp` corresponds to the +The header :hpx-header:`libs/full/include,include/hpx/future.hpp` corresponds to the C++ standard library header :cppreference-header:`future`. See :ref:`extend_futures` for more information about extensions to futures compared to the C++ standard library. @@ -610,7 +610,7 @@ Functions ``hpx/latch.hpp`` ================= -The header :hpx-header:`libs/full/include,hpx/latch.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include,include/hpx/latch.hpp` corresponds to the C++ standard library header :cppreference-header:`latch`. It contains a local and a distributed latch implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -639,7 +639,7 @@ Classes ``hpx/mutex.hpp`` ================= -The header :hpx-header:`libs/full/include,hpx/mutex.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/mutex.hpp` corresponds to the C++ standard library header :cppreference-header:`mutex`. Classes @@ -675,7 +675,7 @@ Functions ``hpx/memory.hpp`` ================== -The header :hpx-header:`libs/full/include,hpx/memory.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/memory.hpp` corresponds to the C++ standard library header :cppreference-header:`memory`. It contains parallel versions of the copy, fill, move, and construct helper functions in :cppreference-header:`memory`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -722,7 +722,7 @@ Functions ``hpx/numeric.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/numeric.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/numeric.hpp` corresponds to the C++ standard library header :cppreference-header:`numeric`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -762,7 +762,7 @@ Functions ``hpx/optional.hpp`` ==================== -The header :hpx-header:`libs/full/include,hpx/optional.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/optional.hpp` corresponds to the C++ standard library header :cppreference-header:`optional`. :cpp:type:`hpx::optional` is compatible with ``std::optional``. @@ -789,7 +789,7 @@ Classes ``hpx/runtime.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/runtime.hpp` contains functions for accessing +The header :hpx-header:`libs/full/include,include/hpx/runtime.hpp` contains functions for accessing local and distributed runtime information. Typedefs @@ -849,7 +849,7 @@ Functions ``hpx/source_location.hpp`` =========================== -The header :hpx-header:`libs/full/include,hpx/source_location.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/source_location.hpp` corresponds to the C++ standard library header :cppreference-header:`source_location`. Classes @@ -868,7 +868,7 @@ Classes ``hpx/system_error.hpp`` ======================== -The header :hpx-header:`libs/full/include,hpx/system_error.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/system_error.hpp` corresponds to the C++ standard library header :cppreference-header:`system_error`. Classes @@ -887,7 +887,7 @@ Classes ``hpx/task_block.hpp`` ====================== -The header :hpx-header:`libs/full/include,hpx/task_block.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/task_block.hpp` corresponds to the ``task_block`` feature in |cpp17_n4755|_. See :ref:`using_task_block` for more details on using task blocks. @@ -922,7 +922,7 @@ Functions ``hpx/experimental/task_group.hpp`` =================================== -The header :hpx-header:`libs/core/include,hpx/experimental/task_group.hpp` +The header :hpx-header:`libs/core/include_local,include/hpx/experimental/task_group.hpp` corresponds to the ``task_group`` feature in |oneTBB|_. Classes @@ -941,7 +941,7 @@ Classes ``hpx/thread.hpp`` ================== -The header :hpx-header:`libs/full/include,hpx/thread.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/thread.hpp` corresponds to the C++ standard library header :cppreference-header:`thread`. The functionality in this header is equivalent to the standard library thread functionality, with the exception that the |hpx| equivalents are implemented on top of lightweight threads and the |hpx| runtime. @@ -977,7 +977,7 @@ Functions ``hpx/semaphore.hpp`` ===================== -The header :hpx-header:`libs/full/include,hpx/semaphore.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/semaphore.hpp` corresponds to the C++ standard library header :cppreference-header:`semaphore`. Classes @@ -997,7 +997,7 @@ Classes ``hpx/shared_mutex.hpp`` ======================== -The header :hpx-header:`libs/full/include,hpx/shared_mutex.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/shared_mutex.hpp` corresponds to the C++ standard library header :cppreference-header:`shared_mutex`. Classes @@ -1016,7 +1016,7 @@ Classes ``hpx/stop_token.hpp`` ====================== -The header :hpx-header:`libs/full/include,hpx/stop_token.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/stop_token.hpp` corresponds to the C++ standard library header :cppreference-header:`stop_token`. Constants @@ -1049,7 +1049,7 @@ Classes ``hpx/tuple.hpp`` ================= -The header :hpx-header:`libs/full/include,hpx/tuple.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/tuple.hpp` corresponds to the C++ standard library header :cppreference-header:`tuple`. :cpp:class:`hpx::tuple` can be used in CUDA device code, unlike ``std::tuple``. @@ -1097,7 +1097,7 @@ Functions ``hpx/type_traits.hpp`` ======================= -The header :hpx-header:`libs/full/include,hpx/type_traits.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local,include/hpx/type_traits.hpp` corresponds to the C++ standard library header :cppreference-header:`type_traits`. Classes @@ -1117,7 +1117,7 @@ Classes ``hpx/unwrap.hpp`` ================== -The header :hpx-header:`libs/full/include,hpx/unwrap.hpp` contains utilities for +The header :hpx-header:`libs/fcore/include_local,include/hpx/unwrap.hpp` contains utilities for unwrapping futures. Classes From e2bf58bcd85f0c7441e25cfcdf0242d661efc5e7 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 21 Sep 2023 12:40:22 -0500 Subject: [PATCH 242/333] Enable using std::reference_wrapper with executor parameters --- .../traits/is_executor_parameters.hpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/libs/core/execution_base/include/hpx/execution_base/traits/is_executor_parameters.hpp b/libs/core/execution_base/include/hpx/execution_base/traits/is_executor_parameters.hpp index 8f63aa16d57e..87ff1af7891f 100644 --- a/libs/core/execution_base/include/hpx/execution_base/traits/is_executor_parameters.hpp +++ b/libs/core/execution_base/include/hpx/execution_base/traits/is_executor_parameters.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022 Hartmut Kaiser +// Copyright (c) 2014-2023 Hartmut Kaiser // Copyright (c) 2016 Marcin Copik // // SPDX-License-Identifier: BSL-1.0 @@ -64,6 +64,12 @@ namespace hpx::parallel::execution { { }; + template + struct extract_has_variable_chunk_size<::std::reference_wrapper> + : extract_has_variable_chunk_size + { + }; + template inline constexpr bool extract_has_variable_chunk_size_v = extract_has_variable_chunk_size::value; @@ -88,6 +94,13 @@ namespace hpx::parallel::execution { }; #endif + template + struct extract_invokes_testing_function< + ::std::reference_wrapper> + : extract_invokes_testing_function + { + }; + template inline constexpr bool extract_invokes_testing_function_v = extract_invokes_testing_function::value; From 96e1945c78ee93e17390e544c3f4ba8bdbd8f475 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 16:27:02 -0400 Subject: [PATCH 243/333] initial import of gasnet parcelport --- cmake/FindGasnet.cmake | 46 ++ cmake/HPX_SetupGASNET.cmake | 48 ++ cmake/HPX_SetupGasnet.cmake | 763 +----------------- libs/core/CMakeLists.txt | 1 + libs/core/gasnet_base/CMakeLists.txt | 2 +- libs/core/gasnet_base/README.rst | 16 + libs/core/gasnet_base/docs/index.rst | 18 + .../gasnet_base/src/gasnet_environment.cpp | 1 - libs/full/CMakeLists.txt | 1 + libs/full/parcelport_gasnet/CMakeLists.txt | 1 + libs/full/parcelport_gasnet/README.rst | 16 + libs/full/parcelport_gasnet/docs/index.rst | 18 + .../include/hpx/parcelport_gasnet/header.hpp | 2 +- .../src/parcelport_gasnet.cpp | 3 +- 14 files changed, 198 insertions(+), 738 deletions(-) create mode 100644 cmake/FindGasnet.cmake create mode 100644 cmake/HPX_SetupGASNET.cmake create mode 100644 libs/core/gasnet_base/README.rst create mode 100644 libs/core/gasnet_base/docs/index.rst create mode 100644 libs/full/parcelport_gasnet/README.rst create mode 100644 libs/full/parcelport_gasnet/docs/index.rst diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake new file mode 100644 index 000000000000..9a20e8b33a5a --- /dev/null +++ b/cmake/FindGasnet.cmake @@ -0,0 +1,46 @@ +# Copyright (c) 2017 Thomas Heller +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +macro(find_gasnet) +set(GASNET_MPI_FOUND FALSE) +set(GASNET_UDP_FOUND FALSE) +set(GASNET_SMP_FOUND FALSE) + +if(HPX_WITH_NETWORKING) + pkg_check_modules(GASNET QUIET gasnet-mpi-seq) + if(NOT GASNET_FOUND) + pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") + else() + set(GASNET_UDP_FOUND TRUE) + endif() + else() + set(GASNET_MPI_FOUND TRUE) + hpx_setup_mpi() + endif() +else() + pkg_check_modules(GASNET QUIET gasnet-smp-seq) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET please set the PKG_CONFIG_PATH environment variable") + else() + set(GASNET_SMP_FOUND TRUE) + endif() +endif() + +if(NOT GASNET_INCLUDE_DIRS OR NOT GASNET_LIBRARY_DIRS) + hpx_error("Could not find GASNET_INCLUDE_DIRS or GASNET_LIBRARY_DIRS please \ + set the PKG_CONFIG_PATH environment variable" + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Gasnet DEFAULT_MSG GASNET_LIBRARY_DIRS GASNET_INCLUDE_DIRS +) + +endmacro() diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake new file mode 100644 index 000000000000..37957d79ba46 --- /dev/null +++ b/cmake/HPX_SetupGASNET.cmake @@ -0,0 +1,48 @@ +# Copyright (c) 2019-2022 Ste||ar Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(FindGasnet) + +macro(hpx_setup_gasnet) + + if(NOT GASNET_FOUND) + + find_gasnet() + + set(GASNET_CXXFLAGS) + set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_LDFLAGS) + + foreach(TOKEN ${GASNET_CFLAGS}) + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endforeach() + + foreach(TOKEN ${GASNET_LDFLAGS_TMP}) + list(APPEND GASNET_LDFLAGS ${TOKEN}) + endforeach() + + add_library(gasnet INTERFACE IMPORTED) + target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) + target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) + target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) + + if(GASNET_MPI_FOUND) + # Setup PMI imported target + find_package(PMI) + if(PMI_FOUND) + hpx_add_config_define_namespace( + DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET + ) + add_library(Pmi::pmi INTERFACE IMPORTED) + target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) + target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) + endif() + endif() + + endif() +endmacro() diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index a648bd4a5f71..37957d79ba46 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -1,751 +1,48 @@ -# Copyright (c) 2017 Thomas Heller -# Copyright (c) 2023 Christopher Taylor +# Copyright (c) 2019-2022 Ste||ar Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# - -macro(hpx_setup_gasnet) - - if(NOT TARGET PkgConfig::GASNET) - - find_package(PkgConfig REQUIRED QUIET COMPONENTS) - - hpx_info( - "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" - ) - pkg_search_module( - GASNET IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - - if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) - - if(NOT CMAKE_C_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") - endif() - if(NOT CMAKE_CXX_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") - endif() - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") - endif() - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") - endif() - - include(FetchContent) - FetchContent_Declare(gasnet - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz - ) - - fetchcontent_getproperties(gasnet) - if(NOT gasnet) - FetchContent_Populate(gasnet) - endif() - - message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") - - set(GASNET_DIR "${gasnet_SOURCE_DIR}") - set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") - set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") - - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - endif() - - if(GASNET_BUILD_STATUS) - message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") - endif() - - pkg_search_module( - GASNET REQUIRED IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) - message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") - endif() - - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") - set(GASNET_MPI_FOUND TRUE) - include(HPX_SetupMPI) - hpx_setup_mpi() - endif() - - if(GASNET_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - if(GASNET_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") +include(HPX_Message) +include(FindGasnet) - foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - - endif() - - if(GASNET_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - - if(GASNET_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") +macro(hpx_setup_gasnet) - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() + if(NOT GASNET_FOUND) - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() + find_gasnet() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") - endforeach() + set(GASNET_CXXFLAGS) + set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_LDFLAGS) - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) + foreach(TOKEN ${GASNET_CFLAGS}) + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endforeach() - list(LENGTH FOUND_LIB IDX) + foreach(TOKEN ${GASNET_LDFLAGS_TMP}) + list(APPEND GASNET_LDFLAGS ${TOKEN}) + endforeach() - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) + add_library(gasnet INTERFACE IMPORTED) + target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) + target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) + target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + if(GASNET_MPI_FOUND) + # Setup PMI imported target + find_package(PMI) + if(PMI_FOUND) + hpx_add_config_define_namespace( + DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() + add_library(Pmi::pmi INTERFACE IMPORTED) + target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) + target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) + endif() endif() - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES - "${GASNET_LIBRARY_DIRS}" - ) - endif() - endmacro() diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 3e6550e93f25..9b9666358b20 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,6 +56,7 @@ set(_hpx_core_modules logging memory mpi_base + gasnet_base pack_traversal plugin prefix diff --git a/libs/core/gasnet_base/CMakeLists.txt b/libs/core/gasnet_base/CMakeLists.txt index 6a6401e03b0e..b3049cd8ce6e 100644 --- a/libs/core/gasnet_base/CMakeLists.txt +++ b/libs/core/gasnet_base/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Chistopher Taylor +# Copyright (c) 2019-2023 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying diff --git a/libs/core/gasnet_base/README.rst b/libs/core/gasnet_base/README.rst new file mode 100644 index 000000000000..1ae4ac39ce3b --- /dev/null +++ b/libs/core/gasnet_base/README.rst @@ -0,0 +1,16 @@ + +.. + Copyright (c) 2020 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +======== +mpi_base +======== + +This library is part of HPX. + +Documentation can be found `here +`__. diff --git a/libs/core/gasnet_base/docs/index.rst b/libs/core/gasnet_base/docs/index.rst new file mode 100644 index 000000000000..43dbdb7c2a03 --- /dev/null +++ b/libs/core/gasnet_base/docs/index.rst @@ -0,0 +1,18 @@ +.. + Copyright (c) 2020 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _modules_mpi_base: + +======== +mpi_base +======== + +This module provides helper functionality for detecting MPI environments. + +See the :ref:`API reference ` of this module for more +details. + diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 3717970a99dc..2ef7004c6966 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -391,7 +391,6 @@ namespace hpx::util { char message[1024 + 1]; std::snprintf(message, 1024 + 1, "%s\n", gasnet_ErrorDesc(retval)); - std::string msg("gasnet_environment::init: gasnet_init failed: "); msg = msg + message + "."; throw std::runtime_error(msg.c_str()); diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index e4a7eb24b923..b6861762f349 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,6 +35,7 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi + parcelport_gasnet parcelport_tcp parcelports parcelset diff --git a/libs/full/parcelport_gasnet/CMakeLists.txt b/libs/full/parcelport_gasnet/CMakeLists.txt index 50c4380e81f2..33cb2af92b42 100644 --- a/libs/full/parcelport_gasnet/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright (c) 2023 Chistopher Taylor +# Copyright (c) 2019-2021 The STE||AR-Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying diff --git a/libs/full/parcelport_gasnet/README.rst b/libs/full/parcelport_gasnet/README.rst new file mode 100644 index 000000000000..c0729a670a32 --- /dev/null +++ b/libs/full/parcelport_gasnet/README.rst @@ -0,0 +1,16 @@ + +.. + Copyright (c) 2020-2021 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +================= +parcelport_gasnet +================= + +This module is part of HPX. + +Documentation can be found `here +`__. diff --git a/libs/full/parcelport_gasnet/docs/index.rst b/libs/full/parcelport_gasnet/docs/index.rst new file mode 100644 index 000000000000..d61191bdd7d7 --- /dev/null +++ b/libs/full/parcelport_gasnet/docs/index.rst @@ -0,0 +1,18 @@ +.. + Copyright (c) 2020-2021 The STE||AR-Group + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _modules_parcelport_gasnet: + +============== +parcelport_gasnet +============== + +TODO: High-level description of the module. + +See the :ref:`API reference ` of this module for more +details. + diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp index 886118b244cf..23af0fae6418 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -1,7 +1,7 @@ // Copyright (c) 2023 Christopher Taylor // Copyright (c) 2013-2021 Hartmut Kaiser // Copyright (c) 2013-2015 Thomas Heller -// +/ // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index 2dd46216b8ae..dee35f5af8af 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -110,7 +110,7 @@ namespace hpx::parcelset { return hpx::util::get_entry_as(ini, "hpx.parcel.gasnet.background_threads", HPX_HAVE_PARCELPORT_GASNET_BACKGROUND_THREADS); -*/ + */ return 1UL; } @@ -234,7 +234,6 @@ namespace hpx::parcelset { void io_service_work() { std::size_t k = 0; - // We only execute work on the IO service while HPX is starting while (hpx::is_starting()) { From 960528ef22e3b75621413fd47e62e017978607b2 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 24 Jun 2023 22:39:57 -0400 Subject: [PATCH 244/333] clean compile --- libs/core/CMakeLists.txt | 1 - libs/full/CMakeLists.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 9b9666358b20..3e6550e93f25 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,7 +56,6 @@ set(_hpx_core_modules logging memory mpi_base - gasnet_base pack_traversal plugin prefix diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index b6861762f349..e4a7eb24b923 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,7 +35,6 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi - parcelport_gasnet parcelport_tcp parcelports parcelset From f1bf4c3111c282eb3367bbfe74fc930c9d8b6d21 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 25 Sep 2023 13:09:26 -0400 Subject: [PATCH 245/333] fixed potential nullptr issue --- libs/core/gasnet_base/src/gasnet_environment.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 2ef7004c6966..520ea6820fae 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -611,8 +611,12 @@ namespace hpx::util { if (enabled() && has_called_init()) { gasnet_exit(1); - delete gasnet_environment::segments; - delete gasnet_environment::segment_mutex; + if(gasnet_environment::segments != nullptr) { + delete gasnet_environment::segments; + } + if(gasnet_environment::segments != segment_mutex) { + delete gasnet_environment::segment_mutex; + } } } From 6773638b35821eded361eecbb09045329c496fd8 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 25 Sep 2023 21:15:02 -0400 Subject: [PATCH 246/333] fixed gasnet cmake --- cmake/HPX_SetupGasnet.cmake | 763 ++++++++++++++++++++++++++++++++++-- 1 file changed, 733 insertions(+), 30 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 37957d79ba46..a648bd4a5f71 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -1,48 +1,751 @@ -# Copyright (c) 2019-2022 Ste||ar Group +# Copyright (c) 2017 Thomas Heller +# Copyright (c) 2023 Christopher Taylor # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -include(HPX_Message) -include(FindGasnet) +# macro(hpx_setup_gasnet) - if(NOT GASNET_FOUND) + if(NOT TARGET PkgConfig::GASNET) + + find_package(PkgConfig REQUIRED QUIET COMPONENTS) + + hpx_info( + "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" + ) + pkg_search_module( + GASNET IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + + if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) + + if(NOT CMAKE_C_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") + endif() + if(NOT CMAKE_CXX_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + + include(FetchContent) + FetchContent_Declare(gasnet + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz + ) + + fetchcontent_getproperties(gasnet) + if(NOT gasnet) + FetchContent_Populate(gasnet) + endif() + + message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") + + set(GASNET_DIR "${gasnet_SOURCE_DIR}") + set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") + set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + endif() + + if(GASNET_BUILD_STATUS) + message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") + endif() + + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) + message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") + endif() + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(GASNET_MPI_FOUND TRUE) + include(HPX_SetupMPI) + hpx_setup_mpi() + endif() + + if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() - find_gasnet() + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() - set(GASNET_CXXFLAGS) - set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) - set(GASNET_LDFLAGS) + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() - foreach(TOKEN ${GASNET_CFLAGS}) - list(APPEND GASNET_CXXFLAGS ${TOKEN}) - endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) - foreach(TOKEN ${GASNET_LDFLAGS_TMP}) - list(APPEND GASNET_LDFLAGS ${TOKEN}) - endforeach() + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") - add_library(gasnet INTERFACE IMPORTED) - target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) - target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) - target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) - if(GASNET_MPI_FOUND) - # Setup PMI imported target - find_package(PMI) - if(PMI_FOUND) - hpx_add_config_define_namespace( - DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX ) - add_library(Pmi::pmi INTERFACE IMPORTED) - target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) - target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) - endif() + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + endif() + + if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() endif() + if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" + ) + endif() + endmacro() From 9e89f2a9069662c8c19262e4fe3c0685937ebb01 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 25 Sep 2023 21:22:17 -0400 Subject: [PATCH 247/333] file changes not being registered --- cmake/HPX_SetupGASNET.cmake | 48 ------------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 cmake/HPX_SetupGASNET.cmake diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake deleted file mode 100644 index 37957d79ba46..000000000000 --- a/cmake/HPX_SetupGASNET.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019-2022 Ste||ar Group -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -include(HPX_Message) -include(FindGasnet) - -macro(hpx_setup_gasnet) - - if(NOT GASNET_FOUND) - - find_gasnet() - - set(GASNET_CXXFLAGS) - set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) - set(GASNET_LDFLAGS) - - foreach(TOKEN ${GASNET_CFLAGS}) - list(APPEND GASNET_CXXFLAGS ${TOKEN}) - endforeach() - - foreach(TOKEN ${GASNET_LDFLAGS_TMP}) - list(APPEND GASNET_LDFLAGS ${TOKEN}) - endforeach() - - add_library(gasnet INTERFACE IMPORTED) - target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) - target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) - target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) - - if(GASNET_MPI_FOUND) - # Setup PMI imported target - find_package(PMI) - if(PMI_FOUND) - hpx_add_config_define_namespace( - DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET - ) - add_library(Pmi::pmi INTERFACE IMPORTED) - target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) - target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) - endif() - endif() - - endif() -endmacro() From ac74ff075964d54a2669ad496a52fe9199d9664a Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 28 Sep 2023 11:40:54 +0200 Subject: [PATCH 248/333] Fix header file links --- docs/sphinx/api/public_api.rst | 54 +++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/sphinx/api/public_api.rst b/docs/sphinx/api/public_api.rst index c81e8e6cd581..4138a0bf1bb3 100644 --- a/docs/sphinx/api/public_api.rst +++ b/docs/sphinx/api/public_api.rst @@ -23,7 +23,7 @@ sub-namespaces will eventually be removed. ``hpx/algorithm.hpp`` ===================== -The header :hpx-header:`libs/full/include,include/hpx/algorithm.hpp` corresponds to the +The header :hpx-header:`libs/full/include/include,hpx/algorithm.hpp` corresponds to the C++ standard library header :cppreference-header:`algorithm`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -191,7 +191,7 @@ Functions ``hpx/any.hpp`` =============== -The header :hpx-header:`libs/core/include_local,include/hpx/any.hpp` corresponds to the C++ +The header :hpx-header:`libs/core/include_local/include,hpx/any.hpp` corresponds to the C++ standard library header :cppreference-header:`any`. :cpp:type:`hpx::any` is compatible with ``std::any``. @@ -253,7 +253,7 @@ Macros ``hpx/barrier.hpp`` =================== -The header :hpx-header:`libs/full/include,include/hpx/barrier.hpp` corresponds to the +The header :hpx-header:`libs/full/include/include,hpx/barrier.hpp` corresponds to the C++ standard library header :cppreference-header:`barrier` and contains a distributed barrier implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -282,7 +282,7 @@ Classes ``hpx/channel.hpp`` =================== -The header :hpx-header:`libs/full/include,include/hpx/channel.hpp` contains a local and a +The header :hpx-header:`libs/full/include/include,hpx/channel.hpp` contains a local and a distributed channel implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -310,7 +310,7 @@ Classes ``hpx/chrono.hpp`` ================== -The header :hpx-header:`libs/core/include_local,include/hpx/chrono.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/chrono.hpp` corresponds to the C++ standard library header :cppreference-header:`chrono`. The following replacements and extensions are provided compared to :cppreference-header:`chrono`. @@ -332,7 +332,7 @@ Classes ``hpx/condition_variable.hpp`` ============================== -The header :hpx-header:`libs/core/include_local,include/hpx/condition_variable.hpp` corresponds to the C++ +The header :hpx-header:`libs/core/include_local/include,hpx/condition_variable.hpp` corresponds to the C++ standard library header :cppreference-header:`condition_variable`. Classes @@ -353,7 +353,7 @@ Classes ``hpx/exception.hpp`` ===================== -The header :hpx-header:`libs/core/include_local,include/hpx/exception.hpp` corresponds to +The header :hpx-header:`libs/core/include_local/include,hpx/exception.hpp` corresponds to the C++ standard library header :cppreference-header:`exception`. :cpp:class:`hpx::exception` extends ``std::exception`` and is the base class for all exceptions thrown in |hpx|. :c:macro:`HPX_THROW_EXCEPTION` can be used to throw |hpx| exceptions with file and line information @@ -380,7 +380,7 @@ Classes ``hpx/execution.hpp`` ===================== -The header :hpx-header:`libs/core/include_local,include/hpx/execution.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/execution.hpp` corresponds to the C++ standard library header :cppreference-header:`execution`. See :ref:`parallel`, :ref:`parallel_algorithms` and :ref:`executor_parameters` for more information about execution policies and executor parameters. @@ -430,7 +430,7 @@ Classes ``hpx/functional.hpp`` ====================== -The header :hpx-header:`libs/core/include_local,include/hpx/functional.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/functional.hpp` corresponds to the C++ standard library header :cppreference-header:`functional`. :cpp:class:`hpx::function` is a more efficient and serializable replacement for ``std::function``. @@ -493,7 +493,7 @@ Functions ``hpx/future.hpp`` ================== -The header :hpx-header:`libs/full/include,include/hpx/future.hpp` corresponds to the +The header :hpx-header:`libs/full/include/include,hpx/future.hpp` corresponds to the C++ standard library header :cppreference-header:`future`. See :ref:`extend_futures` for more information about extensions to futures compared to the C++ standard library. @@ -610,7 +610,7 @@ Functions ``hpx/latch.hpp`` ================= -The header :hpx-header:`libs/full/include,include/hpx/latch.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include/include,hpx/latch.hpp` corresponds to the C++ standard library header :cppreference-header:`latch`. It contains a local and a distributed latch implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -639,7 +639,7 @@ Classes ``hpx/mutex.hpp`` ================= -The header :hpx-header:`libs/core/include_local,include/hpx/mutex.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/mutex.hpp` corresponds to the C++ standard library header :cppreference-header:`mutex`. Classes @@ -675,7 +675,7 @@ Functions ``hpx/memory.hpp`` ================== -The header :hpx-header:`libs/core/include_local,include/hpx/memory.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/memory.hpp` corresponds to the C++ standard library header :cppreference-header:`memory`. It contains parallel versions of the copy, fill, move, and construct helper functions in :cppreference-header:`memory`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -722,7 +722,7 @@ Functions ``hpx/numeric.hpp`` =================== -The header :hpx-header:`libs/core/include_local,include/hpx/numeric.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/numeric.hpp` corresponds to the C++ standard library header :cppreference-header:`numeric`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -762,7 +762,7 @@ Functions ``hpx/optional.hpp`` ==================== -The header :hpx-header:`libs/core/include_local,include/hpx/optional.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/optional.hpp` corresponds to the C++ standard library header :cppreference-header:`optional`. :cpp:type:`hpx::optional` is compatible with ``std::optional``. @@ -789,7 +789,7 @@ Classes ``hpx/runtime.hpp`` =================== -The header :hpx-header:`libs/full/include,include/hpx/runtime.hpp` contains functions for accessing +The header :hpx-header:`libs/full/include/include,hpx/runtime.hpp` contains functions for accessing local and distributed runtime information. Typedefs @@ -849,7 +849,7 @@ Functions ``hpx/source_location.hpp`` =========================== -The header :hpx-header:`libs/core/include_local,include/hpx/source_location.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/source_location.hpp` corresponds to the C++ standard library header :cppreference-header:`source_location`. Classes @@ -868,7 +868,7 @@ Classes ``hpx/system_error.hpp`` ======================== -The header :hpx-header:`libs/core/include_local,include/hpx/system_error.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/system_error.hpp` corresponds to the C++ standard library header :cppreference-header:`system_error`. Classes @@ -887,7 +887,7 @@ Classes ``hpx/task_block.hpp`` ====================== -The header :hpx-header:`libs/core/include_local,include/hpx/task_block.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/task_block.hpp` corresponds to the ``task_block`` feature in |cpp17_n4755|_. See :ref:`using_task_block` for more details on using task blocks. @@ -922,7 +922,7 @@ Functions ``hpx/experimental/task_group.hpp`` =================================== -The header :hpx-header:`libs/core/include_local,include/hpx/experimental/task_group.hpp` +The header :hpx-header:`libs/core/include_local/include,hpx/experimental/task_group.hpp` corresponds to the ``task_group`` feature in |oneTBB|_. Classes @@ -941,7 +941,7 @@ Classes ``hpx/thread.hpp`` ================== -The header :hpx-header:`libs/core/include_local,include/hpx/thread.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/thread.hpp` corresponds to the C++ standard library header :cppreference-header:`thread`. The functionality in this header is equivalent to the standard library thread functionality, with the exception that the |hpx| equivalents are implemented on top of lightweight threads and the |hpx| runtime. @@ -977,7 +977,7 @@ Functions ``hpx/semaphore.hpp`` ===================== -The header :hpx-header:`libs/core/include_local,include/hpx/semaphore.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/semaphore.hpp` corresponds to the C++ standard library header :cppreference-header:`semaphore`. Classes @@ -997,7 +997,7 @@ Classes ``hpx/shared_mutex.hpp`` ======================== -The header :hpx-header:`libs/core/include_local,include/hpx/shared_mutex.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/shared_mutex.hpp` corresponds to the C++ standard library header :cppreference-header:`shared_mutex`. Classes @@ -1016,7 +1016,7 @@ Classes ``hpx/stop_token.hpp`` ====================== -The header :hpx-header:`libs/core/include_local,include/hpx/stop_token.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/stop_token.hpp` corresponds to the C++ standard library header :cppreference-header:`stop_token`. Constants @@ -1049,7 +1049,7 @@ Classes ``hpx/tuple.hpp`` ================= -The header :hpx-header:`libs/core/include_local,include/hpx/tuple.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/tuple.hpp` corresponds to the C++ standard library header :cppreference-header:`tuple`. :cpp:class:`hpx::tuple` can be used in CUDA device code, unlike ``std::tuple``. @@ -1097,7 +1097,7 @@ Functions ``hpx/type_traits.hpp`` ======================= -The header :hpx-header:`libs/core/include_local,include/hpx/type_traits.hpp` corresponds to the +The header :hpx-header:`libs/core/include_local/include,hpx/type_traits.hpp` corresponds to the C++ standard library header :cppreference-header:`type_traits`. Classes @@ -1117,7 +1117,7 @@ Classes ``hpx/unwrap.hpp`` ================== -The header :hpx-header:`libs/fcore/include_local,include/hpx/unwrap.hpp` contains utilities for +The header :hpx-header:`libs/fcore/include_local/include,hpx/unwrap.hpp` contains utilities for unwrapping futures. Classes From e016a2317714161591854f0f740ab476812cdab1 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 28 Sep 2023 13:12:03 +0200 Subject: [PATCH 249/333] Fix broken links docs --- cmake/templates/conf.py.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/templates/conf.py.in b/cmake/templates/conf.py.in index 30a355d720c1..58824243c8c5 100644 --- a/cmake/templates/conf.py.in +++ b/cmake/templates/conf.py.in @@ -275,11 +275,11 @@ rst_prolog += ''' :target: https://joss.theoj.org/papers/022e5917b95517dff20cd3742ab95eca :alt: JOSS Paper about HPX .. |hpx_dependency_report| replace:: |hpx| Dependency report -.. _hpx_dependency_report: ../../report/index.html +.. _hpx_dependency_report: ../report/index.html .. |hpx_single_html| replace:: |hpx| Single HTML documentation -.. _hpx_single_html: ../../singlehtml/index.h +.. _hpx_single_html: ../singlehtml/index.h .. |hpx_pdf| replace:: |hpx| PDF documentation -.. _hpx_pdf: ../../pdf/HPX.pdf +.. _hpx_pdf: ../pdf/HPX.pdf .. |hpxpi| replace:: HPXPI .. _hpxpi: https://github.com/STEllAR-GROUP/hpxpi/ From 35063d2d340548bca49aa51983bd03f60ec80b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kn=C3=BCpfer?= Date: Wed, 27 Sep 2023 08:04:49 +0200 Subject: [PATCH 250/333] Fixed matrix multiplication example output --- docs/sphinx/examples/matrix_multiplication.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/examples/matrix_multiplication.rst b/docs/sphinx/examples/matrix_multiplication.rst index 995e1ebfef3f..3039bf1d5cda 100644 --- a/docs/sphinx/examples/matrix_multiplication.rst +++ b/docs/sphinx/examples/matrix_multiplication.rst @@ -26,7 +26,7 @@ This program will perform a matrix multiplication in parallel. The output will l Resultant Matrix is : 124 93 - 111 127 + 130 82 Setup ===== @@ -73,7 +73,7 @@ This should print: Resultant Matrix is : 124 93 - 111 127 + 130 82 Notice that the numbers may be different because of the random initialization of the matrices. From 43de88c594df655c116d461d2b6c16faddcd8f05 Mon Sep 17 00:00:00 2001 From: Panos Syskakis <75637426+Pansysk75@users.noreply.github.com> Date: Wed, 27 Sep 2023 17:59:51 +0300 Subject: [PATCH 251/333] Fix CMake find_library for HWLOC --- cmake/FindHwloc.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindHwloc.cmake b/cmake/FindHwloc.cmake index 7760048c948f..42d331e65f70 100644 --- a/cmake/FindHwloc.cmake +++ b/cmake/FindHwloc.cmake @@ -35,7 +35,7 @@ if(NOT TARGET Hwloc::hwloc) find_library( Hwloc_LIBRARY - NAMES libhwloc.so hwloc + NAMES libhwloc.so libhwloc.lib hwloc HINTS ${Hwloc_ROOT} ENV HWLOC_ROOT From b8c272ce8db90691d3582b05f8c79dfcf82c2c04 Mon Sep 17 00:00:00 2001 From: srinivasyadav18 Date: Fri, 1 Sep 2023 13:51:13 -0500 Subject: [PATCH 252/333] Fix HPX-APEX cmake integration Existing apex cmake integration does not work for external Apex installation. This fix seperates the git fetch installation using HPX_WITH_FETCH_APEX and external installation using Apex_ROOT, cmake variables respectively. --- CMakeLists.txt | 9 +++++++++ cmake/HPX_SetupApex.cmake | 27 ++++++++++++++++----------- cmake/templates/HPXConfig.cmake.in | 1 + 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fc96518982e..34faaa5d7743 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -937,6 +937,15 @@ hpx_option( CATEGORY "Profiling" ) +hpx_option( + HPX_WITH_FETCH_APEX + BOOL + "Use FetchContent to fetch APEX. By default an installed APEX will be used. (default: OFF)" + OFF + CATEGORY "Build Targets" + ADVANCED +) + if(HPX_WITH_APEX) hpx_add_config_define(HPX_HAVE_APEX) # tell HPX that we use APEX hpx_option( diff --git a/cmake/HPX_SetupApex.cmake b/cmake/HPX_SetupApex.cmake index 01d9b96a844c..3acb6d9ba02e 100644 --- a/cmake/HPX_SetupApex.cmake +++ b/cmake/HPX_SetupApex.cmake @@ -46,11 +46,7 @@ if(HPX_WITH_APEX AND NOT TARGET APEX::apex) # We want to track parent dependencies hpx_add_config_define(HPX_HAVE_THREAD_PARENT_REFERENCE) - if(Apex_ROOT) - # Use given (external) APEX - set(HPX_APEX_ROOT ${Apex_ROOT}) - - else() + if (HPX_WITH_FETCH_APEX) # If Apex_ROOT not specified, local clone into hpx source dir include(FetchContent) fetchcontent_declare( @@ -76,10 +72,10 @@ if(HPX_WITH_APEX AND NOT TARGET APEX::apex) set(Apex_ROOT ${apex_SOURCE_DIR}) hpx_info("Apex_ROOT is not set. Cloning APEX into ${apex_SOURCE_DIR}.") + list(APPEND CMAKE_MODULE_PATH "${Apex_ROOT}/cmake/Modules") + add_subdirectory(${Apex_ROOT}/src/apex ${CMAKE_BINARY_DIR}/apex/src/apex) endif() - list(APPEND CMAKE_MODULE_PATH "${Apex_ROOT}/cmake/Modules") - add_subdirectory(${Apex_ROOT}/src/apex ${CMAKE_BINARY_DIR}/apex/src/apex) if(Amplifier_FOUND) hpx_error("Amplifier_FOUND has been set. Please disable the use of the \ Intel Amplifier (WITH_AMPLIFIER=Off) in order to use APEX" @@ -87,12 +83,21 @@ if(HPX_WITH_APEX AND NOT TARGET APEX::apex) endif() endif() - add_library(APEX::apex INTERFACE IMPORTED) - if(HPX_FIND_PACKAGE) - target_link_libraries(APEX::apex INTERFACE HPX::apex) + if(HPX_WITH_FETCH_APEX) + add_library(APEX::apex INTERFACE IMPORTED) + if(HPX_FIND_PACKAGE) + target_link_libraries(APEX::apex INTERFACE HPX::apex) + else() + target_link_libraries(APEX::apex INTERFACE apex) + endif() else() - target_link_libraries(APEX::apex INTERFACE apex) + if(Apex_ROOT) + find_package(APEX REQUIRED PATHS ${Apex_ROOT}) + else() + hpx_error("Apex_ROOT not set.") + endif() endif() + if((UNIX AND NOT APPLE) OR MINGW) target_link_options(APEX::apex INTERFACE "-Wl,-no-as-needed") endif() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index 81d11112f5e7..c3f0ae31b8f7 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -179,6 +179,7 @@ set(APEX_WITH_ACTIVEHARMONY "@APEX_WITH_ACTIVEHARMONY@") set(Activeharmony_ROOT "@Activeharmony_ROOT@") set(APEX_WITH_OTF2 "@APEX_WITH_OTF2@") set(Otf2_ROOT "@Otf2_ROOT@") +set(Apex_ROOT "@Apex_ROOT@") include(HPX_SetupApex) # ############################################################################## From ae9bf00549233381c8c0b7caa9d715a1dada42f8 Mon Sep 17 00:00:00 2001 From: Srinivas Yadav <43375352+srinivasyadav18@users.noreply.github.com> Date: Fri, 15 Sep 2023 19:35:16 -0500 Subject: [PATCH 253/333] Add -DHPX_WITH_FETCH_APEX=ON to CI builder scripts Add the newly introduced cmake flag -DHPX_WITH_FETCH_APEX=ON, which fetches APEX using cmake_fetch_content. --- .github/workflows/linux_release_fetchcontent.yml | 1 + .jenkins/cscs/env-clang-apex.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/linux_release_fetchcontent.yml b/.github/workflows/linux_release_fetchcontent.yml index be9ad924ae56..356db6d5df2d 100644 --- a/.github/workflows/linux_release_fetchcontent.yml +++ b/.github/workflows/linux_release_fetchcontent.yml @@ -26,6 +26,7 @@ jobs: -DHPX_WITH_MALLOC=system \ -DHPX_WITH_FETCH_ASIO=ON \ -DHPX_WITH_APEX=ON \ + -DHPX_WITH_FETCH_APEX=ON \ -DHPX_WITH_EXAMPLES=ON \ -DHPX_WITH_TESTS=OFF \ -DHPX_REPOSITORY="file:////$(pwd)" \ diff --git a/.jenkins/cscs/env-clang-apex.sh b/.jenkins/cscs/env-clang-apex.sh index c5767649ae62..207d401622c2 100644 --- a/.jenkins/cscs/env-clang-apex.sh +++ b/.jenkins/cscs/env-clang-apex.sh @@ -34,6 +34,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_SPINLOCK_DEADLOCK_DETECTION=ON" #configure_extra_options+=" -DHPX_WITH_LIBCDS=ON" configure_extra_options+=" -DHPX_WITH_APEX=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_APEX=ON" configure_extra_options+=" -DHPX_WITH_PAPI=ON" configure_extra_options+=" -DAPEX_WITH_PAPI=ON" configure_extra_options+=" -DAPEX_WITH_OTF2=ON" From 6bca2504f15869126d691f81c3b9c1adbdc7fb99 Mon Sep 17 00:00:00 2001 From: Srinivas Yadav <43375352+srinivasyadav18@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:30:48 -0500 Subject: [PATCH 254/333] Fix formatting issues and add missing define Fixes cmake formatting issues. Adds missing -DHPX_WITH_FETCH_APEX=ON --- .circleci/config.yml | 1 + cmake/HPX_SetupApex.cmake | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb498cbc59b3..7e8adf3404fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -293,6 +293,7 @@ jobs: -DHPX_WITH_MALLOC=system \ -DHPX_WITH_FETCH_ASIO=Off \ -DHPX_WITH_APEX=On \ + -DHPX_WITH_FETCH_APEX=ON \ -DHPX_WITH_TESTS=On \ -DHPX_WITH_EXAMPLES=On rm -rf * diff --git a/cmake/HPX_SetupApex.cmake b/cmake/HPX_SetupApex.cmake index 3acb6d9ba02e..2b3d071e50c5 100644 --- a/cmake/HPX_SetupApex.cmake +++ b/cmake/HPX_SetupApex.cmake @@ -46,7 +46,7 @@ if(HPX_WITH_APEX AND NOT TARGET APEX::apex) # We want to track parent dependencies hpx_add_config_define(HPX_HAVE_THREAD_PARENT_REFERENCE) - if (HPX_WITH_FETCH_APEX) + if(HPX_WITH_FETCH_APEX) # If Apex_ROOT not specified, local clone into hpx source dir include(FetchContent) fetchcontent_declare( From c454a062c01a7c8881d4a098d1ffcb6c813d6ec0 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sun, 30 Apr 2023 16:27:02 -0400 Subject: [PATCH 255/333] initial import of gasnet parcelport --- cmake/HPX_SetupGASNET.cmake | 48 ++ cmake/HPX_SetupGasnet.cmake | 763 +----------------- libs/core/CMakeLists.txt | 1 + .../gasnet_base/src/gasnet_environment.cpp | 1 - libs/full/CMakeLists.txt | 1 + .../hpx/parcelport_gasnet/receiver.hpp | 1 + .../include/hpx/parcelport_gasnet/sender.hpp | 2 + .../parcelport_gasnet/tests/CMakeLists.txt | 25 + 8 files changed, 108 insertions(+), 734 deletions(-) create mode 100644 cmake/HPX_SetupGASNET.cmake diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake new file mode 100644 index 000000000000..37957d79ba46 --- /dev/null +++ b/cmake/HPX_SetupGASNET.cmake @@ -0,0 +1,48 @@ +# Copyright (c) 2019-2022 Ste||ar Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(FindGasnet) + +macro(hpx_setup_gasnet) + + if(NOT GASNET_FOUND) + + find_gasnet() + + set(GASNET_CXXFLAGS) + set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_LDFLAGS) + + foreach(TOKEN ${GASNET_CFLAGS}) + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endforeach() + + foreach(TOKEN ${GASNET_LDFLAGS_TMP}) + list(APPEND GASNET_LDFLAGS ${TOKEN}) + endforeach() + + add_library(gasnet INTERFACE IMPORTED) + target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) + target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) + target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) + + if(GASNET_MPI_FOUND) + # Setup PMI imported target + find_package(PMI) + if(PMI_FOUND) + hpx_add_config_define_namespace( + DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET + ) + add_library(Pmi::pmi INTERFACE IMPORTED) + target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) + target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) + endif() + endif() + + endif() +endmacro() diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index a648bd4a5f71..37957d79ba46 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -1,751 +1,48 @@ -# Copyright (c) 2017 Thomas Heller -# Copyright (c) 2023 Christopher Taylor +# Copyright (c) 2019-2022 Ste||ar Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# - -macro(hpx_setup_gasnet) - - if(NOT TARGET PkgConfig::GASNET) - - find_package(PkgConfig REQUIRED QUIET COMPONENTS) - - hpx_info( - "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" - ) - pkg_search_module( - GASNET IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - - if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) - - if(NOT CMAKE_C_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") - endif() - if(NOT CMAKE_CXX_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") - endif() - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") - endif() - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") - endif() - - include(FetchContent) - FetchContent_Declare(gasnet - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz - ) - - fetchcontent_getproperties(gasnet) - if(NOT gasnet) - FetchContent_Populate(gasnet) - endif() - - message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") - - set(GASNET_DIR "${gasnet_SOURCE_DIR}") - set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") - set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") - - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - endif() - - if(GASNET_BUILD_STATUS) - message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") - endif() - - pkg_search_module( - GASNET REQUIRED IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) - message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") - endif() - - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") - set(GASNET_MPI_FOUND TRUE) - include(HPX_SetupMPI) - hpx_setup_mpi() - endif() - - if(GASNET_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - if(GASNET_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") +include(HPX_Message) +include(FindGasnet) - foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - - endif() - - if(GASNET_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - - if(GASNET_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") +macro(hpx_setup_gasnet) - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() + if(NOT GASNET_FOUND) - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() + find_gasnet() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") - endforeach() + set(GASNET_CXXFLAGS) + set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_LDFLAGS) - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) + foreach(TOKEN ${GASNET_CFLAGS}) + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endforeach() - list(LENGTH FOUND_LIB IDX) + foreach(TOKEN ${GASNET_LDFLAGS_TMP}) + list(APPEND GASNET_LDFLAGS ${TOKEN}) + endforeach() - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) + add_library(gasnet INTERFACE IMPORTED) + target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) + target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) + target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + if(GASNET_MPI_FOUND) + # Setup PMI imported target + find_package(PMI) + if(PMI_FOUND) + hpx_add_config_define_namespace( + DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() + add_library(Pmi::pmi INTERFACE IMPORTED) + target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) + target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) + endif() endif() - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES - "${GASNET_LIBRARY_DIRS}" - ) - endif() - endmacro() diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 3e6550e93f25..9b9666358b20 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,6 +56,7 @@ set(_hpx_core_modules logging memory mpi_base + gasnet_base pack_traversal plugin prefix diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 520ea6820fae..869fad458dfe 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -215,7 +215,6 @@ static inline void wait_done_obj(done_t* done, bool do_yield) /////////////////////////////////////////////////////////////////////////////// namespace hpx::util { - namespace detail { bool detect_gasnet_environment( diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index e4a7eb24b923..b6861762f349 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,6 +35,7 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi + parcelport_gasnet parcelport_tcp parcelports parcelset diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp index 99911618c058..5d3245bc2b95 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver.hpp @@ -161,6 +161,7 @@ namespace hpx::parcelset::policies::gasnet { connection_list connections_; exp_backoff bo; }; + } // namespace hpx::parcelset::policies::gasnet #endif diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index e56d4b6a7aa7..40e04bbe207e 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -104,6 +104,7 @@ namespace hpx::parcelset::policies::gasnet { has_work = true; } next_free_tag(); + return has_work; } @@ -159,6 +160,7 @@ namespace hpx::parcelset::policies::gasnet { int next_free_tag_request_; int next_free_tag_; }; + } // namespace hpx::parcelset::policies::gasnet #endif diff --git a/libs/full/parcelport_gasnet/tests/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/CMakeLists.txt index 3f1060f293c3..1bdc527c54a2 100644 --- a/libs/full/parcelport_gasnet/tests/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/tests/CMakeLists.txt @@ -8,35 +8,60 @@ include(HPX_Message) if(HPX_WITH_TESTS) if(HPX_WITH_TESTS_UNIT) +<<<<<<< HEAD add_hpx_pseudo_target(tests.unit.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( tests.unit.modules tests.unit.modules.parcelport_gasnet +======= + add_hpx_pseudo_target(tests.unit.modules.parcelport_lci) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.parcelport_lci +>>>>>>> 471497b530 (initial import of gasnet parcelport) ) add_subdirectory(unit) endif() if(HPX_WITH_TESTS_REGRESSIONS) +<<<<<<< HEAD add_hpx_pseudo_target(tests.regressions.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( tests.regressions.modules tests.regressions.modules.parcelport_gasnet +======= + add_hpx_pseudo_target(tests.regressions.modules.parcelport_lci) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.parcelport_lci +>>>>>>> 471497b530 (initial import of gasnet parcelport) ) add_subdirectory(regressions) endif() if(HPX_WITH_TESTS_BENCHMARKS) +<<<<<<< HEAD add_hpx_pseudo_target(tests.performance.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( tests.performance.modules tests.performance.modules.parcelport_gasnet +======= + add_hpx_pseudo_target(tests.performance.modules.parcelport_lci) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.parcelport_lci +>>>>>>> 471497b530 (initial import of gasnet parcelport) ) add_subdirectory(performance) endif() if(HPX_WITH_TESTS_HEADERS) add_hpx_header_tests( +<<<<<<< HEAD modules.parcelport_gasnet HEADERS ${parcelport_gasnet_headers} HEADER_ROOT ${PROJECT_SOURCE_DIR}/include DEPENDENCIES hpx_parcelport_gasnet +======= + modules.parcelport_lci + HEADERS ${parcelport_lci_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + DEPENDENCIES hpx_parcelport_lci +>>>>>>> 471497b530 (initial import of gasnet parcelport) ) endif() endif() From b259cb324f1e45ee5f07d8e31233bdaaf1184917 Mon Sep 17 00:00:00 2001 From: ctaylor Date: Sat, 24 Jun 2023 22:39:57 -0400 Subject: [PATCH 256/333] clean compile --- libs/core/CMakeLists.txt | 1 - libs/full/CMakeLists.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 9b9666358b20..3e6550e93f25 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,7 +56,6 @@ set(_hpx_core_modules logging memory mpi_base - gasnet_base pack_traversal plugin prefix diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index b6861762f349..e4a7eb24b923 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,7 +35,6 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi - parcelport_gasnet parcelport_tcp parcelports parcelset From 50fb54136b9d52e7ea20d3e4f3aeb97a8eb5b811 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 22:06:49 -0400 Subject: [PATCH 257/333] Adding gasnet parcelport to CI builders - flyby: fixing various inspect, clang-format, etc. errors --- cmake/HPX_SetupGasnet.cmake | 527 +++++++++++++++++- libs/core/gasnet_base/README.rst | 16 - libs/core/gasnet_base/docs/index.rst | 12 +- .../gasnet_base/src/gasnet_environment.cpp | 1 + libs/full/parcelport_gasnet/README.rst | 16 - libs/full/parcelport_gasnet/docs/index.rst | 4 +- 6 files changed, 533 insertions(+), 43 deletions(-) delete mode 100644 libs/core/gasnet_base/README.rst delete mode 100644 libs/full/parcelport_gasnet/README.rst diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 37957d79ba46..a0ac69b59fcd 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -13,9 +13,13 @@ macro(hpx_setup_gasnet) find_gasnet() - set(GASNET_CXXFLAGS) - set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) - set(GASNET_LDFLAGS) + hpx_info( + "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" + ) + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) foreach(TOKEN ${GASNET_CFLAGS}) list(APPEND GASNET_CXXFLAGS ${TOKEN}) @@ -44,5 +48,522 @@ macro(hpx_setup_gasnet) endif() endif() + if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + + endif() + + if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + endif() + endif() + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" + ) + endif() endmacro() diff --git a/libs/core/gasnet_base/README.rst b/libs/core/gasnet_base/README.rst deleted file mode 100644 index 1ae4ac39ce3b..000000000000 --- a/libs/core/gasnet_base/README.rst +++ /dev/null @@ -1,16 +0,0 @@ - -.. - Copyright (c) 2020 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -======== -mpi_base -======== - -This library is part of HPX. - -Documentation can be found `here -`__. diff --git a/libs/core/gasnet_base/docs/index.rst b/libs/core/gasnet_base/docs/index.rst index 43dbdb7c2a03..cab19901c949 100644 --- a/libs/core/gasnet_base/docs/index.rst +++ b/libs/core/gasnet_base/docs/index.rst @@ -5,14 +5,14 @@ Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -.. _modules_mpi_base: +.. _modules_gasnet_base: -======== -mpi_base -======== +=========== +gasnet_base +=========== -This module provides helper functionality for detecting MPI environments. +This module provides helper functionality for detecting GASNET environments. -See the :ref:`API reference ` of this module for more +See the :ref:`API reference ` of this module for more details. diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 869fad458dfe..520ea6820fae 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -215,6 +215,7 @@ static inline void wait_done_obj(done_t* done, bool do_yield) /////////////////////////////////////////////////////////////////////////////// namespace hpx::util { + namespace detail { bool detect_gasnet_environment( diff --git a/libs/full/parcelport_gasnet/README.rst b/libs/full/parcelport_gasnet/README.rst deleted file mode 100644 index c0729a670a32..000000000000 --- a/libs/full/parcelport_gasnet/README.rst +++ /dev/null @@ -1,16 +0,0 @@ - -.. - Copyright (c) 2020-2021 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -================= -parcelport_gasnet -================= - -This module is part of HPX. - -Documentation can be found `here -`__. diff --git a/libs/full/parcelport_gasnet/docs/index.rst b/libs/full/parcelport_gasnet/docs/index.rst index d61191bdd7d7..e26471c5e5e6 100644 --- a/libs/full/parcelport_gasnet/docs/index.rst +++ b/libs/full/parcelport_gasnet/docs/index.rst @@ -7,9 +7,9 @@ .. _modules_parcelport_gasnet: -============== +================= parcelport_gasnet -============== +================= TODO: High-level description of the module. From 75fb15f096cb4b3ec753f932572190272292eb39 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 22:11:24 -0400 Subject: [PATCH 258/333] Fixing HPX compilation errors --- cmake/HPX_SetupGasnet.cmake | 569 ------------------------------------ 1 file changed, 569 deletions(-) delete mode 100644 cmake/HPX_SetupGasnet.cmake diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake deleted file mode 100644 index a0ac69b59fcd..000000000000 --- a/cmake/HPX_SetupGasnet.cmake +++ /dev/null @@ -1,569 +0,0 @@ -# Copyright (c) 2019-2022 Ste||ar Group -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -include(HPX_Message) -include(FindGasnet) - -macro(hpx_setup_gasnet) - - if(NOT GASNET_FOUND) - - find_gasnet() - - hpx_info( - "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" - ) - pkg_search_module( - GASNET REQUIRED IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - - foreach(TOKEN ${GASNET_CFLAGS}) - list(APPEND GASNET_CXXFLAGS ${TOKEN}) - endforeach() - - foreach(TOKEN ${GASNET_LDFLAGS_TMP}) - list(APPEND GASNET_LDFLAGS ${TOKEN}) - endforeach() - - add_library(gasnet INTERFACE IMPORTED) - target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) - target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) - target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) - - if(GASNET_MPI_FOUND) - # Setup PMI imported target - find_package(PMI) - if(PMI_FOUND) - hpx_add_config_define_namespace( - DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET - ) - add_library(Pmi::pmi INTERFACE IMPORTED) - target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) - target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) - endif() - endif() - - if(GASNET_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - - if(GASNET_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") - endif() - endif() - - endif() - - if(GASNET_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - - if(GASNET_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - endif() - endif() - - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES - "${GASNET_LIBRARY_DIRS}" - ) - - endif() -endmacro() From 751ec467eae44e504db327bd75abc1f67085f561 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 14 Jul 2023 06:46:48 -0500 Subject: [PATCH 259/333] Fixing HPX compilation errors --- cmake/HPX_SetupGASNET.cmake | 48 ------------------------------------- 1 file changed, 48 deletions(-) delete mode 100644 cmake/HPX_SetupGASNET.cmake diff --git a/cmake/HPX_SetupGASNET.cmake b/cmake/HPX_SetupGASNET.cmake deleted file mode 100644 index 37957d79ba46..000000000000 --- a/cmake/HPX_SetupGASNET.cmake +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019-2022 Ste||ar Group -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -include(HPX_Message) -include(FindGasnet) - -macro(hpx_setup_gasnet) - - if(NOT GASNET_FOUND) - - find_gasnet() - - set(GASNET_CXXFLAGS) - set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) - set(GASNET_LDFLAGS) - - foreach(TOKEN ${GASNET_CFLAGS}) - list(APPEND GASNET_CXXFLAGS ${TOKEN}) - endforeach() - - foreach(TOKEN ${GASNET_LDFLAGS_TMP}) - list(APPEND GASNET_LDFLAGS ${TOKEN}) - endforeach() - - add_library(gasnet INTERFACE IMPORTED) - target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) - target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) - target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) - - if(GASNET_MPI_FOUND) - # Setup PMI imported target - find_package(PMI) - if(PMI_FOUND) - hpx_add_config_define_namespace( - DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET - ) - add_library(Pmi::pmi INTERFACE IMPORTED) - target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) - target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) - endif() - endif() - - endif() -endmacro() From db4696df7f7e7ca517e48b93f5f8c5ae11b507f4 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 25 Jul 2021 13:10:47 -0500 Subject: [PATCH 260/333] Futures attempt to execute threads directly if those have not started executing - adding new API function hpx::threads::get_outer_self_id - flyby: fixing initialization of resource partitioner --- libs/core/execution/include/hpx/execution/detail/future_exec.hpp | 1 + libs/core/futures/src/future_data.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp index c49002325c64..c980b00b76e4 100644 --- a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp +++ b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp @@ -178,6 +178,7 @@ namespace hpx::lcos::detail { { id = threads::invalid_thread_id; hpx::parallel::execution::post(exec, HPX_FORWARD(F, f)); + return threads::invalid_thread_id; } }; diff --git a/libs/core/futures/src/future_data.cpp b/libs/core/futures/src/future_data.cpp index 38f0b92a2712..9de3d277080e 100644 --- a/libs/core/futures/src/future_data.cpp +++ b/libs/core/futures/src/future_data.cpp @@ -130,6 +130,7 @@ namespace hpx::lcos::detail { } // this thread would block on the future + threads::thread_id_ref_type runs_child = runs_child_; auto* thrd = get_thread_id_data(runs_child); HPX_UNUSED(thrd); // might be unused From c62fc1a02b1c177b68ec21851b23432b08b38ddb Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Tue, 11 Jul 2023 13:22:10 -0500 Subject: [PATCH 261/333] Working around the need to use a lock during thread creation --- libs/core/execution/include/hpx/execution/detail/future_exec.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp index c980b00b76e4..c49002325c64 100644 --- a/libs/core/execution/include/hpx/execution/detail/future_exec.hpp +++ b/libs/core/execution/include/hpx/execution/detail/future_exec.hpp @@ -178,7 +178,6 @@ namespace hpx::lcos::detail { { id = threads::invalid_thread_id; hpx::parallel::execution::post(exec, HPX_FORWARD(F, f)); - return threads::invalid_thread_id; } }; From 027b1a39917bc3ee2bff85749c07b4c449ab95c6 Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Thu, 20 Jul 2023 08:51:33 -0500 Subject: [PATCH 262/333] Fix race condition --- libs/core/futures/src/future_data.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/futures/src/future_data.cpp b/libs/core/futures/src/future_data.cpp index 9de3d277080e..38f0b92a2712 100644 --- a/libs/core/futures/src/future_data.cpp +++ b/libs/core/futures/src/future_data.cpp @@ -130,7 +130,6 @@ namespace hpx::lcos::detail { } // this thread would block on the future - threads::thread_id_ref_type runs_child = runs_child_; auto* thrd = get_thread_id_data(runs_child); HPX_UNUSED(thrd); // might be unused From 9393b567bcdf667a19c166c1fbd6c0e734ccd13a Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sat, 3 Jun 2023 06:21:25 -0500 Subject: [PATCH 263/333] move as buffers when data type is trivially copyable Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 8fb58d086586..a6eafc67fba1 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -7,6 +7,7 @@ /// \file parallel/algorithms/uninitialized_copy.hpp #pragma once +#include #if defined(DOXYGEN) namespace hpx { From 61e7bbc0fbbea4612a1a6b60065ee8781ef3b8d9 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 13:53:22 -0500 Subject: [PATCH 264/333] added cancellation token Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index a6eafc67fba1..8fb58d086586 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -7,7 +7,6 @@ /// \file parallel/algorithms/uninitialized_copy.hpp #pragma once -#include #if defined(DOXYGEN) namespace hpx { From 79bd6fa6ad6600499bcd80a0d923d23c0300bb74 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 4 Jun 2023 15:24:54 -0500 Subject: [PATCH 265/333] fixed unused error Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/algorithms/uninitialized_copy.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp index 8fb58d086586..013eca34486f 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_copy.hpp @@ -380,6 +380,7 @@ namespace hpx::parallel { template )> + static util::in_out_result sequential( ExPolicy&& policy, InIter first, std::size_t count, FwdIter2 dest) From b6857d448c13111ab4217855debc77d91f7ca0c0 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 04:59:55 -0500 Subject: [PATCH 266/333] customisation point for loop_with_cleanup Signed-off-by: Hari Hara Naveen S --- .../include/hpx/parallel/util/loop.hpp | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index b3207bc980e7..ba45bebd1f51 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -92,6 +92,51 @@ namespace hpx::parallel::util { HPX_CONCEPT_REQUIRES_(hpx::is_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) + { + return call( + hpx::execution::seq, it, end, tok, HPX_FORWARD(F, f)); + } + + template && + !hpx::is_parallel_execution_policy_v)> + HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( + ExPolicy, Begin it, End end, F&& f) + { + // clang-format off + HPX_IVDEP HPX_UNROLL HPX_VECTORIZE + for (/**/; it != end; ++it) + { + HPX_INVOKE(f, it); + } + // clang-format on + + return it; + } + + template && + !hpx::is_parallel_execution_policy_v)> + HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( + ExPolicy, Begin it, End end, F&& f) + { + for (/**/; it != end; ++it) + { + HPX_INVOKE(f, it); + } + + return it; + } + + template + HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( + typename ExPolicy policy, Begin it, End end, CancelToken& tok, + F&& f) { // check at the start of a partition only if (tok.was_cancelled()) From 9dd52ccfc34f256ef1b74716f5b88483ebf5a20d Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 05:17:19 -0500 Subject: [PATCH 267/333] fixed template and typename mismatch Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index ba45bebd1f51..44e7ebf8f2af 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -117,12 +117,12 @@ namespace hpx::parallel::util { } template && !hpx::is_parallel_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy, Begin it, End end, F&& f) + ExPolicy&&, Begin it, End end, F&& f) { for (/**/; it != end; ++it) { @@ -135,8 +135,7 @@ namespace hpx::parallel::util { template HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( - typename ExPolicy policy, Begin it, End end, CancelToken& tok, - F&& f) + ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) { // check at the start of a partition only if (tok.was_cancelled()) From a2820863caa3d4f84d2a666e425a15a713fa4d2b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Wed, 7 Jun 2023 09:22:02 -0500 Subject: [PATCH 268/333] fixed ambiguity using concepts Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 44e7ebf8f2af..527a021c7c5b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -119,8 +119,7 @@ namespace hpx::parallel::util { template && - !hpx::is_parallel_execution_policy_v)> + hpx::is_sequenced_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( ExPolicy&&, Begin it, End end, F&& f) { From 2502620586abc05fb08954823ad296a99f2714f6 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 11 Jun 2023 17:24:55 -0500 Subject: [PATCH 269/333] fixed concept Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_default_construct.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 47d3967b9dde..91b04c17a9da 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -289,7 +289,7 @@ namespace hpx::parallel { static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - return parallel_uninitialized_default_construct_n( + parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } From 1b234da8cda0366e35b45cbfd552ea563809f7e8 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Tue, 13 Jun 2023 16:21:26 -0500 Subject: [PATCH 270/333] fixed error from test which hass seninal not convertible to iterable type Signed-off-by: Hari Hara Naveen S --- .../hpx/parallel/algorithms/uninitialized_default_construct.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 91b04c17a9da..47d3967b9dde 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -289,7 +289,7 @@ namespace hpx::parallel { static util::detail::algorithm_result_t parallel( ExPolicy&& policy, FwdIter first, Sent last) { - parallel_uninitialized_default_construct_n( + return parallel_uninitialized_default_construct_n( HPX_FORWARD(ExPolicy, policy), first, detail::distance(first, last)); } From f8e52e2e3be353bde43d6136c4410670de7ab41b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 10:59:38 -0500 Subject: [PATCH 271/333] reeplaced try catch loops with loop_with_cleanup Signed-off-by: Hari Hara Naveen S --- .../parallel/algorithms/uninitialized_default_construct.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 47d3967b9dde..863f2646a9c6 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -198,7 +198,8 @@ namespace hpx::parallel { Iter sequential_uninitialized_default_construct( ExPolicy&& policy, Iter first, Sent last) { - using value_type = typename std::iterator_traits::value_type; + using value_type = + typename std::iterator_traits::value_type; return util::loop_with_cleanup( HPX_FORWARD(ExPolicy, policy), first, last, From b14f6d1f6ea2f2ca366078ddf6b17ef7ca59784b Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Sun, 16 Jul 2023 12:07:58 -0500 Subject: [PATCH 272/333] namespace errors Signed-off-by: Hari Hara Naveen S --- .../parallel/algorithms/uninitialized_default_construct.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp index 863f2646a9c6..47d3967b9dde 100644 --- a/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp +++ b/libs/core/algorithms/include/hpx/parallel/algorithms/uninitialized_default_construct.hpp @@ -198,8 +198,7 @@ namespace hpx::parallel { Iter sequential_uninitialized_default_construct( ExPolicy&& policy, Iter first, Sent last) { - using value_type = - typename std::iterator_traits::value_type; + using value_type = typename std::iterator_traits::value_type; return util::loop_with_cleanup( HPX_FORWARD(ExPolicy, policy), first, last, From efa48dc3d47a11bc3fc54b26a6c205607636d245 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Mon, 17 Jul 2023 02:01:10 -0500 Subject: [PATCH 273/333] fixed template errors Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 527a021c7c5b..ec9bca1c7ec2 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -851,7 +851,6 @@ namespace hpx::parallel::util { } namespace detail { - // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. struct loop_with_cleanup_n final From 080d41cd529a2cfa8ed796fb7e7dbb5615f28ca7 Mon Sep 17 00:00:00 2001 From: Hari Hara Naveen S Date: Fri, 28 Jul 2023 15:28:03 -0500 Subject: [PATCH 274/333] unseq dispatch psuhed to tag_invoke stage Signed-off-by: Hari Hara Naveen S --- libs/core/algorithms/include/hpx/parallel/util/loop.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index ec9bca1c7ec2..527a021c7c5b 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -851,6 +851,7 @@ namespace hpx::parallel::util { } namespace detail { + // Helper class to repeatedly call a function a given number of times // starting from a given iterator position. struct loop_with_cleanup_n final From 1afe2bd0c2a0505abc0b7b796859480aaee0177c Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 13 Apr 2023 16:12:18 +0200 Subject: [PATCH 275/333] Remove unnecessary README.rst files --- docs/sphinx/releases/whats_new_1_10_0.rst | 31 ---------------------- libs/core/gasnet_base/docs/index.rst | 18 ------------- libs/full/parcelport_gasnet/docs/index.rst | 18 ------------- 3 files changed, 67 deletions(-) delete mode 100644 docs/sphinx/releases/whats_new_1_10_0.rst delete mode 100644 libs/core/gasnet_base/docs/index.rst delete mode 100644 libs/full/parcelport_gasnet/docs/index.rst diff --git a/docs/sphinx/releases/whats_new_1_10_0.rst b/docs/sphinx/releases/whats_new_1_10_0.rst deleted file mode 100644 index 38ed581aa559..000000000000 --- a/docs/sphinx/releases/whats_new_1_10_0.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. - Copyright (C) 2007-2023 Hartmut Kaiser - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -.. _hpx_1_10_0: - -=========================== -|hpx| V1.10.0 (TBD) -=========================== - -General changes -=============== - -Breaking changes -================ - -- The |cmake| configuration keys ``SOMELIB_ROOT`` (e.g., ``BOOST_ROOT``) have been - renamed to ``Somelib_ROOT`` (e.g., ``Boost_ROOT``) to avoid warnings when using - newer versions of |cmake|. Please update your scripts accordingly. For now, the - old variable names are re-assigned to the new names and unset in the |cmake| - cache. - -Closed issues -============= - -Closed pull requests -==================== - diff --git a/libs/core/gasnet_base/docs/index.rst b/libs/core/gasnet_base/docs/index.rst deleted file mode 100644 index cab19901c949..000000000000 --- a/libs/core/gasnet_base/docs/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. - Copyright (c) 2020 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -.. _modules_gasnet_base: - -=========== -gasnet_base -=========== - -This module provides helper functionality for detecting GASNET environments. - -See the :ref:`API reference ` of this module for more -details. - diff --git a/libs/full/parcelport_gasnet/docs/index.rst b/libs/full/parcelport_gasnet/docs/index.rst deleted file mode 100644 index e26471c5e5e6..000000000000 --- a/libs/full/parcelport_gasnet/docs/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. - Copyright (c) 2020-2021 The STE||AR-Group - - SPDX-License-Identifier: BSL-1.0 - Distributed under the Boost Software License, Version 1.0. (See accompanying - file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -.. _modules_parcelport_gasnet: - -================= -parcelport_gasnet -================= - -TODO: High-level description of the module. - -See the :ref:`API reference ` of this module for more -details. - From 0ffaeb22c24af54e97575daf0c190dc03fb4042b Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 22 May 2023 09:26:06 -0500 Subject: [PATCH 276/333] Preparing V1.9.1 release --- docs/sphinx/releases/whats_new_1_9_1.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/sphinx/releases/whats_new_1_9_1.rst b/docs/sphinx/releases/whats_new_1_9_1.rst index 34c56d86a857..792f2206e487 100644 --- a/docs/sphinx/releases/whats_new_1_9_1.rst +++ b/docs/sphinx/releases/whats_new_1_9_1.rst @@ -34,7 +34,6 @@ here was on completing the API documentation of the most important API functions We have started adding migration guides for people interested in moving their codes away from other, commonplace parallelization frameworks like OpenMP. - Breaking changes ================ @@ -54,7 +53,6 @@ Closed issues * :hpx-issue:`6247` - HPX 1.9.0 does not compile with GCC on Windows * :hpx-issue:`6282` - The "attach-debugger" option is broken on the current master branch. - Closed pull requests ==================== From 1e01519521f4b79d17fa8e33d8428e4191ab738a Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 18 May 2023 21:03:40 -0500 Subject: [PATCH 277/333] Resolve gcc problems on Windows - flyby: resolve winsock2.h warnings on windows - flyby: add CI builder for gcc/mingw on Windows --- .github/workflows/windows_debug_gcc_mingw.yml | 77 +++++++++++++++++++ libs/core/runtime_local/CMakeLists.txt | 4 + 2 files changed, 81 insertions(+) create mode 100644 .github/workflows/windows_debug_gcc_mingw.yml diff --git a/.github/workflows/windows_debug_gcc_mingw.yml b/.github/workflows/windows_debug_gcc_mingw.yml new file mode 100644 index 000000000000..4f61f32fc9b3 --- /dev/null +++ b/.github/workflows/windows_debug_gcc_mingw.yml @@ -0,0 +1,77 @@ +# Copyright (c) 2023 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +name: Windows CI (Debug, gcc/mingw toolset) + +on: [pull_request] + +jobs: + build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + - uses: jwlawson/actions-setup-cmake@v1.9 + with: + cmake-version: '3.22.x' + - name: Install dependencies + run: | + choco install ninja -y + md C:\projects + $client = new-object System.Net.WebClient + $client.DownloadFile("https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z","C:\projects\boost_1_78_0.7z") + 7z x C:\projects\boost_1_78_0.7z -y -oC:\projects\boost + cd C:\projects\boost\boost_1_78_0 + .\bootstrap.bat gcc + .\b2.exe ` + link=shared ` + variant=debug ` + architecture=x86 ` + address-model=64 ` + threading=multi ` + --with-iostreams ` + --build-type=complete ` + install + $client.DownloadFile("https://download.open-mpi.org/release/hwloc/v2.8/hwloc-win64-build-2.8.0.zip","C:\projects\hwloc-win64-build-2.8.0.zip") + 7z x C:\projects\hwloc-win64-build-2.8.0.zip -y -oC:\projects + - name: Configure + shell: bash + run: | + cmake . -Bbuild -G'Ninja' \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=g++ \ + -DBOOST_ROOT="C:\Boost" \ + -DHWLOC_ROOT="C:\projects\hwloc-win64-build-2.8.0" \ + -DHPX_WITH_CXX_STANDARD=20 \ + -DHPX_WITH_STATIC_LINKING=ON \ + -DHPX_WITH_MALLOC=system \ + -DHPX_WITH_FETCH_ASIO=ON \ + -DHPX_WITH_PKGCONFIG=OFF \ + -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ + -DHPX_WITH_EXAMPLES=ON \ + -DHPX_WITH_TESTS=ON \ + -DHPX_WITH_TESTS_UNIT=ON \ + -DHPX_WITH_DEPRECATION_WARNINGS=OFF \ + -DHPX_WITH_MODULES_AS_STATIC_LIBRARIES=OFF + - name: Build + shell: bash + run: | + cmake --build build --config Debug -- -j 4 + - name: Install + shell: bash + run: | + cmake --install build --config Debug + - name: Test + run: | + Set-Alias -Name grep -Value 'C:\Program Files\Git\usr\bin\grep.exe' + Set-Alias -Name sed -Value 'C:\Program Files\Git\usr\bin\sed.exe' + cd build + ctest ` + --output-on-failure ` + --build-config Debug ` + --tests-regex tests.examples ` + --exclude-regex ` + $(grep -v -e ^# -e ^$ D:/a/hpx/hpx/.github/workflows/tests.examples.targets | sed ':b;N;$!bb;s/\n/|/g') diff --git a/libs/core/runtime_local/CMakeLists.txt b/libs/core/runtime_local/CMakeLists.txt index b0215a7d8942..864a8bc32fc4 100644 --- a/libs/core/runtime_local/CMakeLists.txt +++ b/libs/core/runtime_local/CMakeLists.txt @@ -38,6 +38,10 @@ set(runtime_local_headers hpx/runtime_local/thread_stacktrace.hpp ) +if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) + set(runtime_local_headers ${runtime_local_headers} hpx/init.hpp) +endif() + # cmake-format: off set(runtime_local_compat_headers hpx/custom_exception_info.hpp => hpx/modules/runtime_local.hpp From d6d095581e957fbe1d3f4cba6ba78087e1340ef6 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 21 May 2023 14:45:17 -0500 Subject: [PATCH 278/333] Remove duplicate headers, instead generate those on demand --- .github/workflows/windows_debug_gcc_mingw.yml | 77 ------------------- docs/sphinx/api/public_api.rst | 12 +-- libs/core/runtime_local/CMakeLists.txt | 4 - 3 files changed, 6 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/windows_debug_gcc_mingw.yml diff --git a/.github/workflows/windows_debug_gcc_mingw.yml b/.github/workflows/windows_debug_gcc_mingw.yml deleted file mode 100644 index 4f61f32fc9b3..000000000000 --- a/.github/workflows/windows_debug_gcc_mingw.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2023 The STE||AR-Group -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -name: Windows CI (Debug, gcc/mingw toolset) - -on: [pull_request] - -jobs: - build: - runs-on: windows-latest - - steps: - - uses: actions/checkout@v3 - - uses: jwlawson/actions-setup-cmake@v1.9 - with: - cmake-version: '3.22.x' - - name: Install dependencies - run: | - choco install ninja -y - md C:\projects - $client = new-object System.Net.WebClient - $client.DownloadFile("https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z","C:\projects\boost_1_78_0.7z") - 7z x C:\projects\boost_1_78_0.7z -y -oC:\projects\boost - cd C:\projects\boost\boost_1_78_0 - .\bootstrap.bat gcc - .\b2.exe ` - link=shared ` - variant=debug ` - architecture=x86 ` - address-model=64 ` - threading=multi ` - --with-iostreams ` - --build-type=complete ` - install - $client.DownloadFile("https://download.open-mpi.org/release/hwloc/v2.8/hwloc-win64-build-2.8.0.zip","C:\projects\hwloc-win64-build-2.8.0.zip") - 7z x C:\projects\hwloc-win64-build-2.8.0.zip -y -oC:\projects - - name: Configure - shell: bash - run: | - cmake . -Bbuild -G'Ninja' \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_CXX_COMPILER=g++ \ - -DBOOST_ROOT="C:\Boost" \ - -DHWLOC_ROOT="C:\projects\hwloc-win64-build-2.8.0" \ - -DHPX_WITH_CXX_STANDARD=20 \ - -DHPX_WITH_STATIC_LINKING=ON \ - -DHPX_WITH_MALLOC=system \ - -DHPX_WITH_FETCH_ASIO=ON \ - -DHPX_WITH_PKGCONFIG=OFF \ - -DHPX_WITH_TESTS_MAX_THREADS_PER_LOCALITY=2 \ - -DHPX_WITH_EXAMPLES=ON \ - -DHPX_WITH_TESTS=ON \ - -DHPX_WITH_TESTS_UNIT=ON \ - -DHPX_WITH_DEPRECATION_WARNINGS=OFF \ - -DHPX_WITH_MODULES_AS_STATIC_LIBRARIES=OFF - - name: Build - shell: bash - run: | - cmake --build build --config Debug -- -j 4 - - name: Install - shell: bash - run: | - cmake --install build --config Debug - - name: Test - run: | - Set-Alias -Name grep -Value 'C:\Program Files\Git\usr\bin\grep.exe' - Set-Alias -Name sed -Value 'C:\Program Files\Git\usr\bin\sed.exe' - cd build - ctest ` - --output-on-failure ` - --build-config Debug ` - --tests-regex tests.examples ` - --exclude-regex ` - $(grep -v -e ^# -e ^$ D:/a/hpx/hpx/.github/workflows/tests.examples.targets | sed ':b;N;$!bb;s/\n/|/g') diff --git a/docs/sphinx/api/public_api.rst b/docs/sphinx/api/public_api.rst index 4138a0bf1bb3..288282783f6b 100644 --- a/docs/sphinx/api/public_api.rst +++ b/docs/sphinx/api/public_api.rst @@ -23,7 +23,7 @@ sub-namespaces will eventually be removed. ``hpx/algorithm.hpp`` ===================== -The header :hpx-header:`libs/full/include/include,hpx/algorithm.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/algorithm.hpp` corresponds to the C++ standard library header :cppreference-header:`algorithm`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -253,7 +253,7 @@ Macros ``hpx/barrier.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/barrier.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/barrier.hpp` corresponds to the C++ standard library header :cppreference-header:`barrier` and contains a distributed barrier implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -282,7 +282,7 @@ Classes ``hpx/channel.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/channel.hpp` contains a local and a +The header :hpx-header:`libs/full/include,hpx/channel.hpp` contains a local and a distributed channel implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -493,7 +493,7 @@ Functions ``hpx/future.hpp`` ================== -The header :hpx-header:`libs/full/include/include,hpx/future.hpp` corresponds to the +The header :hpx-header:`libs/full/include,hpx/future.hpp` corresponds to the C++ standard library header :cppreference-header:`future`. See :ref:`extend_futures` for more information about extensions to futures compared to the C++ standard library. @@ -610,7 +610,7 @@ Functions ``hpx/latch.hpp`` ================= -The header :hpx-header:`libs/full/include/include,hpx/latch.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include,hpx/latch.hpp` corresponds to the C++ standard library header :cppreference-header:`latch`. It contains a local and a distributed latch implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -789,7 +789,7 @@ Classes ``hpx/runtime.hpp`` =================== -The header :hpx-header:`libs/full/include/include,hpx/runtime.hpp` contains functions for accessing +The header :hpx-header:`libs/full/include,hpx/runtime.hpp` contains functions for accessing local and distributed runtime information. Typedefs diff --git a/libs/core/runtime_local/CMakeLists.txt b/libs/core/runtime_local/CMakeLists.txt index 864a8bc32fc4..b0215a7d8942 100644 --- a/libs/core/runtime_local/CMakeLists.txt +++ b/libs/core/runtime_local/CMakeLists.txt @@ -38,10 +38,6 @@ set(runtime_local_headers hpx/runtime_local/thread_stacktrace.hpp ) -if(NOT HPX_WITH_DISTRIBUTED_RUNTIME) - set(runtime_local_headers ${runtime_local_headers} hpx/init.hpp) -endif() - # cmake-format: off set(runtime_local_compat_headers hpx/custom_exception_info.hpp => hpx/modules/runtime_local.hpp From b1bff0e11e5c23034fb5f5d8c05288672718890b Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 10 Apr 2023 15:51:24 -0500 Subject: [PATCH 279/333] Cleaning up #includes in hpx/ folder - unifying `` and `` --- libs/full/init_runtime/src/hpx_init.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/full/init_runtime/src/hpx_init.cpp b/libs/full/init_runtime/src/hpx_init.cpp index cb0fc1eab038..455038ed253a 100644 --- a/libs/full/init_runtime/src/hpx_init.cpp +++ b/libs/full/init_runtime/src/hpx_init.cpp @@ -99,7 +99,7 @@ namespace hpx_startup { std::vector (*user_main_config_function)( std::vector const&) = nullptr; -} +} // namespace hpx_startup /////////////////////////////////////////////////////////////////////////////// namespace hpx::detail { From 1350bb5c6feddaa1de0d0d780dc46166899f1fda Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Wed, 19 Jul 2023 17:49:57 -0500 Subject: [PATCH 280/333] Adapting build system for CMake V3.27 --- cmake/HPX_SetupLCI.cmake | 4 ++-- .../command_line_handling_local/cmake/HPX_SetupJSON.cmake | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/HPX_SetupLCI.cmake b/cmake/HPX_SetupLCI.cmake index d34cc6e08ec0..26b1ecd18076 100644 --- a/cmake/HPX_SetupLCI.cmake +++ b/cmake/HPX_SetupLCI.cmake @@ -38,14 +38,14 @@ macro(hpx_setup_lci) ) else() hpx_info( - "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_LCI_TAG=${HPX_WITH_LCI_TAG})" + "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Lci_TAG=${HPX_WITH_Lci_TAG})" ) endif() include(FetchContent) fetchcontent_declare( lci GIT_REPOSITORY https://github.com/uiuc-hpc/LC.git - GIT_TAG ${HPX_WITH_LCI_TAG} + GIT_TAG ${HPX_WITH_Lci_TAG} ) fetchcontent_getproperties(lci) diff --git a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake index faf92256426e..7e566489dd56 100644 --- a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake +++ b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake @@ -26,7 +26,7 @@ elseif(NOT TARGET Json::json) ) else() hpx_info( - "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_JSON_TAG=${HPX_WITH_JSON_TAG})" + "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Json_TAG=${HPX_WITH_Json_TAG})" ) endif() @@ -45,7 +45,7 @@ elseif(NOT TARGET Json::json) json SYSTEM INTERFACE $ $ ) - target_compile_definitions(json INTERFACE JSON_HAS_CPP_17) + target_compile_definitions(json INTERFACE Json_HAS_CPP_17) install( TARGETS json From c7ab7443370fa67abcc1a8353e3cd8af44f19561 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 19 Aug 2023 10:16:43 -0500 Subject: [PATCH 281/333] Remove obsolete and broken Daint builders --- .jenkins/lsu-test-coverage/env-test-coverage.sh | 14 -------------- .../slurm-constraint-test-coverage.sh | 7 ------- 2 files changed, 21 deletions(-) delete mode 100755 .jenkins/lsu-test-coverage/env-test-coverage.sh delete mode 100755 .jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh diff --git a/.jenkins/lsu-test-coverage/env-test-coverage.sh b/.jenkins/lsu-test-coverage/env-test-coverage.sh deleted file mode 100755 index ab12b28817f4..000000000000 --- a/.jenkins/lsu-test-coverage/env-test-coverage.sh +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2023 Panos Syskakis -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -set -eu - -module avail -module purge -module load cmake -module load gcc/12 -module load boost/1.79.0-debug -module load hwloc -module load openmpi diff --git a/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh b/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh deleted file mode 100755 index 008965ef474b..000000000000 --- a/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2023 Panos Syskakis -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -configuration_slurm_partition="jenkins-compute" From 946673046f9d8376f4c3fd363a85a238a7450bc7 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sun, 20 Aug 2023 09:52:20 -0500 Subject: [PATCH 282/333] Cleanup previous patch adapting to CMake V3.27 --- .../command_line_handling_local/cmake/HPX_SetupJSON.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake index 7e566489dd56..faf92256426e 100644 --- a/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake +++ b/libs/core/command_line_handling_local/cmake/HPX_SetupJSON.cmake @@ -26,7 +26,7 @@ elseif(NOT TARGET Json::json) ) else() hpx_info( - "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Json_TAG=${HPX_WITH_Json_TAG})" + "HPX_WITH_FETCH_JSON=${HPX_WITH_FETCH_JSON}, JSON will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_JSON_TAG=${HPX_WITH_JSON_TAG})" ) endif() @@ -45,7 +45,7 @@ elseif(NOT TARGET Json::json) json SYSTEM INTERFACE $ $ ) - target_compile_definitions(json INTERFACE Json_HAS_CPP_17) + target_compile_definitions(json INTERFACE JSON_HAS_CPP_17) install( TARGETS json From a932827b915b65e9c84100407e7bbf2fd7600e21 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Thu, 24 Aug 2023 11:45:15 -0500 Subject: [PATCH 283/333] More fixes for CMake V3.27 --- cmake/HPX_SetupLCI.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupLCI.cmake b/cmake/HPX_SetupLCI.cmake index 26b1ecd18076..d34cc6e08ec0 100644 --- a/cmake/HPX_SetupLCI.cmake +++ b/cmake/HPX_SetupLCI.cmake @@ -38,14 +38,14 @@ macro(hpx_setup_lci) ) else() hpx_info( - "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_Lci_TAG=${HPX_WITH_Lci_TAG})" + "HPX_WITH_FETCH_LCI=${HPX_WITH_FETCH_LCI}, LCI will be fetched using CMake's FetchContent and installed alongside HPX (HPX_WITH_LCI_TAG=${HPX_WITH_LCI_TAG})" ) endif() include(FetchContent) fetchcontent_declare( lci GIT_REPOSITORY https://github.com/uiuc-hpc/LC.git - GIT_TAG ${HPX_WITH_Lci_TAG} + GIT_TAG ${HPX_WITH_LCI_TAG} ) fetchcontent_getproperties(lci) From 69a2e55d102ecb5c732094df0f3aa2b627f83873 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 23 Aug 2023 11:24:12 +0200 Subject: [PATCH 284/333] Add scatter and gather doc --- docs/sphinx/manual/migration_guide.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index 31c0622c74bf..efc391644196 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1233,6 +1233,7 @@ the gathered data in the root process. std::cout << std::endl; } } + std::cout << std::endl; MPI_Finalize(); return 0; From afd942cbc6eca96dbbc1b4b250478c0260e501b8 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 24 Aug 2023 15:59:28 +0200 Subject: [PATCH 285/333] Add MPI_Alltoall doc --- docs/sphinx/manual/migration_guide.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index efc391644196..af3a790db23d 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1639,6 +1639,12 @@ MPI_Alltoall ------------- The following code gathers data from and scatters data to all processes. +======= +MPI_Alltoall +------------- + +The following code cGathers data from and scatters data to all processes. +>>>>>>> ac67290f86 (Add MPI_Alltoall doc) |mpi| code: @@ -1739,6 +1745,7 @@ detail: - The `get()` function waits until the result is available and then stores it in the variable `r`. +<<<<<<< HEAD .. _MPI_Barrier: MPI_Barrier From 12b30537e87c6152f7ad3fc37d996d3ad08c6e6a Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Mon, 25 Nov 2019 12:49:34 -0600 Subject: [PATCH 286/333] Adding local work stealing scheduler that is based on message passing internally - Using uniform_int_distribution with proper bounds - Removing queue index from thread_queues as it was unused - flyby: remove commented out options from .clang-format - Renaming workstealing --> workrequesting - Adding adaptive work stealing (steal half/steal one) - this makes this scheduler consistently (albeit only slightly) faster than the (default) local-priority scheduler - Adding LIFO and FIFO variations of local work-stealing scheduler - flyby: fixing HPX_WITH_SWAP_CONTEXT_EMULATION - flyby: minor changes to fibonacci_local example - Adding high- and low- priority queues - flyby: cache_line_data now does not generate warnings errors if padding is not needed - Adding bound queues - flyby: using cache_line_data for scheduler states --- .../include/hpx/modules/schedulers.hpp | 2 -- .../include/hpx/schedulers/thread_queue.hpp | 3 ++- .../include/hpx/thread_pools/scheduling_loop.hpp | 7 ++++++- .../hpx/threading_base/thread_helpers.hpp | 3 --- libs/core/threading_base/src/thread_helpers.cpp | 16 ---------------- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/libs/core/schedulers/include/hpx/modules/schedulers.hpp b/libs/core/schedulers/include/hpx/modules/schedulers.hpp index 2de14cc98363..d6790d352d73 100644 --- a/libs/core/schedulers/include/hpx/modules/schedulers.hpp +++ b/libs/core/schedulers/include/hpx/modules/schedulers.hpp @@ -6,8 +6,6 @@ #pragma once -#include - #include #include #include diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 4c8524bad734..25260cc85ca4 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -457,7 +457,8 @@ namespace hpx::threads::policies { if (delete_all) { - // do not lock mutex while deleting all threads, do it piece-wise + // do not lock mutex while deleting all threads, do it + // piece-wise while (true) { std::unique_lock lk(mtx_, std::try_to_lock); diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp index 6a6dbb766d69..35b47b8a3d3a 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #if defined(HPX_HAVE_APEX) #include @@ -178,6 +179,7 @@ namespace hpx::threads::detail { idle_collect_rate idle_rate(counters.tfunc_time_, counters.exec_time_); [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector(idle_rate); + HPX_UNUSED(tfunc_time_collector); // spin for some time after queues have become empty bool may_exit = false; @@ -235,6 +237,8 @@ namespace hpx::threads::detail { scheduler.get_next_thread( num_thread, running, thrd, enable_stealing))) { + HPX_UNUSED(tfunc_time_collector); + HPX_ASSERT(get_thread_id_data(thrd)->get_scheduler_base() == &scheduler); @@ -269,6 +273,7 @@ namespace hpx::threads::detail { [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector_inner(idle_rate); + HPX_UNUSED(tfunc_time_collector); // thread returns new required state store the // returned state in the thread @@ -289,7 +294,7 @@ namespace hpx::threads::detail { // and add to aggregate execution time. [[maybe_unused]] exec_time_wrapper exec_time_collector(idle_rate); - + HPX_UNUSED(exec_time_collector); #if defined(HPX_HAVE_APEX) // get the APEX data pointer, in case we are // resuming the thread and have to restore any diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp index 131369ccdc6c..70f5ad7c5685 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp @@ -387,9 +387,6 @@ namespace hpx::threads { HPX_CORE_EXPORT bool add_thread_exit_callback(thread_id_type const& id, hpx::function const& f, error_code& ec = throws); - HPX_CORE_EXPORT void free_thread_exit_callbacks( - thread_id_type const& id, error_code& ec = throws); - /////////////////////////////////////////////////////////////////////////// HPX_CORE_EXPORT std::size_t get_thread_data( thread_id_type const& id, error_code& ec = throws); diff --git a/libs/core/threading_base/src/thread_helpers.cpp b/libs/core/threading_base/src/thread_helpers.cpp index ecbd95b0c87d..6e3830f65430 100644 --- a/libs/core/threading_base/src/thread_helpers.cpp +++ b/libs/core/threading_base/src/thread_helpers.cpp @@ -344,22 +344,6 @@ namespace hpx::threads { return get_thread_id_data(id)->add_thread_exit_callback(f); } - void free_thread_exit_callbacks(thread_id_type const& id, error_code& ec) - { - if (HPX_UNLIKELY(!id)) - { - HPX_THROWS_IF(ec, hpx::error::null_thread_id, - "hpx::threads::add_thread_exit_callback", - "null thread id encountered"); - return; - } - - if (&ec != &throws) - ec = make_success_code(); - - get_thread_id_data(id)->free_thread_exit_callbacks(); - } - /////////////////////////////////////////////////////////////////////////// #ifdef HPX_HAVE_THREAD_FULLBACKTRACE_ON_SUSPENSION char const* get_thread_backtrace(thread_id_type const& id, error_code& ec) From 25e075d6dfb0f8ac9aa1a78bcccd1c8a997e9fc3 Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Fri, 27 Jan 2023 16:07:16 -0600 Subject: [PATCH 287/333] Fixing merge conflicts, revert changes to thread exit callbacks --- .../include/hpx/modules/schedulers.hpp | 2 ++ .../local_workrequesting_scheduler.hpp | 5 +++++ .../include/hpx/schedulers/thread_queue.hpp | 3 +-- .../include/hpx/thread_pools/scheduling_loop.hpp | 7 +------ .../hpx/threading_base/thread_helpers.hpp | 3 +++ libs/core/threading_base/src/thread_helpers.cpp | 16 ++++++++++++++++ 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/libs/core/schedulers/include/hpx/modules/schedulers.hpp b/libs/core/schedulers/include/hpx/modules/schedulers.hpp index d6790d352d73..2de14cc98363 100644 --- a/libs/core/schedulers/include/hpx/modules/schedulers.hpp +++ b/libs/core/schedulers/include/hpx/modules/schedulers.hpp @@ -6,6 +6,8 @@ #pragma once +#include + #include #include #include diff --git a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp index bd9909fb9472..eae1ca8c77ea 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp @@ -221,6 +221,11 @@ namespace hpx::threads::policies { scheduler_data& operator=(scheduler_data const&) = delete; scheduler_data& operator=(scheduler_data&&) = delete; + scheduler_data(scheduler_data const&) = delete; + scheduler_data(scheduler_data&&) = delete; + scheduler_data& operator=(scheduler_data const&) = delete; + scheduler_data& operator=(scheduler_data&&) = delete; + ~scheduler_data() { delete queue_; diff --git a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp index 25260cc85ca4..4c8524bad734 100644 --- a/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/thread_queue.hpp @@ -457,8 +457,7 @@ namespace hpx::threads::policies { if (delete_all) { - // do not lock mutex while deleting all threads, do it - // piece-wise + // do not lock mutex while deleting all threads, do it piece-wise while (true) { std::unique_lock lk(mtx_, std::try_to_lock); diff --git a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp index 35b47b8a3d3a..6a6dbb766d69 100644 --- a/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp +++ b/libs/core/thread_pools/include/hpx/thread_pools/scheduling_loop.hpp @@ -19,7 +19,6 @@ #include #include #include -#include #if defined(HPX_HAVE_APEX) #include @@ -179,7 +178,6 @@ namespace hpx::threads::detail { idle_collect_rate idle_rate(counters.tfunc_time_, counters.exec_time_); [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector(idle_rate); - HPX_UNUSED(tfunc_time_collector); // spin for some time after queues have become empty bool may_exit = false; @@ -237,8 +235,6 @@ namespace hpx::threads::detail { scheduler.get_next_thread( num_thread, running, thrd, enable_stealing))) { - HPX_UNUSED(tfunc_time_collector); - HPX_ASSERT(get_thread_id_data(thrd)->get_scheduler_base() == &scheduler); @@ -273,7 +269,6 @@ namespace hpx::threads::detail { [[maybe_unused]] tfunc_time_wrapper tfunc_time_collector_inner(idle_rate); - HPX_UNUSED(tfunc_time_collector); // thread returns new required state store the // returned state in the thread @@ -294,7 +289,7 @@ namespace hpx::threads::detail { // and add to aggregate execution time. [[maybe_unused]] exec_time_wrapper exec_time_collector(idle_rate); - HPX_UNUSED(exec_time_collector); + #if defined(HPX_HAVE_APEX) // get the APEX data pointer, in case we are // resuming the thread and have to restore any diff --git a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp index 70f5ad7c5685..131369ccdc6c 100644 --- a/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp +++ b/libs/core/threading_base/include/hpx/threading_base/thread_helpers.hpp @@ -387,6 +387,9 @@ namespace hpx::threads { HPX_CORE_EXPORT bool add_thread_exit_callback(thread_id_type const& id, hpx::function const& f, error_code& ec = throws); + HPX_CORE_EXPORT void free_thread_exit_callbacks( + thread_id_type const& id, error_code& ec = throws); + /////////////////////////////////////////////////////////////////////////// HPX_CORE_EXPORT std::size_t get_thread_data( thread_id_type const& id, error_code& ec = throws); diff --git a/libs/core/threading_base/src/thread_helpers.cpp b/libs/core/threading_base/src/thread_helpers.cpp index 6e3830f65430..ecbd95b0c87d 100644 --- a/libs/core/threading_base/src/thread_helpers.cpp +++ b/libs/core/threading_base/src/thread_helpers.cpp @@ -344,6 +344,22 @@ namespace hpx::threads { return get_thread_id_data(id)->add_thread_exit_callback(f); } + void free_thread_exit_callbacks(thread_id_type const& id, error_code& ec) + { + if (HPX_UNLIKELY(!id)) + { + HPX_THROWS_IF(ec, hpx::error::null_thread_id, + "hpx::threads::add_thread_exit_callback", + "null thread id encountered"); + return; + } + + if (&ec != &throws) + ec = make_success_code(); + + get_thread_id_data(id)->free_thread_exit_callbacks(); + } + /////////////////////////////////////////////////////////////////////////// #ifdef HPX_HAVE_THREAD_FULLBACKTRACE_ON_SUSPENSION char const* get_thread_backtrace(thread_id_type const& id, error_code& ec) From 7f9931aaab46676e03e42c5f72fa784f3694aa43 Mon Sep 17 00:00:00 2001 From: isidorostsa Date: Fri, 18 Aug 2023 20:10:49 +0300 Subject: [PATCH 288/333] tests --- libs/core/type_support/tests/unit/is_trivially_relocatable.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp index 6952efb8a6ba..0ac3339364c9 100644 --- a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp +++ b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp @@ -197,7 +197,6 @@ static_assert(!hpx::is_trivially_relocatable_v< static_assert(!hpx::is_trivially_relocatable_v< explicitly_trivially_relocatable_1 const volatile&>); // clang-format on - // Trivial relocatability is not inherited struct derived_from_explicitly_trivially_relocatable : explicitly_trivially_relocatable_1 From f42f11f8e79ed52f570792d7671a1eef24cc8da6 Mon Sep 17 00:00:00 2001 From: dimitraka Date: Wed, 30 Aug 2023 16:17:17 +0200 Subject: [PATCH 289/333] Add MPI_Barrier docs --- docs/sphinx/manual/migration_guide.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index af3a790db23d..b2a2d9b9d57c 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1643,8 +1643,7 @@ The following code gathers data from and scatters data to all processes. MPI_Alltoall ------------- -The following code cGathers data from and scatters data to all processes. ->>>>>>> ac67290f86 (Add MPI_Alltoall doc) +The following code gathers data from and scatters data to all processes. |mpi| code: @@ -1745,7 +1744,6 @@ detail: - The `get()` function waits until the result is available and then stores it in the variable `r`. -<<<<<<< HEAD .. _MPI_Barrier: MPI_Barrier From a463053ab5b8e7cda2a17ae0955159fbcda7ab6f Mon Sep 17 00:00:00 2001 From: Panos Syskakis Date: Fri, 1 Sep 2023 13:21:23 -0500 Subject: [PATCH 290/333] Add coverage testing on Jenkins --- .jenkins/lsu-test-coverage/env-test-coverage.sh | 14 ++++++++++++++ .../slurm-constraint-test-coverage.sh | 7 +++++++ 2 files changed, 21 insertions(+) create mode 100755 .jenkins/lsu-test-coverage/env-test-coverage.sh create mode 100755 .jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh diff --git a/.jenkins/lsu-test-coverage/env-test-coverage.sh b/.jenkins/lsu-test-coverage/env-test-coverage.sh new file mode 100755 index 000000000000..ab12b28817f4 --- /dev/null +++ b/.jenkins/lsu-test-coverage/env-test-coverage.sh @@ -0,0 +1,14 @@ +# Copyright (c) 2023 Panos Syskakis +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +set -eu + +module avail +module purge +module load cmake +module load gcc/12 +module load boost/1.79.0-debug +module load hwloc +module load openmpi diff --git a/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh b/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh new file mode 100755 index 000000000000..008965ef474b --- /dev/null +++ b/.jenkins/lsu-test-coverage/slurm-constraint-test-coverage.sh @@ -0,0 +1,7 @@ +# Copyright (c) 2023 Panos Syskakis +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +configuration_slurm_partition="jenkins-compute" From 217dd47349519e72d398bf61065fa36c8e588b20 Mon Sep 17 00:00:00 2001 From: Johan511 Date: Tue, 12 Sep 2023 00:04:33 +0530 Subject: [PATCH 291/333] changed chat link to discord Signed-off-by: Johan511 --- .github/SUPPORT.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/SUPPORT.md b/.github/SUPPORT.md index 46f50f619e67..e5c12f5c473b 100644 --- a/.github/SUPPORT.md +++ b/.github/SUPPORT.md @@ -30,7 +30,6 @@ for details. [`#hpx`](https://cpplang.slack.com/messages/C68QLPZB3) channel is another place where people usually offer support. - ### Mailing Lists/Groups * [hpx-users group](mailto:hpx-users@stellar-group.org), see From 29c6c214fe6f7343cf5eea01295b769a6fe0ba3d Mon Sep 17 00:00:00 2001 From: dimitraka Date: Thu, 21 Sep 2023 19:43:44 +0200 Subject: [PATCH 292/333] Fix missing links for header files --- docs/sphinx/api/public_api.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/sphinx/api/public_api.rst b/docs/sphinx/api/public_api.rst index 288282783f6b..7493a0149080 100644 --- a/docs/sphinx/api/public_api.rst +++ b/docs/sphinx/api/public_api.rst @@ -23,7 +23,7 @@ sub-namespaces will eventually be removed. ``hpx/algorithm.hpp`` ===================== -The header :hpx-header:`libs/full/include,hpx/algorithm.hpp` corresponds to the +The header :hpx-header:`libs/full/include,include/hpx/algorithm.hpp` corresponds to the C++ standard library header :cppreference-header:`algorithm`. See :ref:`parallel_algorithms` for more information about the parallel algorithms. @@ -253,7 +253,7 @@ Macros ``hpx/barrier.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/barrier.hpp` corresponds to the +The header :hpx-header:`libs/full/include,include/hpx/barrier.hpp` corresponds to the C++ standard library header :cppreference-header:`barrier` and contains a distributed barrier implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -282,7 +282,7 @@ Classes ``hpx/channel.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/channel.hpp` contains a local and a +The header :hpx-header:`libs/full/include,include/hpx/channel.hpp` contains a local and a distributed channel implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -493,7 +493,7 @@ Functions ``hpx/future.hpp`` ================== -The header :hpx-header:`libs/full/include,hpx/future.hpp` corresponds to the +The header :hpx-header:`libs/full/include,include/hpx/future.hpp` corresponds to the C++ standard library header :cppreference-header:`future`. See :ref:`extend_futures` for more information about extensions to futures compared to the C++ standard library. @@ -610,7 +610,7 @@ Functions ``hpx/latch.hpp`` ================= -The header :hpx-header:`libs/full/include,hpx/latch.hpp` corresponds to the C++ +The header :hpx-header:`libs/full/include,include/hpx/latch.hpp` corresponds to the C++ standard library header :cppreference-header:`latch`. It contains a local and a distributed latch implementation. This functionality is also exposed through the ``hpx::distributed`` namespace. The name in ``hpx::distributed`` should be preferred. @@ -789,7 +789,7 @@ Classes ``hpx/runtime.hpp`` =================== -The header :hpx-header:`libs/full/include,hpx/runtime.hpp` contains functions for accessing +The header :hpx-header:`libs/full/include,include/hpx/runtime.hpp` contains functions for accessing local and distributed runtime information. Typedefs From ee0548ee06d7d0826ef209d69bf1f43d77bdf855 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 25 Sep 2023 21:15:02 -0400 Subject: [PATCH 293/333] fixed gasnet cmake --- cmake/HPX_SetupGasnet.cmake | 750 ++++++++++++++++++++++++++++++++++++ 1 file changed, 750 insertions(+) create mode 100644 cmake/HPX_SetupGasnet.cmake diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake new file mode 100644 index 000000000000..e895582472d2 --- /dev/null +++ b/cmake/HPX_SetupGasnet.cmake @@ -0,0 +1,750 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +# + +macro(hpx_setup_gasnet) + + if(NOT TARGET PkgConfig::GASNET) + + find_package(PkgConfig REQUIRED QUIET COMPONENTS) + + hpx_info( + "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" + ) + pkg_search_module( + GASNET IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + + if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) + + if(NOT CMAKE_C_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") + endif() + if(NOT CMAKE_CXX_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + + include(FetchContent) + FetchContent_Declare(gasnet + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz + ) + + fetchcontent_getproperties(gasnet) + if(NOT gasnet) + FetchContent_Populate(gasnet) + endif() + + message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") + + set(GASNET_DIR "${gasnet_SOURCE_DIR}") + set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") + set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + endif() + + if(GASNET_BUILD_STATUS) + message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") + endif() + + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) + message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") + endif() + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(GASNET_MPI_FOUND TRUE) + include(HPX_SetupMPI) + hpx_setup_mpi() + endif() + + if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + endif() + + if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" + ) + + endif() + +endmacro() From ff2ea4f47aa955d20dbb3c3f6b1610c2f746a6ee Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 25 Sep 2023 21:22:17 -0400 Subject: [PATCH 294/333] file changes not being registered --- cmake/HPX_SetupGasnet.cmake | 762 ++---------------------------------- 1 file changed, 30 insertions(+), 732 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index e895582472d2..37957d79ba46 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -1,750 +1,48 @@ -# Copyright (c) 2023 Christopher Taylor +# Copyright (c) 2019-2022 Ste||ar Group # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -# - -macro(hpx_setup_gasnet) - - if(NOT TARGET PkgConfig::GASNET) - - find_package(PkgConfig REQUIRED QUIET COMPONENTS) - - hpx_info( - "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" - ) - pkg_search_module( - GASNET IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - - if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) - - if(NOT CMAKE_C_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") - endif() - if(NOT CMAKE_CXX_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") - endif() - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") - endif() - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") - endif() - - include(FetchContent) - FetchContent_Declare(gasnet - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz - ) - - fetchcontent_getproperties(gasnet) - if(NOT gasnet) - FetchContent_Populate(gasnet) - endif() - - message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") - - set(GASNET_DIR "${gasnet_SOURCE_DIR}") - set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") - set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") - - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") - execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} - ) - endif() - - if(GASNET_BUILD_STATUS) - message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") - endif() - - pkg_search_module( - GASNET REQUIRED IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par - ) - elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) - message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") - endif() - - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") - set(GASNET_MPI_FOUND TRUE) - include(HPX_SetupMPI) - hpx_setup_mpi() - endif() - - if(GASNET_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - if(GASNET_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") +include(HPX_Message) +include(FindGasnet) - foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_LDFLAGS_OTHER "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - - endif() - - if(GASNET_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS "${X}") - endforeach() - endif() - - if(GASNET_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") - endforeach() - endif() - - if(GASNET_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() - - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) - endforeach() - - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS "${X}") - endforeach() - - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() - endif() - - if(GASNET_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") +macro(hpx_setup_gasnet) - foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lgasnet" IDX) - string(FIND "${X}" "-L" DIRIDX) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IDX}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" - ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-l" "" TMPSTR "${X}") - list(APPEND LIB_LIST "${TMPSTR}") - set(IDX 0) - elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") - list(APPEND FLAG_LIST "${X}") - endif() - if(NOT "${DIRIDX}" EQUAL "-1") - set(TMPSTR "") - string(REPLACE "-L" "" TMPSTR "${X}") - list(APPEND DIR_LIST "${TMPSTR}") - endif() - endforeach() + if(NOT GASNET_FOUND) - set(IDX 0) - list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() + find_gasnet() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") - endforeach() + set(GASNET_CXXFLAGS) + set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) + set(GASNET_LDFLAGS) - set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) + foreach(TOKEN ${GASNET_CFLAGS}) + list(APPEND GASNET_CXXFLAGS ${TOKEN}) + endforeach() - list(LENGTH FOUND_LIB IDX) + foreach(TOKEN ${GASNET_LDFLAGS_TMP}) + list(APPEND GASNET_LDFLAGS ${TOKEN}) + endforeach() - message(STATUS "${FOUND_LIB} ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(APPEND NEWLINK " -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" - "${NEWLINK}" IDX - ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) + add_library(gasnet INTERFACE IMPORTED) + target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) + target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) + target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) + target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + if(GASNET_MPI_FOUND) + # Setup PMI imported target + find_package(PMI) + if(PMI_FOUND) + hpx_add_config_define_namespace( + DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET ) - if("${IDX}" EQUAL "-1") - list(APPEND GASNET_LDFLAGS "${NEWLINK}") - endif() - endif() - endif() + add_library(Pmi::pmi INTERFACE IMPORTED) + target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) + target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) + endif() endif() - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES - "${GASNET_LIBRARY_DIRS}" - ) - endif() - endmacro() From 9b178ab22ac88634fea22941e68dc63f01d88657 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 22:17:25 -0400 Subject: [PATCH 295/333] fixed rebase issue with gasnet --- cmake/HPX_SetupGasnet.cmake | 763 ++++++++++++++++++++++++++++++++++-- 1 file changed, 733 insertions(+), 30 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 37957d79ba46..a648bd4a5f71 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -1,48 +1,751 @@ -# Copyright (c) 2019-2022 Ste||ar Group +# Copyright (c) 2017 Thomas Heller +# Copyright (c) 2023 Christopher Taylor # # SPDX-License-Identifier: BSL-1.0 # Distributed under the Boost Software License, Version 1.0. (See accompanying # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -include(HPX_Message) -include(FindGasnet) +# macro(hpx_setup_gasnet) - if(NOT GASNET_FOUND) + if(NOT TARGET PkgConfig::GASNET) + + find_package(PkgConfig REQUIRED QUIET COMPONENTS) + + hpx_info( + "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" + ) + pkg_search_module( + GASNET IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + + if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) + + if(NOT CMAKE_C_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") + endif() + if(NOT CMAKE_CXX_COMPILER) + message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) + message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + endif() + + include(FetchContent) + FetchContent_Declare(gasnet + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz + ) + + fetchcontent_getproperties(gasnet) + if(NOT gasnet) + FetchContent_Populate(gasnet) + endif() + + message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") + + set(GASNET_DIR "${gasnet_SOURCE_DIR}") + set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") + set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") + execute_process( + COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + endif() + + if(GASNET_BUILD_STATUS) + message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") + endif() + + pkg_search_module( + GASNET REQUIRED IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + ) + elseif((NOT GASNET_FOUND) AND (NOT HPX_WITH_FETCH_GASNET)) + message(FATAL_ERROR "GASNet not found and HPX_WITH_FETCH_GASNET not set!") + endif() + + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + set(GASNET_MPI_FOUND TRUE) + include(HPX_SetupMPI) + hpx_setup_mpi() + endif() + + if(GASNET_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS "${X}") + endforeach() + endif() + + if(GASNET_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(GASNET_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() - find_gasnet() + set(IDX 0) + list(LENGTH GASNET_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_LDFLAGS_OTHER NEWPARAM) + endforeach() - set(GASNET_CXXFLAGS) - set(GASNET_LDFLAGS_TMP ${GASNET_LDFLAGS}) - set(GASNET_LDFLAGS) + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_LDFLAGS_OTHER "${X}") + endforeach() - foreach(TOKEN ${GASNET_CFLAGS}) - list(APPEND GASNET_CXXFLAGS ${TOKEN}) - endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) - foreach(TOKEN ${GASNET_LDFLAGS_TMP}) - list(APPEND GASNET_LDFLAGS ${TOKEN}) - endforeach() + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") - add_library(gasnet INTERFACE IMPORTED) - target_compile_options(gasnet INTERFACE ${GASNET_CXXFLAGS}) - target_include_directories(gasnet INTERFACE ${GASNET_LIBRARY_DIRS}) - target_link_options(gasnet INTERFACE ${GASNET_LDFLAGS}) - target_link_libraries(gasnet INTERFACE ${GASNET_LIBRARIES}) + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) - if(GASNET_MPI_FOUND) - # Setup PMI imported target - find_package(PMI) - if(PMI_FOUND) - hpx_add_config_define_namespace( - DEFINE HPX_PARCELPORT_GASNET_HAVE_PMI NAMESPACE PARCELPORT_GASNET + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX ) - add_library(Pmi::pmi INTERFACE IMPORTED) - target_include_directories(Pmi::pmi SYSTEM INTERFACE ${PMI_INCLUDE_DIR}) - target_link_libraries(Pmi::pmi INTERFACE ${PMI_LIBRARY}) - endif() + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + endif() + + if(GASNET_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS "${X}") + endforeach() endif() + if(GASNET_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH GASNET_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(GASNET_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(GASNET_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${GASNET_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lgasnet" IDX) + string(FIND "${X}" "-L" DIRIDX) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endforeach() + + set(IDX 0) + list(LENGTH GASNET_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT GASNET_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND GASNET_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" + ) + endif() + endmacro() From d8777d151fa0c7bd5ebf5af3fab3635b64b12fae Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 22:26:42 -0400 Subject: [PATCH 296/333] updated --- CMakeLists.txt | 5 ++++ cmake/tests/set_thread_name.cpp | 12 +++++++++ .../parcelport_gasnet/tests/CMakeLists.txt | 25 ------------------- 3 files changed, 17 insertions(+), 25 deletions(-) create mode 100644 cmake/tests/set_thread_name.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 34faaa5d7743..2ec009a3eacc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1156,6 +1156,11 @@ if(HPX_WITH_NETWORKING) HPX_WITH_PARCELPORT_GASNET BOOL "Enable the GASNET based parcelport." OFF CATEGORY "Parcelport" ) + hpx_option( + HPX_WITH_FETCH_GASNET BOOL "Use FetchContent to fetch GASNET. By default an installed GASNET will be used. (default: OFF)." OFF + CATEGORY "Build Targets" + ADVANCED + ) if(HPX_WITH_PARCELPORT_GASNET) hpx_add_config_define(HPX_HAVE_PARCELPORT_GASNET) hpx_option( diff --git a/cmake/tests/set_thread_name.cpp b/cmake/tests/set_thread_name.cpp new file mode 100644 index 000000000000..d3ab3b4e1fdb --- /dev/null +++ b/cmake/tests/set_thread_name.cpp @@ -0,0 +1,12 @@ +// Copyright (c) 2023 STE||AR Group +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +int main() +{ + pthread_setname_np(pthread_self(), ""); +} diff --git a/libs/full/parcelport_gasnet/tests/CMakeLists.txt b/libs/full/parcelport_gasnet/tests/CMakeLists.txt index 1bdc527c54a2..3f1060f293c3 100644 --- a/libs/full/parcelport_gasnet/tests/CMakeLists.txt +++ b/libs/full/parcelport_gasnet/tests/CMakeLists.txt @@ -8,60 +8,35 @@ include(HPX_Message) if(HPX_WITH_TESTS) if(HPX_WITH_TESTS_UNIT) -<<<<<<< HEAD add_hpx_pseudo_target(tests.unit.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( tests.unit.modules tests.unit.modules.parcelport_gasnet -======= - add_hpx_pseudo_target(tests.unit.modules.parcelport_lci) - add_hpx_pseudo_dependencies( - tests.unit.modules tests.unit.modules.parcelport_lci ->>>>>>> 471497b530 (initial import of gasnet parcelport) ) add_subdirectory(unit) endif() if(HPX_WITH_TESTS_REGRESSIONS) -<<<<<<< HEAD add_hpx_pseudo_target(tests.regressions.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( tests.regressions.modules tests.regressions.modules.parcelport_gasnet -======= - add_hpx_pseudo_target(tests.regressions.modules.parcelport_lci) - add_hpx_pseudo_dependencies( - tests.regressions.modules tests.regressions.modules.parcelport_lci ->>>>>>> 471497b530 (initial import of gasnet parcelport) ) add_subdirectory(regressions) endif() if(HPX_WITH_TESTS_BENCHMARKS) -<<<<<<< HEAD add_hpx_pseudo_target(tests.performance.modules.parcelport_gasnet) add_hpx_pseudo_dependencies( tests.performance.modules tests.performance.modules.parcelport_gasnet -======= - add_hpx_pseudo_target(tests.performance.modules.parcelport_lci) - add_hpx_pseudo_dependencies( - tests.performance.modules tests.performance.modules.parcelport_lci ->>>>>>> 471497b530 (initial import of gasnet parcelport) ) add_subdirectory(performance) endif() if(HPX_WITH_TESTS_HEADERS) add_hpx_header_tests( -<<<<<<< HEAD modules.parcelport_gasnet HEADERS ${parcelport_gasnet_headers} HEADER_ROOT ${PROJECT_SOURCE_DIR}/include DEPENDENCIES hpx_parcelport_gasnet -======= - modules.parcelport_lci - HEADERS ${parcelport_lci_headers} - HEADER_ROOT ${PROJECT_SOURCE_DIR}/include - DEPENDENCIES hpx_parcelport_lci ->>>>>>> 471497b530 (initial import of gasnet parcelport) ) endif() endif() From 38d3c9c8225db756663106dfd3879a03409d1126 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 23:05:17 -0400 Subject: [PATCH 297/333] fixing build issues caused by rebase --- cmake/HPX_SetupGasnet.cmake | 5 +-- .../include/hpx/parallel/util/loop.hpp | 43 ------------------- .../gasnet_base/src/gasnet_environment.cpp | 2 +- libs/core/io_service/CMakeLists.txt | 2 +- .../local_workrequesting_scheduler.hpp | 5 --- .../include/hpx/parcelport_gasnet/header.hpp | 2 +- 6 files changed, 5 insertions(+), 54 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index a648bd4a5f71..0bf50336588e 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -12,14 +12,12 @@ macro(hpx_setup_gasnet) find_package(PkgConfig REQUIRED QUIET COMPONENTS) - hpx_info( - "GASNet needs to be compiled with the following environment variables set during autoconf: `CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure ...`" - ) pkg_search_module( GASNET IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par ) +message(STATUS "GASNET? ${GASNET_FOUND} ${HPX_WITH_FETCH_GASNET}") if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) if(NOT CMAKE_C_COMPILER) @@ -35,6 +33,7 @@ macro(hpx_setup_gasnet) message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") endif() + message(STATUS "Fetching GASNET") include(FetchContent) FetchContent_Declare(gasnet DOWNLOAD_EXTRACT_TIMESTAMP TRUE diff --git a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp index 527a021c7c5b..b3207bc980e7 100644 --- a/libs/core/algorithms/include/hpx/parallel/util/loop.hpp +++ b/libs/core/algorithms/include/hpx/parallel/util/loop.hpp @@ -92,49 +92,6 @@ namespace hpx::parallel::util { HPX_CONCEPT_REQUIRES_(hpx::is_execution_policy_v)> HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) - { - return call( - hpx::execution::seq, it, end, tok, HPX_FORWARD(F, f)); - } - - template && - !hpx::is_parallel_execution_policy_v)> - HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy, Begin it, End end, F&& f) - { - // clang-format off - HPX_IVDEP HPX_UNROLL HPX_VECTORIZE - for (/**/; it != end; ++it) - { - HPX_INVOKE(f, it); - } - // clang-format on - - return it; - } - - template )> - HPX_HOST_DEVICE HPX_FORCEINLINE static constexpr Begin call( - ExPolicy&&, Begin it, End end, F&& f) - { - for (/**/; it != end; ++it) - { - HPX_INVOKE(f, it); - } - - return it; - } - - template - HPX_HOST_DEVICE HPX_FORCEINLINE static Begin call( - ExPolicy&& policy, Begin it, End end, CancelToken& tok, F&& f) { // check at the start of a partition only if (tok.was_cancelled()) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 520ea6820fae..b36757b8d8af 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -614,7 +614,7 @@ namespace hpx::util { if(gasnet_environment::segments != nullptr) { delete gasnet_environment::segments; } - if(gasnet_environment::segments != segment_mutex) { + if(gasnet_environment::segment_mutex != nullptr) { delete gasnet_environment::segment_mutex; } } diff --git a/libs/core/io_service/CMakeLists.txt b/libs/core/io_service/CMakeLists.txt index ee32f0ab87a1..0563ad022cca 100644 --- a/libs/core/io_service/CMakeLists.txt +++ b/libs/core/io_service/CMakeLists.txt @@ -7,7 +7,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(io_service_headers - hpx/io_service/io_service_pool.hpp hpx/io_service/io_service_pool_fwd.hpp + hpx/io_service/io_service_pool.hpp hpx/io_service/io_service_pool.hpp hpx/io_service/io_service_thread_pool.hpp ) diff --git a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp index eae1ca8c77ea..bd9909fb9472 100644 --- a/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp +++ b/libs/core/schedulers/include/hpx/schedulers/local_workrequesting_scheduler.hpp @@ -221,11 +221,6 @@ namespace hpx::threads::policies { scheduler_data& operator=(scheduler_data const&) = delete; scheduler_data& operator=(scheduler_data&&) = delete; - scheduler_data(scheduler_data const&) = delete; - scheduler_data(scheduler_data&&) = delete; - scheduler_data& operator=(scheduler_data const&) = delete; - scheduler_data& operator=(scheduler_data&&) = delete; - ~scheduler_data() { delete queue_; diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp index 23af0fae6418..886118b244cf 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/header.hpp @@ -1,7 +1,7 @@ // Copyright (c) 2023 Christopher Taylor // Copyright (c) 2013-2021 Hartmut Kaiser // Copyright (c) 2013-2015 Thomas Heller -/ +// // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From 80efa661c468cc03beb42ade253e7a2aeef4e9b6 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 23:09:10 -0400 Subject: [PATCH 298/333] clangformatting --- libs/core/gasnet_base/src/gasnet_environment.cpp | 10 ++++++---- .../hpx/parcelport_gasnet/receiver_connection.hpp | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index b36757b8d8af..97f3c38f50d5 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -611,11 +611,13 @@ namespace hpx::util { if (enabled() && has_called_init()) { gasnet_exit(1); - if(gasnet_environment::segments != nullptr) { - delete gasnet_environment::segments; + if (gasnet_environment::segments != nullptr) + { + delete gasnet_environment::segments; } - if(gasnet_environment::segment_mutex != nullptr) { - delete gasnet_environment::segment_mutex; + if (gasnet_environment::segment_mutex != nullptr) + { + delete gasnet_environment::segment_mutex; } } } diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp index bc63472cad43..9fad1d521e90 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/receiver_connection.hpp @@ -106,7 +106,7 @@ namespace hpx::parcelset::policies::gasnet { buffer_.chunks_.resize(num_zero_copy_chunks); { hpx::util::gasnet_environment::scoped_lock l; - unsigned long elem[2] = { 0, 0 }; + unsigned long elem[2] = {0, 0}; std::memcpy(elem, hpx::util::gasnet_environment::segments[self_].addr, static_cast(buffer_.transmission_chunks_.size() * From 0c6172cd7267a753e5b400fb74d0b47ba25cd293 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 23:14:54 -0400 Subject: [PATCH 299/333] formatting fixes --- cmake/HPX_SetupGasnet.cmake | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 0bf50336588e..8bb0c559307f 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -13,11 +13,9 @@ macro(hpx_setup_gasnet) find_package(PkgConfig REQUIRED QUIET COMPONENTS) pkg_search_module( - GASNET IMPORTED_TARGET GLOBAL - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + GASNET IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par ) -message(STATUS "GASNET? ${GASNET_FOUND} ${HPX_WITH_FETCH_GASNET}") if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) if(NOT CMAKE_C_COMPILER) @@ -741,8 +739,7 @@ message(STATUS "GASNET? ${GASNET_FOUND} ${HPX_WITH_FETCH_GASNET}") PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" ) set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES - "${GASNET_LIBRARY_DIRS}" + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES "${GASNET_LIBRARY_DIRS}" ) endif() From a9a1a88797db26bd0085eccfea9cc07411a2a0e2 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 23:31:09 -0400 Subject: [PATCH 300/333] several additional fixes caused by an error in rebase --- cmake/FindGasnet.cmake | 36 +++++++++---------- libs/core/io_service/CMakeLists.txt | 4 +-- .../hpx/io_service/io_service_pool_fwd.hpp | 14 ++++++++ 3 files changed, 34 insertions(+), 20 deletions(-) create mode 100644 libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 9a20e8b33a5a..74f8d9a5706d 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -11,25 +11,25 @@ set(GASNET_UDP_FOUND FALSE) set(GASNET_SMP_FOUND FALSE) if(HPX_WITH_NETWORKING) - pkg_check_modules(GASNET QUIET gasnet-mpi-seq) - if(NOT GASNET_FOUND) - pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") - else() - set(GASNET_UDP_FOUND TRUE) - endif() - else() - set(GASNET_MPI_FOUND TRUE) - hpx_setup_mpi() - endif() + pkg_check_modules(GASNET QUIET gasnet-mpi-seq) + if(NOT GASNET_FOUND) + pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") + else() + set(GASNET_UDP_FOUND TRUE) + endif() + else() + set(GASNET_MPI_FOUND TRUE) + hpx_setup_mpi() + endif() else() - pkg_check_modules(GASNET QUIET gasnet-smp-seq) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET please set the PKG_CONFIG_PATH environment variable") - else() - set(GASNET_SMP_FOUND TRUE) - endif() + pkg_check_modules(GASNET QUIET gasnet-smp-seq) + if(NOT GASNET_FOUND) + hpx_error("Could not find GASNET please set the PKG_CONFIG_PATH environment variable") + else() + set(GASNET_SMP_FOUND TRUE) + endif() endif() if(NOT GASNET_INCLUDE_DIRS OR NOT GASNET_LIBRARY_DIRS) diff --git a/libs/core/io_service/CMakeLists.txt b/libs/core/io_service/CMakeLists.txt index 0563ad022cca..a682e30d2237 100644 --- a/libs/core/io_service/CMakeLists.txt +++ b/libs/core/io_service/CMakeLists.txt @@ -7,13 +7,13 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(io_service_headers - hpx/io_service/io_service_pool.hpp hpx/io_service/io_service_pool.hpp + hpx/io_service/io_service_pool.hpp hpx/io_service/io_service_pool_fwd.hpp hpx/io_service/io_service_thread_pool.hpp ) # cmake-format: off set(io_service_compat_headers - hpx/util/io_service_pool.hpp => hpx/modules/io_service.hpp + hpx/util/io_service_pool_fwd.hpp => hpx/modules/io_service.hpp ) # cmake-format: on diff --git a/libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp b/libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp new file mode 100644 index 000000000000..9d871fd96cd5 --- /dev/null +++ b/libs/core/io_service/include/hpx/io_service/io_service_pool_fwd.hpp @@ -0,0 +1,14 @@ +// Copyright (c) 2023 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +namespace hpx::util { + + class HPX_CORE_EXPORT io_service_pool; +} // namespace hpx::util From bdf920241ce89c59018aae9ab9829b7139f59f62 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 29 Sep 2023 23:55:56 -0400 Subject: [PATCH 301/333] fixed formatting issue caused by rebase --- .../include/hpx/parcelport_lci/parcelport_lci.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp index 95cb7ae38908..42d9b17b4cb3 100644 --- a/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp +++ b/libs/full/parcelport_lci/include/hpx/parcelport_lci/parcelport_lci.hpp @@ -1,4 +1,4 @@ -../data/kokkos/taskspawn/test1.cpp// Copyright (c) 2014-2023 Thomas Heller +// Copyright (c) 2014-2023 Thomas Heller // // SPDX-License-Identifier: BSL-1.0 // Distributed under the Boost Software License, Version 1.0. (See accompanying From 910bef041f369023002bd2fc4d39a746da076fa8 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 30 Sep 2023 00:09:40 -0400 Subject: [PATCH 302/333] rm'd rebase files that are not in main anymore --- .../type_support/uninitialized_relocate.hpp | 176 ----------- .../tests/unit/uninitialized_relocate.cpp | 276 ------------------ 2 files changed, 452 deletions(-) delete mode 100644 libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp delete mode 100644 libs/core/type_support/tests/unit/uninitialized_relocate.cpp diff --git a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp b/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp deleted file mode 100644 index 750dc601d660..000000000000 --- a/libs/core/type_support/include/hpx/type_support/uninitialized_relocate.hpp +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2023 Isidoros Tsaousis-Seiras -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#pragma once - -#include -#include -#include - -#include -#include - -#if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) -#include -#endif - -namespace hpx::experimental { - -#if defined(HPX_HAVE_P1144_STD_RELOCATE_AT) - using std::uninitialized_relocate; -#else - - namespace detail { - - enum struct relocate_strategy - { - buffer_memcpy = 0, - for_loop_nothrow, - for_loop_try_catch - }; - - template - struct choose_uninitialized_relocate_helper - { - using in_type = typename std::iterator_traits::value_type; - using out_type = typename std::iterator_traits::value_type; - - constexpr static bool valid_relocation = - hpx::is_relocatable_from_v; - - constexpr static bool is_buffer_memcpyable = - hpx::is_trivially_relocatable_v && - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The important check - std::is_same_v, - std::remove_cv_t> && - // can only relocate between same types - !std::is_volatile_v && !std::is_volatile_v && - // volatile types are not memcpyable - std::is_pointer_v && std::is_pointer_v; - // ^^ the best we can do to check for contiguous iterators - - constexpr static bool can_move_construct_nothrow = - std::is_nothrow_constructible_v>; - // Checks if the move constructor is noexcept to skip - // the try-catch block - - // Using an enum to distinguish implementations - constexpr static relocate_strategy value = is_buffer_memcpyable ? - relocate_strategy::buffer_memcpy : - can_move_construct_nothrow ? - relocate_strategy::for_loop_nothrow : - relocate_strategy::for_loop_try_catch; - }; - - template ::value == - relocate_strategy::buffer_memcpy, - int> = 0> - FwdIter uninitialized_relocate_helper( - InIter first, InIter last, FwdIter dst) noexcept - { - auto n_objects = std::distance(first, last); - - if (n_objects != 0) - { - std::byte const* first_byte = - reinterpret_cast(std::addressof(*first)); - std::byte const* last_byte = - reinterpret_cast(std::addressof(*last)); - - std::byte* dst_byte = const_cast( - reinterpret_cast(std::addressof(*dst))); - - auto n_bytes = std::distance(first_byte, last_byte); - - // Ideally we would want to convey to the compiler - // That the new buffer actually contains objects - // within their lifetime. But this is not possible - // with current language features. - std::memmove(dst_byte, first_byte, n_bytes); - - dst += n_objects; - } - - return dst; - } - - template ::value == - relocate_strategy::for_loop_nothrow, - // Either the buffer is not contiguous or the types are no-throw - // move constructible but not trivially relocatable - int> = 0> - FwdIter uninitialized_relocate_helper( - InIter first, InIter last, FwdIter dst) noexcept - { - for (; first != last; ++first, ++dst) - { - // if the type is trivially relocatable this will be a memcpy - // otherwise it will be a move + destroy - relocate_at_helper( - std::addressof(*first), std::addressof(*dst)); - } - - return dst; - } - - template ::value == - relocate_strategy::for_loop_try_catch, - int> = 0> - FwdIter uninitialized_relocate_helper( - InIter first, InIter last, FwdIter dst) - { - FwdIter original_dst = dst; - - for (; first != last; ++first, ++dst) - { - try - { - // the move + destroy version will be used - relocate_at_helper( - std::addressof(*first), std::addressof(*dst)); - } - catch (...) - { - // destroy all objects other that the one - // that caused the exception - // (relocate_at already destroyed that one) - - // destroy all objects constructed so far - std::destroy(original_dst, dst); - // destroy all the objects not relocated yet - std::destroy(++first, last); - - throw; - } - } - - return dst; - } - - } // namespace detail - - template - FwdIter - uninitialized_relocate(InIter first, InIter last, FwdIter dst) noexcept( - detail::choose_uninitialized_relocate_helper::value != - detail::relocate_strategy::for_loop_try_catch) - { - static_assert(detail::choose_uninitialized_relocate_helper::valid_relocation, - "uninitialized_move(first, last, dst) must be well-formed"); - return detail::uninitialized_relocate_helper(first, last, dst); - } - -#endif // defined(HPX_HAVE_P1144_STD_RELOCATE_AT) - -} // namespace hpx::experimental diff --git a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp b/libs/core/type_support/tests/unit/uninitialized_relocate.cpp deleted file mode 100644 index c2d55b671b67..000000000000 --- a/libs/core/type_support/tests/unit/uninitialized_relocate.cpp +++ /dev/null @@ -1,276 +0,0 @@ -// Copyright (c) 2023 Isidoros Tsaousis-Seiras -// -// SPDX-License-Identifier: BSL-1.0 -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -#include -#include -#include - -#define N 50 -#define K 10 - -struct trivially_relocatable_struct -{ - static int count; - static int move_count; - static int dtor_count; - int data; - - explicit trivially_relocatable_struct(int data) - : data(data) - { - count++; - } - trivially_relocatable_struct(trivially_relocatable_struct&& other) - : data(other.data) - { - move_count++; - count++; - } - ~trivially_relocatable_struct() - { - dtor_count++; - count--; - } - - // making sure the address is never directly accessed - friend void operator&(trivially_relocatable_struct) = delete; -}; -int trivially_relocatable_struct::count = 0; -int trivially_relocatable_struct::move_count = 0; -int trivially_relocatable_struct::dtor_count = 0; - -HPX_DECLARE_TRIVIALLY_RELOCATABLE(trivially_relocatable_struct); -static_assert(hpx::is_trivially_relocatable_v); - -struct non_trivially_relocatable_struct -{ - static int count; - static int move_count; - static int dtor_count; - int data; - - explicit non_trivially_relocatable_struct(int data) - : data(data) - { - count++; - } - // mark as noexcept to enter simpler relocation path - non_trivially_relocatable_struct( - non_trivially_relocatable_struct&& other) noexcept - : data(other.data) - { - move_count++; - count++; - } - ~non_trivially_relocatable_struct() - { - dtor_count++; - count--; - } - - // making sure the address is never directly accessed - friend void operator&(non_trivially_relocatable_struct) = delete; -}; -int non_trivially_relocatable_struct::count = 0; -int non_trivially_relocatable_struct::move_count = 0; -int non_trivially_relocatable_struct::dtor_count = 0; - -static_assert( - !hpx::is_trivially_relocatable_v); - -struct non_trivially_relocatable_struct_throwing -{ - static int count; - static int move_count; - static int dtor_count; - - int data; - - explicit non_trivially_relocatable_struct_throwing(int data) - : data(data) - { - count++; - } - // do not mark as noexcept to enter try-catch relocation path - non_trivially_relocatable_struct_throwing( - non_trivially_relocatable_struct_throwing&& other) - : data(other.data) - { - if (move_count == K) - { - throw 42; - } - move_count++; - - count++; - } - ~non_trivially_relocatable_struct_throwing() - { - dtor_count++; - count--; - } - - // making sure the address is never directly accessed - friend void operator&(non_trivially_relocatable_struct_throwing) = delete; -}; - -int non_trivially_relocatable_struct_throwing::count = 0; -int non_trivially_relocatable_struct_throwing::move_count = 0; -int non_trivially_relocatable_struct_throwing::dtor_count = 0; - -static_assert(!hpx::is_trivially_relocatable_v< - non_trivially_relocatable_struct_throwing>); - -int hpx_main() -{ - { - void* mem1 = std::malloc(N * sizeof(trivially_relocatable_struct)); - void* mem2 = std::malloc(N * sizeof(trivially_relocatable_struct)); - - HPX_TEST(mem1 && mem2); - - trivially_relocatable_struct* ptr1 = - static_cast(mem1); - trivially_relocatable_struct* ptr2 = - static_cast(mem2); - - HPX_TEST(trivially_relocatable_struct::count == 0); - HPX_TEST(trivially_relocatable_struct::move_count == 0); - HPX_TEST(trivially_relocatable_struct::dtor_count == 0); - - for (int i = 0; i < N; i++) - { - hpx::construct_at(ptr1 + i, 1234); - } - - // N objects constructed - HPX_TEST(trivially_relocatable_struct::count == N); - - // relocate them to ptr2 - hpx::experimental::uninitialized_relocate(ptr1, ptr1 + N, ptr2); - - // All creations - destructions balance out - HPX_TEST(trivially_relocatable_struct::count == N); - - // No move constructor or destructor should be called - HPX_TEST(trivially_relocatable_struct::move_count == 0); - HPX_TEST(trivially_relocatable_struct::dtor_count == 0); - - for (int i = 0; i < N; i++) - { - HPX_TEST(ptr2[i].data == 1234); - } - - std::destroy(ptr2, ptr2 + N); - HPX_TEST(trivially_relocatable_struct::dtor_count == N); - - std::free(mem1); - std::free(mem2); - } - { - void* mem1 = std::malloc(N * sizeof(non_trivially_relocatable_struct)); - void* mem2 = std::malloc(N * sizeof(non_trivially_relocatable_struct)); - - HPX_TEST(mem1 && mem2); - - non_trivially_relocatable_struct* ptr1 = - static_cast(mem1); - non_trivially_relocatable_struct* ptr2 = - static_cast(mem2); - - HPX_TEST(non_trivially_relocatable_struct::count == 0); - HPX_TEST(non_trivially_relocatable_struct::move_count == 0); - HPX_TEST(non_trivially_relocatable_struct::dtor_count == 0); - - for (int i = 0; i < N; i++) - { - hpx::construct_at(ptr1 + i, 1234); - } - - // N objects constructed - HPX_TEST(non_trivially_relocatable_struct::count == N); - - // relocate them to ptr2 - hpx::experimental::uninitialized_relocate(ptr1, ptr1 + N, ptr2); - - // All creations - destructions balance out - HPX_TEST(non_trivially_relocatable_struct::count == N); - - // Every object was moved from and then destroyed - HPX_TEST(non_trivially_relocatable_struct::move_count == N); - HPX_TEST(non_trivially_relocatable_struct::dtor_count == N); - - for (int i = 0; i < N; i++) - { - HPX_TEST(ptr2[i].data == 1234); - } - - std::destroy(ptr2, ptr2 + N); - - std::free(mem1); - std::free(mem2); - } - { - void* mem1 = - std::malloc(N * sizeof(non_trivially_relocatable_struct_throwing)); - void* mem2 = - std::malloc(N * sizeof(non_trivially_relocatable_struct_throwing)); - - HPX_TEST(mem1 && mem2); - - non_trivially_relocatable_struct_throwing* ptr1 = - static_cast(mem1); - non_trivially_relocatable_struct_throwing* ptr2 = - static_cast(mem2); - - HPX_TEST(non_trivially_relocatable_struct_throwing::count == 0); - HPX_TEST(non_trivially_relocatable_struct_throwing::move_count == 0); - HPX_TEST(non_trivially_relocatable_struct_throwing::dtor_count == 0); - - for (int i = 0; i < N; i++) - { - hpx::construct_at(ptr1 + i, 1234); - } - - // N objects constructed - HPX_TEST(non_trivially_relocatable_struct_throwing::count == N); - - // relocate them to ptr2 - try - { - hpx::experimental::uninitialized_relocate(ptr1, ptr1 + N, ptr2); - HPX_TEST(false); // should never reach this - } - catch (int forty_two) - { - HPX_TEST(forty_two == 42); - } - - // K move constructors were called - HPX_TEST(non_trivially_relocatable_struct_throwing::move_count == K); - - // K - 1 destructors were called to balance out the move constructors - // (- 1 because the last move constructor throws) - // and then N + 1 destructors were called: K on the old range and - // N - (K - 1) = N - K + 1 on the new range - HPX_TEST( - non_trivially_relocatable_struct_throwing::dtor_count == N + K); - - // It stops at K, so K-1 move-destruct pairs have been executed - // after this N - (K - 1) destructs will be done on the old range - // and K - 1 on the new range. giving 2*N total destructs - - std::free(mem1); - std::free(mem2); - } - return hpx::local::finalize(); -} - -int main(int argc, char* argv[]) -{ - hpx::local::init(hpx_main, argc, argv); - return hpx::util::report_errors(); -} From cb9982b4430aded968f4e9d435bd45a9620c7010 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 30 Sep 2023 00:37:46 -0400 Subject: [PATCH 303/333] cmake-format applied --- cmake/FindGasnet.cmake | 59 +++++++++--------- cmake/HPX_SetupGasnet.cmake | 116 +++++++++++++++++++++--------------- 2 files changed, 100 insertions(+), 75 deletions(-) diff --git a/cmake/FindGasnet.cmake b/cmake/FindGasnet.cmake index 74f8d9a5706d..1d993b773588 100644 --- a/cmake/FindGasnet.cmake +++ b/cmake/FindGasnet.cmake @@ -6,41 +6,46 @@ # file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) macro(find_gasnet) -set(GASNET_MPI_FOUND FALSE) -set(GASNET_UDP_FOUND FALSE) -set(GASNET_SMP_FOUND FALSE) + set(GASNET_MPI_FOUND FALSE) + set(GASNET_UDP_FOUND FALSE) + set(GASNET_SMP_FOUND FALSE) -if(HPX_WITH_NETWORKING) - pkg_check_modules(GASNET QUIET gasnet-mpi-seq) - if(NOT GASNET_FOUND) - pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) + if(HPX_WITH_NETWORKING) + pkg_check_modules(GASNET QUIET gasnet-mpi-seq) if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable") + pkg_check_modules(PC_GASNET QUIET gasnet-udp-seq) + if(NOT GASNET_FOUND) + hpx_error( + "Could not find GASNET MPI/udp please set the PKG_CONFIG_PATH environment variable" + ) + else() + set(GASNET_UDP_FOUND TRUE) + endif() else() - set(GASNET_UDP_FOUND TRUE) + set(GASNET_MPI_FOUND TRUE) + hpx_setup_mpi() endif() else() - set(GASNET_MPI_FOUND TRUE) - hpx_setup_mpi() - endif() -else() - pkg_check_modules(GASNET QUIET gasnet-smp-seq) - if(NOT GASNET_FOUND) - hpx_error("Could not find GASNET please set the PKG_CONFIG_PATH environment variable") - else() - set(GASNET_SMP_FOUND TRUE) + pkg_check_modules(GASNET QUIET gasnet-smp-seq) + if(NOT GASNET_FOUND) + hpx_error( + "Could not find GASNET please set the PKG_CONFIG_PATH environment variable" + ) + else() + set(GASNET_SMP_FOUND TRUE) + endif() endif() -endif() -if(NOT GASNET_INCLUDE_DIRS OR NOT GASNET_LIBRARY_DIRS) - hpx_error("Could not find GASNET_INCLUDE_DIRS or GASNET_LIBRARY_DIRS please \ + if(NOT GASNET_INCLUDE_DIRS OR NOT GASNET_LIBRARY_DIRS) + hpx_error( + "Could not find GASNET_INCLUDE_DIRS or GASNET_LIBRARY_DIRS please \ set the PKG_CONFIG_PATH environment variable" - ) -endif() + ) + endif() -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - Gasnet DEFAULT_MSG GASNET_LIBRARY_DIRS GASNET_INCLUDE_DIRS -) + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args( + Gasnet DEFAULT_MSG GASNET_LIBRARY_DIRS GASNET_INCLUDE_DIRS + ) endmacro() diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 8bb0c559307f..068abf031fb3 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -13,70 +13,97 @@ macro(hpx_setup_gasnet) find_package(PkgConfig REQUIRED QUIET COMPONENTS) pkg_search_module( - GASNET IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par + GASNET IMPORTED_TARGET GLOBAL + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par ) if((NOT GASNET_FOUND) AND HPX_WITH_FETCH_GASNET) if(NOT CMAKE_C_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}") + message( + FATAL_ERROR + "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}" + ) endif() if(NOT CMAKE_CXX_COMPILER) - message(FATAL_ERROR "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}") + message( + FATAL_ERROR + "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}" + ) endif() if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + message( + FATAL_ERROR + "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}" + ) endif() if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) - message(FATAL_ERROR "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}") + message( + FATAL_ERROR + "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}" + ) endif() message(STATUS "Fetching GASNET") include(FetchContent) - FetchContent_Declare(gasnet + fetchcontent_declare( + gasnet DOWNLOAD_EXTRACT_TIMESTAMP TRUE URL https://gasnet.lbl.gov/EX/GASNet-2023.3.0.tar.gz ) fetchcontent_getproperties(gasnet) if(NOT gasnet) - FetchContent_Populate(gasnet) + fetchcontent_populate(gasnet) endif() - message(STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}") - + message( + STATUS "Building GASNET and installing into ${CMAKE_INSTALL_PREFIX}" + ) + set(GASNET_DIR "${gasnet_SOURCE_DIR}") set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" + OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp" + ) execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} ) elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} ) elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") execute_process( - COMMAND bash -c "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" - WORKING_DIRECTORY ${GASNET_DIR} - RESULT_VARIABLE GASNET_BUILD_STATUS - OUTPUT_FILE ${GASNET_BUILD_OUTPUT} - ERROR_FILE ${GASNET_ERROR_FILE} + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} ) endif() - + if(GASNET_BUILD_STATUS) - message(FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details") + message( + FATAL_ERROR + "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details" + ) endif() pkg_search_module( @@ -253,10 +280,10 @@ macro(hpx_setup_gasnet) if("${IDX}" EQUAL "-1") list(APPEND GASNET_LDFLAGS "${NEWLINK}") endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(APPLE) set(NEWLINK "SHELL:-Wl,-force_load,") - else() + else() set(NEWLINK "SHELL: ") endif() foreach(X IN ITEMS ${LIB_LIST}) @@ -278,9 +305,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) + string(FIND "SHELL:" "${NEWLINK}" IDX) if("${IDX}" EQUAL "-1") list(APPEND GASNET_LDFLAGS "${NEWLINK}") endif() @@ -376,10 +401,10 @@ macro(hpx_setup_gasnet) if("${IDX}" EQUAL "-1") list(APPEND GASNET_LDFLAGS_OTHER "${NEWLINK}") endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(APPLE) set(NEWLINK "SHELL:-Wl,-force_load,") - else() + else() set(NEWLINK "SHELL: ") endif() foreach(X IN ITEMS ${LIB_LIST}) @@ -401,9 +426,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) + string(FIND "SHELL:" "${NEWLINK}" IDX) if("${IDX}" EQUAL "-1") list(APPEND GASNET_LDFLAGS "${NEWLINK}") endif() @@ -573,10 +596,10 @@ macro(hpx_setup_gasnet) if("${IDX}" EQUAL "-1") list(APPEND GASNET_STATIC_LDFLAGS "${NEWLINK}") endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(APPLE) set(NEWLINK "SHELL:-Wl,-force_load,") - else() + else() set(NEWLINK "SHELL: ") endif() foreach(X IN ITEMS ${LIB_LIST}) @@ -598,9 +621,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) + string(FIND "SHELL:" "${NEWLINK}" IDX) if("${IDX}" EQUAL "-1") list(APPEND GASNET_LDFLAGS "${NEWLINK}") endif() @@ -697,10 +718,10 @@ macro(hpx_setup_gasnet) if("${IDX}" EQUAL "-1") list(APPEND GASNET_STATIC_LDFLAGS_OTHER "${NEWLINK}") endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") if(APPLE) set(NEWLINK "SHELL:-Wl,-force_load,") - else() + else() set(NEWLINK "SHELL: ") endif() foreach(X IN ITEMS ${LIB_LIST}) @@ -722,9 +743,7 @@ macro(hpx_setup_gasnet) endif() endforeach() endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX - ) + string(FIND "SHELL:" "${NEWLINK}" IDX) if("${IDX}" EQUAL "-1") list(APPEND GASNET_LDFLAGS "${NEWLINK}") endif() @@ -739,7 +758,8 @@ macro(hpx_setup_gasnet) PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" ) set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES "${GASNET_LIBRARY_DIRS}" + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" ) endif() From d3da8c1f374ffec68a32526919f570f7637f7d87 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 30 Sep 2023 01:18:56 -0400 Subject: [PATCH 304/333] cmake-format fixes --- CMakeLists.txt | 5 ++++- libs/core/version/CMakeLists.txt | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ec009a3eacc..3aadfa157421 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1157,7 +1157,10 @@ if(HPX_WITH_NETWORKING) CATEGORY "Parcelport" ) hpx_option( - HPX_WITH_FETCH_GASNET BOOL "Use FetchContent to fetch GASNET. By default an installed GASNET will be used. (default: OFF)." OFF + HPX_WITH_FETCH_GASNET + BOOL + "Use FetchContent to fetch GASNET. By default an installed GASNET will be used. (default: OFF)." + OFF CATEGORY "Build Targets" ADVANCED ) diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index e4dd1faeec16..5a82861e5ec2 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -21,8 +21,9 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() if(HPX_WITH_NETWORKING - AND HPX_WITH_PARCELPORT_GASNET - AND HPX_WITH_PARCELPORT_GASNET_CONDUIT) + AND HPX_WITH_PARCELPORT_GASNET + AND HPX_WITH_PARCELPORT_GASNET_CONDUIT +) include(HPX_SetupGasnet) hpx_setup_gasnet() set(additional_dependencies ${additional_dependencies} PkgConfig::GASNET) From 3c2d3cb1b7c8ecddfc7a4afe7896cbda52b1780e Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 30 Sep 2023 10:18:40 -0500 Subject: [PATCH 305/333] Applying @isidorostsa's patch --- .../tests/unit/is_trivially_relocatable.cpp | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp index 0ac3339364c9..de6535175a79 100644 --- a/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp +++ b/libs/core/type_support/tests/unit/is_trivially_relocatable.cpp @@ -163,40 +163,6 @@ static_assert(!is_trivially_relocatable_v< // c-v-ref-array qualified versions of explicitly declared trivially relocatable // types are trivially relocatable -static_assert( - hpx::is_trivially_relocatable_v); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 volatile>); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 const volatile>); -static_assert( - hpx::is_trivially_relocatable_v); -static_assert( - hpx::is_trivially_relocatable_v); - -// Chain of c-v-array qualifiers are supported -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1[10][10]>); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 const[10]>); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 volatile[10]>); -static_assert(hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 const volatile[10]>); - -// References and temporaries are not trivially relocatable -// clang-format off -static_assert(!hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1&>); -static_assert(!hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1&&>); -static_assert(!hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 (&)[10]>); -static_assert(!hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 (&&)[10]>); -static_assert(!hpx::is_trivially_relocatable_v< - explicitly_trivially_relocatable_1 const volatile&>); -// clang-format on // Trivial relocatability is not inherited struct derived_from_explicitly_trivially_relocatable : explicitly_trivially_relocatable_1 @@ -316,9 +282,4 @@ static_assert(!is_trivially_relocatable_v); static_assert(!is_trivially_relocatable_v); static_assert(!is_trivially_relocatable_v); -// Void and function types are not trivially relocatable -static_assert(!hpx::is_trivially_relocatable_v); -static_assert(!hpx::is_trivially_relocatable_v); -static_assert(!hpx::is_trivially_relocatable_v); - int main(int, char*[]) {} From 70d132e81330d2c1ea6fdd0382641ba9b841126f Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 30 Sep 2023 15:43:29 -0500 Subject: [PATCH 306/333] Re-enable gasnet parcelport for clang based builders - remove loading gasnet module --- .jenkins/lsu/env-clang-11.sh | 3 +-- .jenkins/lsu/env-clang-12.sh | 3 +-- .jenkins/lsu/env-clang-13.sh | 3 +-- .jenkins/lsu/env-clang-14.sh | 3 +-- .jenkins/lsu/env-clang-15.sh | 3 +-- .jenkins/lsu/env-gcc-10.sh | 1 - .jenkins/lsu/env-gcc-12.sh | 1 - .jenkins/lsu/env-gcc-13.sh | 1 - 8 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.jenkins/lsu/env-clang-11.sh b/.jenkins/lsu/env-clang-11.sh index 42c5aea9169e..1fe3ff398735 100644 --- a/.jenkins/lsu/env-clang-11.sh +++ b/.jenkins/lsu/env-clang-11.sh @@ -11,7 +11,6 @@ module load boost/1.73.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" @@ -24,7 +23,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index cea6a999dd14..00a326a21b4a 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -11,7 +11,6 @@ module load boost/1.75.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -25,7 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 9b00cbd75a03..3c95f8df27e4 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -11,7 +11,6 @@ module load boost/1.78.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" @@ -25,7 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 09b9022267f4..4a8d77e2e7df 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -11,7 +11,6 @@ module load boost/1.79.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -25,7 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 27129b473f0e..dd31ffb1e5c0 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -11,7 +11,6 @@ module load boost/1.82.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -# module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" @@ -25,7 +24,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -# configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 4835f03edd2c..876c380f9257 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -11,7 +11,6 @@ module load boost/1.75.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 10d48af9bd60..b04c43fe41ad 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -11,7 +11,6 @@ module load boost/1.80.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index c4b4326b57da..5210c1e60a9f 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -11,7 +11,6 @@ module load boost/1.82.0-${build_type,,} module load hwloc module load openmpi module load pwrapi/1.1.1 -module load gasnet/2023.3.0 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" From 54add8bc6ef28289f076f2d81c43013c372831cb Mon Sep 17 00:00:00 2001 From: Hartmut Kaiser Date: Sat, 30 Sep 2023 16:32:59 -0500 Subject: [PATCH 307/333] Adding conduits to gasnet testers - flyby: more merge cleanup --- .jenkins/lsu/env-clang-11.sh | 36 ------------------- .jenkins/lsu/env-clang-12.sh | 2 ++ .jenkins/lsu/env-clang-13.sh | 2 ++ .jenkins/lsu/env-clang-14.sh | 2 ++ .jenkins/lsu/env-clang-15.sh | 2 ++ .jenkins/lsu/env-gcc-10.sh | 2 ++ .jenkins/lsu/env-gcc-12.sh | 2 ++ .jenkins/lsu/env-gcc-13.sh | 2 ++ .jenkins/lsu/env-gcc-9.sh | 32 ----------------- docs/sphinx/manual/migration_guide.rst | 5 --- docs/sphinx/releases/whats_new_1_10_0.rst | 31 ++++++++++++++++ .../datastructures/tests/unit/bitset_test.hpp | 4 --- 12 files changed, 45 insertions(+), 77 deletions(-) delete mode 100644 .jenkins/lsu/env-clang-11.sh delete mode 100644 .jenkins/lsu/env-gcc-9.sh create mode 100644 docs/sphinx/releases/whats_new_1_10_0.rst diff --git a/.jenkins/lsu/env-clang-11.sh b/.jenkins/lsu/env-clang-11.sh deleted file mode 100644 index 1fe3ff398735..000000000000 --- a/.jenkins/lsu/env-clang-11.sh +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -module purge -module load cmake -module load llvm/11 -module load boost/1.73.0-${build_type,,} -module load hwloc -module load openmpi -module load pwrapi/1.1.1 - -export HPXRUN_RUNWRAPPER=srun -export CXX_STD="17" - -configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" -configure_extra_options+=" -DHPX_WITH_MALLOC=system" -configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" - -# The pwrapi library still needs to be set up properly on rostam -# configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" - -# Make sure HWLOC does not report 'cores'. This is purely an option to enable -# testing the topology code under conditions close to those on FreeBSD. -configure_extra_options+=" -DHPX_TOPOLOGY_WITH_ADDITIONAL_HWLOC_TESTING=ON" - -configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 00a326a21b4a..ebf1dc6e433b 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 3c95f8df27e4..7d972f9f518a 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 4a8d77e2e7df..1d401f3bbc36 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index dd31ffb1e5c0..d759ddefbc4c 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 876c380f9257..f5a12c7c4cd4 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -25,6 +25,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index b04c43fe41ad..95b0b6ba85d4 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -24,6 +24,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 5210c1e60a9f..3067806724fe 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -24,6 +24,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-9.sh b/.jenkins/lsu/env-gcc-9.sh deleted file mode 100644 index abdb272af42a..000000000000 --- a/.jenkins/lsu/env-gcc-9.sh +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2020 ETH Zurich -# -# SPDX-License-Identifier: BSL-1.0 -# Distributed under the Boost Software License, Version 1.0. (See accompanying -# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - -module purge -module load cmake -module load gcc/9 -module load boost/1.73.0-${build_type,,} -module load hwloc -module load openmpi -module load pwrapi/1.1.1 -module load gasnet/2023.3.0 - -export HPXRUN_RUNWRAPPER=srun -export CXX_STD="17" - -configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" -configure_extra_options+=" -DHPX_WITH_MALLOC=system" -configure_extra_options+=" -DHPX_WITH_FETCH_ASIO=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON" -configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" - -# The pwrapi library still needs to be set up properly on rostam -# configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" - -configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON" diff --git a/docs/sphinx/manual/migration_guide.rst b/docs/sphinx/manual/migration_guide.rst index b2a2d9b9d57c..efc391644196 100644 --- a/docs/sphinx/manual/migration_guide.rst +++ b/docs/sphinx/manual/migration_guide.rst @@ -1638,11 +1638,6 @@ detail: MPI_Alltoall ------------- -The following code gathers data from and scatters data to all processes. -======= -MPI_Alltoall -------------- - The following code gathers data from and scatters data to all processes. |mpi| code: diff --git a/docs/sphinx/releases/whats_new_1_10_0.rst b/docs/sphinx/releases/whats_new_1_10_0.rst new file mode 100644 index 000000000000..38ed581aa559 --- /dev/null +++ b/docs/sphinx/releases/whats_new_1_10_0.rst @@ -0,0 +1,31 @@ +.. + Copyright (C) 2007-2023 Hartmut Kaiser + + SPDX-License-Identifier: BSL-1.0 + Distributed under the Boost Software License, Version 1.0. (See accompanying + file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +.. _hpx_1_10_0: + +=========================== +|hpx| V1.10.0 (TBD) +=========================== + +General changes +=============== + +Breaking changes +================ + +- The |cmake| configuration keys ``SOMELIB_ROOT`` (e.g., ``BOOST_ROOT``) have been + renamed to ``Somelib_ROOT`` (e.g., ``Boost_ROOT``) to avoid warnings when using + newer versions of |cmake|. Please update your scripts accordingly. For now, the + old variable names are re-assigned to the new names and unset in the |cmake| + cache. + +Closed issues +============= + +Closed pull requests +==================== + diff --git a/libs/core/datastructures/tests/unit/bitset_test.hpp b/libs/core/datastructures/tests/unit/bitset_test.hpp index 99cb8f4b20e7..e9304192ca4e 100644 --- a/libs/core/datastructures/tests/unit/bitset_test.hpp +++ b/libs/core/datastructures/tests/unit/bitset_test.hpp @@ -320,10 +320,6 @@ struct bitset_test b = std::move(b); // self assignment check #endif -#if defined(HPX_GCC_VERSION) && HPX_GCC_VERSION >= 130000 -#pragma GCC diagnostic pop -#endif - // NOLINTNEXTLINE(bugprone-use-after-move) HPX_TEST(b == rhs); } From bdb906b167494d10f867b96ae543dae243b71820 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sat, 30 Sep 2023 23:17:28 -0400 Subject: [PATCH 308/333] updated with ofi/ucx search when fetch is turned on; removed ibv conduit option --- CMakeLists.txt | 2 +- cmake/HPX_SetupGasnet.cmake | 41 +++++++++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3aadfa157421..d4a76c53cec0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1169,7 +1169,7 @@ if(HPX_WITH_NETWORKING) hpx_option( HPX_WITH_PARCELPORT_GASNET_CONDUIT STRING "Define which conduit to use for the gasnet parcelport" "udp" - STRINGS "smp;udp;mpi;ofi;ucx;ibv" + STRINGS "smp;udp;mpi;ofi;ucx" ) endif() diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 068abf031fb3..3e6d35119737 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -22,26 +22,59 @@ macro(hpx_setup_gasnet) if(NOT CMAKE_C_COMPILER) message( FATAL_ERROR - "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset.}" + "HPX_WITH_FETCH_GASNET requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset." ) endif() if(NOT CMAKE_CXX_COMPILER) message( FATAL_ERROR - "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset.}" + "HPX_WITH_FETCH_GASNET requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset." ) endif() if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi" AND NOT OFI_DIR) message( FATAL_ERROR - "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}" + "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ofi AND HPX_WITH_FETCH_GASNET requires `-DOFI_DIR` to be set; OFI_DIR is currently unset." ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ofi") + pkg_search_module( + OFI + REQUIRED + IMPORTED_TARGET + GLOBAL + libfabric + libfabric-1.5 + libfabric-1.7 + libfabric-1.15 + ) + if(NOT OFI_FOUND) + message( + FATAL_ERROR + "libfabric 1.5, 1.7, or 1.15 was not found. Your `$PKG_CONFIG_PATH` or `-DOFI_DIR` may need to be updated" + ) + endif() endif() if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx" AND NOT UCX_DIR) message( FATAL_ERROR - "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset.}" + "HPX_WITH_PARCELPORT_GASNET_CONDUIT=ucx AND HPX_WITH_FETCH_GASNET requires `-DUCX_DIR` to be set; UCX_DIR is currently unset." ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "ucx") + pkg_search_module( + UCX + REQUIRED + IMPORTED_TARGET + GLOBAL + ucx + ucx-1.14.0 + ucx-1.15.0 + ) + if(NOT UCX_FOUND) + message( + FATAL_ERROR + "UCX 1.14.0, or 1.15.0 was not found. Your `$PKG_CONFIG_PATH` or `-DUCX_DIR` may need to be updated" + ) + endif() endif() message(STATUS "Fetching GASNET") From 3b49ab8b9a64a773e7af91b966f1c5df995e6b1b Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 1 Oct 2023 15:32:29 -0400 Subject: [PATCH 309/333] updated builders for clang 12,13,14,15 and gcc 10,12,13 --- .jenkins/lsu/env-clang-12.sh | 1 + .jenkins/lsu/env-clang-13.sh | 1 + .jenkins/lsu/env-clang-14.sh | 1 + .jenkins/lsu/env-clang-15.sh | 1 + .jenkins/lsu/env-gcc-10.sh | 1 + .jenkins/lsu/env-gcc-12.sh | 1 + .jenkins/lsu/env-gcc-13.sh | 1 + 7 files changed, 7 insertions(+) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index ebf1dc6e433b..f9157dd6ce83 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -26,6 +26,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 7d972f9f518a..4cf064851fec 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -25,6 +25,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 1d401f3bbc36..82db6f64041a 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,6 +27,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index d759ddefbc4c..21b5ded55608 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -27,6 +27,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index f5a12c7c4cd4..862f5658d65c 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -26,6 +26,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 95b0b6ba85d4..095a94e7a81f 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -26,6 +26,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 3067806724fe..ac432e0f7e35 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -26,6 +26,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" From 488715d4573f8645f98a726ed585f510c5814b9e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 1 Oct 2023 18:08:06 -0400 Subject: [PATCH 310/333] updated builders PKG_CONFIG_PATH to point to the correct location; should fix gasnet fetch/build detection issue --- .jenkins/lsu/env-clang-12.sh | 1 + .jenkins/lsu/env-clang-13.sh | 1 + .jenkins/lsu/env-clang-14.sh | 1 + .jenkins/lsu/env-clang-15.sh | 1 + .jenkins/lsu/env-gcc-10.sh | 1 + .jenkins/lsu/env-gcc-12.sh | 1 + .jenkins/lsu/env-gcc-13.sh | 1 + 7 files changed, 7 insertions(+) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index f9157dd6ce83..5dae0cb5c34f 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 4cf064851fec..2de9af710811 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 82db6f64041a..76fbfeb63640 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 21b5ded55608..43c36f57b8e7 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DCMAKE_BUILD_TYPE=${build_type}" configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 862f5658d65c..d15c46bf0d85 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="17" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 095a94e7a81f..cc3fe281c51a 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index ac432e0f7e35..c88f3b1d7bac 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -14,6 +14,7 @@ module load pwrapi/1.1.1 export HPXRUN_RUNWRAPPER=srun export CXX_STD="20" +export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH configure_extra_options+=" -DHPX_WITH_CXX_STANDARD=${CXX_STD}" configure_extra_options+=" -DHPX_WITH_MALLOC=system" From 0d3f086a63b19e45225dc6c88daffbf1dd7ca03e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Sun, 1 Oct 2023 20:34:35 -0400 Subject: [PATCH 311/333] added pkgconfig search path --- cmake/HPX_SetupGasnet.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 3e6d35119737..219ceddb902a 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -11,6 +11,7 @@ macro(hpx_setup_gasnet) if(NOT TARGET PkgConfig::GASNET) find_package(PkgConfig REQUIRED QUIET COMPONENTS) + set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH TRUE) pkg_search_module( GASNET IMPORTED_TARGET GLOBAL @@ -78,6 +79,10 @@ macro(hpx_setup_gasnet) endif() message(STATUS "Fetching GASNET") + + set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + include(FetchContent) fetchcontent_declare( gasnet From 24da04fef98f1ac3ac8dfd4bb8d3fc776e7528a7 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 10:11:59 -0400 Subject: [PATCH 312/333] potential fix for gnu-autotools/gasnet build --- cmake/HPX_SetupGasnet.cmake | 44 ++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 219ceddb902a..b4f815347334 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -109,7 +109,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -119,7 +119,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -129,7 +129,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-FPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${CMAKE_INSTALL_PREFIX} --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -142,6 +142,44 @@ macro(hpx_setup_gasnet) FATAL_ERROR "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details" ) + else() + message(STATUS "INSTALLING GASNET to\t${CMAKE_INSTALL_PREFIX}") + + find_file( + GASNET_PKGCONFIG_FILE_FOUND + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc + HINTS ${GASNET_DIR}/install/lib/pkgconfig + PATHS ${GASNET_DIR}/install/lib/pkgconfig + ) + + if(NOT GASNET_PKGCONFIG_FILE_FOUND) + message(FATAL "ERROR INSTALLING GASNET") + endif() + + file( + READ + ${GASNET_DIR}/install/lib/pkgconfig/gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc + GASNET_PKGCONFIG_FILE_CONTENT + ) + if(NOT GASNET_PKGCONFIG_FILE_CONTENT) + message(FATAL "ERROR INSTALLING GASNET") + endif() + + string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_PKGCONFIG_FILE_CONTENT + ${GASNET_PKGCONFIG_FILE_CONTENT} + ) + + file( + WRITE + ${GASNET_DIR}/install/lib/pkgconfig/gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc + ${GASNET_PKGCONFIG_FILE_CONTENT} + ) + + file(INSTALL ${GASNET_DIR}/install + DESTINATION ${CMAKE_INSTALL_PREFIX}/.. + ) + endif() pkg_search_module( From 3bb425715f1c69b56f12d53627482086f8a2759f Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 16:22:07 -0400 Subject: [PATCH 313/333] several fixes for cmake/autotool support --- cmake/HPX_SetupGasnet.cmake | 55 +++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index b4f815347334..3284f3280320 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -103,13 +103,21 @@ macro(hpx_setup_gasnet) set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") - if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp" - OR "${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp" - ) + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp") + execute_process( + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "smp") execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -143,8 +151,6 @@ macro(hpx_setup_gasnet) "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details" ) else() - message(STATUS "INSTALLING GASNET to\t${CMAKE_INSTALL_PREFIX}") - find_file( GASNET_PKGCONFIG_FILE_FOUND gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc @@ -153,7 +159,7 @@ macro(hpx_setup_gasnet) ) if(NOT GASNET_PKGCONFIG_FILE_FOUND) - message(FATAL "ERROR INSTALLING GASNET") + message(FATAL_ERROR "ERROR INSTALLING GASNET") endif() file( @@ -162,7 +168,7 @@ macro(hpx_setup_gasnet) GASNET_PKGCONFIG_FILE_CONTENT ) if(NOT GASNET_PKGCONFIG_FILE_CONTENT) - message(FATAL "ERROR INSTALLING GASNET") + message(FATAL_ERROR "ERROR INSTALLING GASNET") endif() string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" @@ -176,10 +182,37 @@ macro(hpx_setup_gasnet) ${GASNET_PKGCONFIG_FILE_CONTENT} ) - file(INSTALL ${GASNET_DIR}/install - DESTINATION ${CMAKE_INSTALL_PREFIX}/.. - ) + file(GLOB_RECURSE GASNET_FILES ${GASNET_DIR}/install/*) + + if(NOT GASNET_FILES) + message(STATUS "ERROR INSTALLING GASNET") + endif() + + foreach(GASNET_FILE ${GASNET_FILES}) + set(GASNET_FILE_CACHED "${GASNET_FILE}") + string(REGEX + MATCH "(^\/.*\/)" + GASNET_FILE_PATH ${GASNET_FILE}) + + string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_FILE + ${GASNET_FILE}) + string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_FILE_PATH + ${GASNET_FILE_PATH}) + + file(MAKE_DIRECTORY ${GASNET_FILE_PATH}) + + string(LENGTH ${GASNET_FILE_PATH} GASNET_FILE_PATH_SIZE) + MATH(EXPR GASNET_FILE_PATH_SIZE "${GASNET_FILE_PATH_SIZE}-1") + + string(SUBSTRING ${GASNET_FILE_PATH} 0 ${GASNET_FILE_PATH_SIZE} GASNET_FILE_PATH) + + file(COPY ${GASNET_FILE_CACHED} + DESTINATION ${GASNET_FILE_PATH} + ) + endforeach() endif() pkg_search_module( From 20b1540d35b4315817130c2f47a4f3ccf67df48e Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 16:24:03 -0400 Subject: [PATCH 314/333] several fixes for cmake/autotool support --- cmake/HPX_SetupGasnet.cmake | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 3284f3280320..4c1bc9bb2129 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -190,28 +190,26 @@ macro(hpx_setup_gasnet) foreach(GASNET_FILE ${GASNET_FILES}) set(GASNET_FILE_CACHED "${GASNET_FILE}") - string(REGEX - MATCH "(^\/.*\/)" - GASNET_FILE_PATH ${GASNET_FILE}) + string(REGEX MATCH "(^\/.*\/)" GASNET_FILE_PATH ${GASNET_FILE}) string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" - GASNET_FILE - ${GASNET_FILE}) + GASNET_FILE ${GASNET_FILE} + ) string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" - GASNET_FILE_PATH - ${GASNET_FILE_PATH}) + GASNET_FILE_PATH ${GASNET_FILE_PATH} + ) file(MAKE_DIRECTORY ${GASNET_FILE_PATH}) string(LENGTH ${GASNET_FILE_PATH} GASNET_FILE_PATH_SIZE) - MATH(EXPR GASNET_FILE_PATH_SIZE "${GASNET_FILE_PATH_SIZE}-1") + math(EXPR GASNET_FILE_PATH_SIZE "${GASNET_FILE_PATH_SIZE}-1") - string(SUBSTRING ${GASNET_FILE_PATH} 0 ${GASNET_FILE_PATH_SIZE} GASNET_FILE_PATH) - - file(COPY ${GASNET_FILE_CACHED} - DESTINATION ${GASNET_FILE_PATH} + string(SUBSTRING ${GASNET_FILE_PATH} 0 ${GASNET_FILE_PATH_SIZE} + GASNET_FILE_PATH ) + + file(COPY ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) endforeach() endif() From ce070693ac6290aa4d52713976e5c47d93216486 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 16:54:57 -0400 Subject: [PATCH 315/333] several fixes for cmake/autotool support --- cmake/HPX_SetupGasnet.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 4c1bc9bb2129..d49731037f10 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -159,7 +159,7 @@ macro(hpx_setup_gasnet) ) if(NOT GASNET_PKGCONFIG_FILE_FOUND) - message(FATAL_ERROR "ERROR INSTALLING GASNET") + message(FATAL_ERROR "PKG-CONFIG ERROR (${GASNET_PKGCONFIG_FILE_FOUND}); CANNOT FIND COMPILED GASNET: ${GASNET_DIR}/install/lib/pkgconfig") endif() file( @@ -209,7 +209,7 @@ macro(hpx_setup_gasnet) GASNET_FILE_PATH ) - file(COPY ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) + file(INSTALL ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) endforeach() endif() From d6c01acd22d28058c5509eec2e9ccbb0434fdf97 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 17:16:10 -0400 Subject: [PATCH 316/333] several fixes for cmake/autotool support --- cmake/HPX_SetupGasnet.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index d49731037f10..2bb9be79ba7a 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -151,6 +151,9 @@ macro(hpx_setup_gasnet) "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details" ) else() + set(CMAKE_PREFIX_PATH "${GASNET_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${GASNET_DIR}/install/lib/pkgconfig") + find_file( GASNET_PKGCONFIG_FILE_FOUND gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc @@ -213,6 +216,9 @@ macro(hpx_setup_gasnet) endforeach() endif() + set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + pkg_search_module( GASNET REQUIRED IMPORTED_TARGET GLOBAL gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par From 3e41ea637b9bf4d3b6a82342abe1910f9328a930 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 21:11:19 -0400 Subject: [PATCH 317/333] modified file search --- cmake/HPX_SetupGasnet.cmake | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 2bb9be79ba7a..0ac770af5d35 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -151,18 +151,15 @@ macro(hpx_setup_gasnet) "GASNet build result = ${GASNET_BUILD_STATUS} - see ${GASNET_BUILD_OUTPUT} for more details" ) else() - set(CMAKE_PREFIX_PATH "${GASNET_DIR}/install/lib/pkgconfig") - set(ENV{PKG_CONFIG_PATH} "${GASNET_DIR}/install/lib/pkgconfig") find_file( GASNET_PKGCONFIG_FILE_FOUND gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc - HINTS ${GASNET_DIR}/install/lib/pkgconfig - PATHS ${GASNET_DIR}/install/lib/pkgconfig + ${GASNET_DIR}/install/lib/pkgconfig ) if(NOT GASNET_PKGCONFIG_FILE_FOUND) - message(FATAL_ERROR "PKG-CONFIG ERROR (${GASNET_PKGCONFIG_FILE_FOUND}); CANNOT FIND COMPILED GASNET: ${GASNET_DIR}/install/lib/pkgconfig") + message(FATAL_ERROR "PKG-CONFIG ERROR (${GASNET_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED GASNET: ${GASNET_DIR}/install/lib/pkgconfig") endif() file( @@ -170,6 +167,7 @@ macro(hpx_setup_gasnet) ${GASNET_DIR}/install/lib/pkgconfig/gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc GASNET_PKGCONFIG_FILE_CONTENT ) + if(NOT GASNET_PKGCONFIG_FILE_CONTENT) message(FATAL_ERROR "ERROR INSTALLING GASNET") endif() @@ -212,7 +210,7 @@ macro(hpx_setup_gasnet) GASNET_FILE_PATH ) - file(INSTALL ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) + file(COPY ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) endforeach() endif() From d678302ad2fcdf425d16d7e546f514a6c0053709 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 4 Oct 2023 21:11:58 -0400 Subject: [PATCH 318/333] modified file search --- cmake/HPX_SetupGasnet.cmake | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 0ac770af5d35..cddfaf231648 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -152,14 +152,16 @@ macro(hpx_setup_gasnet) ) else() - find_file( - GASNET_PKGCONFIG_FILE_FOUND - gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc - ${GASNET_DIR}/install/lib/pkgconfig + find_file(GASNET_PKGCONFIG_FILE_FOUND + gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc + ${GASNET_DIR}/install/lib/pkgconfig ) if(NOT GASNET_PKGCONFIG_FILE_FOUND) - message(FATAL_ERROR "PKG-CONFIG ERROR (${GASNET_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED GASNET: ${GASNET_DIR}/install/lib/pkgconfig") + message( + FATAL_ERROR + "PKG-CONFIG ERROR (${GASNET_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED GASNET: ${GASNET_DIR}/install/lib/pkgconfig" + ) endif() file( From 607dba68d8c885e1d2a8878d371b0ecb81ae6ad3 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 9 Oct 2023 21:34:38 -0400 Subject: [PATCH 319/333] modifications for builder --- cmake/HPX_SetupGasnet.cmake | 153 ++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 52 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index cddfaf231648..0f1b30aad222 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -143,6 +143,30 @@ macro(hpx_setup_gasnet) OUTPUT_FILE ${GASNET_BUILD_OUTPUT} ERROR_FILE ${GASNET_ERROR_FILE} ) + elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") + if(NOT MPI_FOUND) + message( + FATAL_ERROR + "GASNet MPI Conduit selected; MPI not found!" + ) + endif() + + if(NOT TARGET Mpi::mpi) + message( + FATAL_ERROR + "GASNet MPI Conduit selected; MPI not found!" + ) + endif() + + execute_process( + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${MPI_C_COMPILER} --with-mpi-cflags=${MPI_C_COMPILER_FLAGS} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + WORKING_DIRECTORY ${GASNET_DIR} + RESULT_VARIABLE GASNET_BUILD_STATUS + OUTPUT_FILE ${GASNET_BUILD_OUTPUT} + ERROR_FILE ${GASNET_ERROR_FILE} + ) endif() if(GASNET_BUILD_STATUS) @@ -164,60 +188,68 @@ macro(hpx_setup_gasnet) ) endif() - file( - READ - ${GASNET_DIR}/install/lib/pkgconfig/gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc - GASNET_PKGCONFIG_FILE_CONTENT - ) - - if(NOT GASNET_PKGCONFIG_FILE_CONTENT) - message(FATAL_ERROR "ERROR INSTALLING GASNET") - endif() - - string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" - GASNET_PKGCONFIG_FILE_CONTENT - ${GASNET_PKGCONFIG_FILE_CONTENT} - ) + install(CODE "set(GASNET_CONDUIT \"${HPX_WITH_PARCELPORT_GASNET_CONDUIT}\")") + install(CODE "set(GASNET_PATH \"${GASNET_DIR}\")") - file( - WRITE - ${GASNET_DIR}/install/lib/pkgconfig/gasnet-${HPX_WITH_PARCELPORT_GASNET_CONDUIT}-par.pc - ${GASNET_PKGCONFIG_FILE_CONTENT} - ) - - file(GLOB_RECURSE GASNET_FILES ${GASNET_DIR}/install/*) - - if(NOT GASNET_FILES) - message(STATUS "ERROR INSTALLING GASNET") - endif() + install(CODE [[ + file( + READ + ${GASNET_PATH}/install/lib/pkgconfig/gasnet-${GASNET_CONDUIT}-par.pc + GASNET_PKGCONFIG_FILE_CONTENT + ) - foreach(GASNET_FILE ${GASNET_FILES}) - set(GASNET_FILE_CACHED "${GASNET_FILE}") - string(REGEX MATCH "(^\/.*\/)" GASNET_FILE_PATH ${GASNET_FILE}) + if(NOT GASNET_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR INSTALLING GASNET") + endif() - string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" - GASNET_FILE ${GASNET_FILE} + string(REPLACE "${GASNET_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_PKGCONFIG_FILE_CONTENT + ${GASNET_PKGCONFIG_FILE_CONTENT} ) - string(REPLACE "${GASNET_DIR}/install" "${CMAKE_INSTALL_PREFIX}" - GASNET_FILE_PATH ${GASNET_FILE_PATH} + file( + WRITE + ${GASNET_PATH}/install/lib/pkgconfig/gasnet-${GASNET_CONDUIT}-par.pc + ${GASNET_PKGCONFIG_FILE_CONTENT} ) - file(MAKE_DIRECTORY ${GASNET_FILE_PATH}) + file(GLOB_RECURSE GASNET_FILES ${GASNET_PATH}/install/*) - string(LENGTH ${GASNET_FILE_PATH} GASNET_FILE_PATH_SIZE) - math(EXPR GASNET_FILE_PATH_SIZE "${GASNET_FILE_PATH_SIZE}-1") + if(NOT GASNET_FILES) + message(STATUS "ERROR INSTALLING GASNET") + endif() - string(SUBSTRING ${GASNET_FILE_PATH} 0 ${GASNET_FILE_PATH_SIZE} - GASNET_FILE_PATH - ) + foreach(GASNET_FILE ${GASNET_FILES}) + set(GASNET_FILE_CACHED "${GASNET_FILE}") + + string(REGEX MATCH "(^\/.*\/)" GASNET_FILE_PATH ${GASNET_FILE}) + + string(REPLACE "${GASNET_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_FILE ${GASNET_FILE} + ) + + string(REPLACE "${GASNET_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + GASNET_FILE_PATH ${GASNET_FILE_PATH} + ) + + file(MAKE_DIRECTORY ${GASNET_FILE_PATH}) + + string(LENGTH ${GASNET_FILE_PATH} GASNET_FILE_PATH_SIZE) + math(EXPR GASNET_FILE_PATH_SIZE "${GASNET_FILE_PATH_SIZE}-1") + + string(SUBSTRING ${GASNET_FILE_PATH} 0 ${GASNET_FILE_PATH_SIZE} + GASNET_FILE_PATH + ) + + file(COPY ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) + endforeach() + ]]) - file(COPY ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) - endforeach() + #install(FILES ${GASNET_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) endif() - set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") - set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + set(CMAKE_PREFIX_PATH "${GASNET_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${GASNET_DIR}/install/lib/pkgconfig") pkg_search_module( GASNET REQUIRED IMPORTED_TARGET GLOBAL @@ -864,16 +896,33 @@ macro(hpx_setup_gasnet) endif() endif() - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" - ) - set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES - "${GASNET_LIBRARY_DIRS}" - ) + if(GASNET_DIR) + list(TRANSFORM GASNET_CFLAGS REPLACE "${GASNET_DIR}/install" "$") + list(TRANSFORM GASNET_LDFLAGS REPLACE "${GASNET_DIR}/install" "$") + list(TRANSFORM GASNET_LIBRARY_DIRS REPLACE "${GASNET_DIR}/install" "$") + + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" + ) + else() + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" + ) + set_target_properties( + PkgConfig::GASNET PROPERTIES INTERFACE_LINK_DIRECTORIES + "${GASNET_LIBRARY_DIRS}" + ) + endif() endif() From 95bd6d8466187f79ccf62dc6378f3dc4cb8eb8cc Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 9 Oct 2023 21:36:03 -0400 Subject: [PATCH 320/333] modifications for builder --- cmake/HPX_SetupGasnet.cmake | 51 ++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 0f1b30aad222..abe557f6c5b3 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -145,17 +145,11 @@ macro(hpx_setup_gasnet) ) elseif("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "mpi") if(NOT MPI_FOUND) - message( - FATAL_ERROR - "GASNet MPI Conduit selected; MPI not found!" - ) + message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") endif() if(NOT TARGET Mpi::mpi) - message( - FATAL_ERROR - "GASNet MPI Conduit selected; MPI not found!" - ) + message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") endif() execute_process( @@ -188,10 +182,13 @@ macro(hpx_setup_gasnet) ) endif() - install(CODE "set(GASNET_CONDUIT \"${HPX_WITH_PARCELPORT_GASNET_CONDUIT}\")") + install( + CODE "set(GASNET_CONDUIT \"${HPX_WITH_PARCELPORT_GASNET_CONDUIT}\")" + ) install(CODE "set(GASNET_PATH \"${GASNET_DIR}\")") - install(CODE [[ + install( + CODE [[ file( READ ${GASNET_PATH}/install/lib/pkgconfig/gasnet-${GASNET_CONDUIT}-par.pc @@ -231,21 +228,22 @@ macro(hpx_setup_gasnet) string(REPLACE "${GASNET_PATH}/install" "${CMAKE_INSTALL_PREFIX}" GASNET_FILE_PATH ${GASNET_FILE_PATH} ) - + file(MAKE_DIRECTORY ${GASNET_FILE_PATH}) - + string(LENGTH ${GASNET_FILE_PATH} GASNET_FILE_PATH_SIZE) math(EXPR GASNET_FILE_PATH_SIZE "${GASNET_FILE_PATH_SIZE}-1") - + string(SUBSTRING ${GASNET_FILE_PATH} 0 ${GASNET_FILE_PATH_SIZE} GASNET_FILE_PATH ) - + file(COPY ${GASNET_FILE_CACHED} DESTINATION ${GASNET_FILE_PATH}) endforeach() - ]]) + ]] + ) - #install(FILES ${GASNET_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) + # install(FILES ${GASNET_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) endif() set(CMAKE_PREFIX_PATH "${GASNET_DIR}/install/lib/pkgconfig") @@ -897,12 +895,22 @@ macro(hpx_setup_gasnet) endif() if(GASNET_DIR) - list(TRANSFORM GASNET_CFLAGS REPLACE "${GASNET_DIR}/install" "$") - list(TRANSFORM GASNET_LDFLAGS REPLACE "${GASNET_DIR}/install" "$") - list(TRANSFORM GASNET_LIBRARY_DIRS REPLACE "${GASNET_DIR}/install" "$") + list(TRANSFORM GASNET_CFLAGS + REPLACE "${GASNET_DIR}/install" + "$" + ) + list(TRANSFORM GASNET_LDFLAGS + REPLACE "${GASNET_DIR}/install" + "$" + ) + list(TRANSFORM GASNET_LIBRARY_DIRS + REPLACE "${GASNET_DIR}/install" + "$" + ) set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS + "${GASNET_CFLAGS}" ) set_target_properties( PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" @@ -913,7 +921,8 @@ macro(hpx_setup_gasnet) ) else() set_target_properties( - PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS "${GASNET_CFLAGS}" + PkgConfig::GASNET PROPERTIES INTERFACE_COMPILE_OPTIONS + "${GASNET_CFLAGS}" ) set_target_properties( PkgConfig::GASNET PROPERTIES INTERFACE_LINK_OPTIONS "${GASNET_LDFLAGS}" From a839e2a19b00237a76f6e10e2ef094343b107619 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 10 Oct 2023 10:27:12 -0400 Subject: [PATCH 321/333] updated build scripts and environment variables --- .jenkins/lsu/env-clang-12.sh | 2 +- .jenkins/lsu/env-clang-14.sh | 2 +- .jenkins/lsu/env-clang-15.sh | 2 +- .jenkins/lsu/env-gcc-12.sh | 2 +- .jenkins/lsu/env-gcc-13.sh | 2 +- tests/unit/build/CMakeLists.txt | 5 ++++- 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 5dae0cb5c34f..fb4b94c48406 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -28,7 +28,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 76fbfeb63640..fe58b0120ff3 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,7 +27,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 43c36f57b8e7..0d15f52290ad 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -27,7 +27,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index cc3fe281c51a..7649ab4eb3ea 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -26,7 +26,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index c88f3b1d7bac..c5c8308cf0ab 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -26,7 +26,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/tests/unit/build/CMakeLists.txt b/tests/unit/build/CMakeLists.txt index ed2d68db1a5e..038e5066c1aa 100644 --- a/tests/unit/build/CMakeLists.txt +++ b/tests/unit/build/CMakeLists.txt @@ -42,6 +42,9 @@ function( set(ADDITIONAL_CMAKE_OPTIONS ${ADDITIONAL_CMAKE_OPTIONS} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} ) + set(ADDITIONAL_CMAKE_OPTIONS ${ADDITIONAL_CMAKE_OPTIONS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + ) else() set(ADDITIONAL_CMAKE_OPTIONS ${ADDITIONAL_CMAKE_OPTIONS} @@ -86,7 +89,7 @@ function(create_pkgconfig_test name hpx_dir) "${CMAKE_COMMAND}" -E chdir "${build_dir}" make -f "${PROJECT_SOURCE_DIR}/examples/hello_world_component/Makefile" SRC_DIR=${PROJECT_SOURCE_DIR}/examples/hello_world_component - HPX_DIR=${hpx_dir} CXX=${CMAKE_CXX_COMPILER} + HPX_DIR=${hpx_dir} CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} CXX_FLAGS=${CMAKE_CXX_FLAGS_SAFE} PKG_CONFIG=${PKG_CONFIG_EXECUTABLE} BUILD_TYPE=$ VERBATIM From 080937d0a36ef3cbfec5bd97f596d2c2ad6dabe0 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 10 Oct 2023 12:14:23 -0400 Subject: [PATCH 322/333] fixes for linking issues in ctest/builder scripts --- .jenkins/lsu/env-clang-12.sh | 1 + .jenkins/lsu/env-clang-13.sh | 3 ++- .jenkins/lsu/env-clang-14.sh | 3 ++- .jenkins/lsu/env-clang-15.sh | 3 ++- .jenkins/lsu/env-gcc-10.sh | 1 + .jenkins/lsu/env-gcc-12.sh | 3 ++- .jenkins/lsu/env-gcc-13.sh | 3 ++- tests/unit/build/CMakeLists.txt | 1 + 8 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index fb4b94c48406..5364e4f18315 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -28,6 +28,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 2de9af710811..ad7ce0b8cb10 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -26,8 +26,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DCMAKE_C_COMPILER=clang" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index fe58b0120ff3..6b9de5ca3074 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,8 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 0d15f52290ad..17e3111d33c3 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -27,8 +27,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index d15c46bf0d85..753ce7afca7d 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -28,6 +28,7 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 7649ab4eb3ea..a9acf8aa4649 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -26,8 +26,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index c5c8308cf0ab..636f5d78690a 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -26,8 +26,9 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" +configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/tests/unit/build/CMakeLists.txt b/tests/unit/build/CMakeLists.txt index 038e5066c1aa..c4ef5e855637 100644 --- a/tests/unit/build/CMakeLists.txt +++ b/tests/unit/build/CMakeLists.txt @@ -90,6 +90,7 @@ function(create_pkgconfig_test name hpx_dir) "${PROJECT_SOURCE_DIR}/examples/hello_world_component/Makefile" SRC_DIR=${PROJECT_SOURCE_DIR}/examples/hello_world_component HPX_DIR=${hpx_dir} CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} + CFLAGS=-fPIC CCFLAGS=-fPIC C_FLAGS=-fPIC CC_FLAGS=-fPIC CXX_FLAGS=${CMAKE_CXX_FLAGS_SAFE} PKG_CONFIG=${PKG_CONFIG_EXECUTABLE} BUILD_TYPE=$ VERBATIM From cf7e8855a5408903cf29ebd31ac91e1473aff3a4 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 10 Oct 2023 12:18:34 -0400 Subject: [PATCH 323/333] reformatted unit test cmakefile --- tests/unit/build/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/build/CMakeLists.txt b/tests/unit/build/CMakeLists.txt index c4ef5e855637..0cd13feb543b 100644 --- a/tests/unit/build/CMakeLists.txt +++ b/tests/unit/build/CMakeLists.txt @@ -90,7 +90,7 @@ function(create_pkgconfig_test name hpx_dir) "${PROJECT_SOURCE_DIR}/examples/hello_world_component/Makefile" SRC_DIR=${PROJECT_SOURCE_DIR}/examples/hello_world_component HPX_DIR=${hpx_dir} CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} - CFLAGS=-fPIC CCFLAGS=-fPIC C_FLAGS=-fPIC CC_FLAGS=-fPIC + CFLAGS=-fPIC CCFLAGS=-fPIC C_FLAGS=-fPIC CC_FLAGS=-fPIC CXX_FLAGS=${CMAKE_CXX_FLAGS_SAFE} PKG_CONFIG=${PKG_CONFIG_EXECUTABLE} BUILD_TYPE=$ VERBATIM From c6e640c8e229054bd7c407de307877d54375712f Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 10 Oct 2023 14:04:24 -0400 Subject: [PATCH 324/333] attempt to correct linking issue in builder/ctest --- tests/unit/build/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/build/CMakeLists.txt b/tests/unit/build/CMakeLists.txt index 0cd13feb543b..038e5066c1aa 100644 --- a/tests/unit/build/CMakeLists.txt +++ b/tests/unit/build/CMakeLists.txt @@ -90,7 +90,6 @@ function(create_pkgconfig_test name hpx_dir) "${PROJECT_SOURCE_DIR}/examples/hello_world_component/Makefile" SRC_DIR=${PROJECT_SOURCE_DIR}/examples/hello_world_component HPX_DIR=${hpx_dir} CXX=${CMAKE_CXX_COMPILER} CC=${CMAKE_C_COMPILER} - CFLAGS=-fPIC CCFLAGS=-fPIC C_FLAGS=-fPIC CC_FLAGS=-fPIC CXX_FLAGS=${CMAKE_CXX_FLAGS_SAFE} PKG_CONFIG=${PKG_CONFIG_EXECUTABLE} BUILD_TYPE=$ VERBATIM From eb169ea3b8e56d0aa1113c89a8ad58135dd657c8 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 10 Oct 2023 14:57:19 -0400 Subject: [PATCH 325/333] changing builders to test against udp conduit --- .jenkins/lsu/env-clang-12.sh | 2 +- .jenkins/lsu/env-clang-13.sh | 2 +- .jenkins/lsu/env-clang-14.sh | 2 +- .jenkins/lsu/env-clang-15.sh | 2 +- .jenkins/lsu/env-gcc-10.sh | 2 +- .jenkins/lsu/env-gcc-12.sh | 2 +- .jenkins/lsu/env-gcc-13.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 5364e4f18315..d793c8105772 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index ad7ce0b8cb10..a1b32010c6cb 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 6b9de5ca3074..277e6aef1736 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 17e3111d33c3..41bbf2ad3478 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 753ce7afca7d..ac725902dd0e 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index a9acf8aa4649..4d6144d08813 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -28,7 +28,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 636f5d78690a..c93a741d047f 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -28,7 +28,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=mpi" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" From 8efadc10190eca7e97ff115e615eceec364ad5b6 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Tue, 10 Oct 2023 15:28:43 -0400 Subject: [PATCH 326/333] changing builders to test against udp conduit --- .jenkins/lsu/env-clang-12.sh | 2 +- .jenkins/lsu/env-clang-13.sh | 2 +- .jenkins/lsu/env-clang-14.sh | 2 +- .jenkins/lsu/env-clang-15.sh | 2 +- .jenkins/lsu/env-gcc-10.sh | 2 +- .jenkins/lsu/env-gcc-12.sh | 2 +- .jenkins/lsu/env-gcc-13.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index d793c8105772..15e920431f54 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index a1b32010c6cb..4582dd0dba9f 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 277e6aef1736..5a52f3c2dca5 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index 41bbf2ad3478..ccb1e962fd0b 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index ac725902dd0e..3953fae9420a 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -29,7 +29,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 4d6144d08813..1ae1a867f932 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -28,7 +28,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index c93a741d047f..1454588eec09 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -28,7 +28,7 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=udp" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" From cc0240b6d8ef7174f1d3e009d7c6fa529321f832 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Thu, 12 Oct 2023 15:42:31 -0400 Subject: [PATCH 327/333] added 'maybe unused' --- libs/core/gasnet_base/src/gasnet_environment.cpp | 11 +++++------ .../include/hpx/parcelport_gasnet/sender.hpp | 4 +--- libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 97f3c38f50d5..1f7ca03cfa36 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -115,8 +115,8 @@ typedef struct volatile int flag; } done_t; -static void AM_signal( - gasnet_token_t token, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) +static void AM_signal([[maybe_unused]] gasnet_token_t token, + gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { done_t* done = reinterpret_cast(get_ptr_from_args(a0, a1)); uint_least32_t prev; @@ -125,7 +125,8 @@ static void AM_signal( done->flag = 1; } -static void AM_signal_long(gasnet_token_t token, void* buf, size_t nbytes, +static void AM_signal_long([[maybe_unused]] gasnet_token_t token, + [[maybe_unused]] void* buf, [[maybe_unused]] size_t nbytes, gasnet_handlerarg_t a0, gasnet_handlerarg_t a1) { done_t* done = reinterpret_cast(get_ptr_from_args(a0, a1)); @@ -292,7 +293,7 @@ namespace hpx::util { /////////////////////////////////////////////////////////////////////////// int gasnet_environment::init(int* argc, char*** argv, const int minimal, - const int required, int& provided) + [[maybe_unused]] const int required, int& provided) { if (!has_called_init_) { @@ -349,8 +350,6 @@ namespace hpx::util { GASNET_Safe(gasnet_getSegmentInfo(segments, size())); - int retval; - gasnet_barrier_notify(0, GASNET_BARRIERFLAG_ANONYMOUS); gasnet_barrier_wait(0, GASNET_BARRIERFLAG_ANONYMOUS); diff --git a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp index 40e04bbe207e..ffc5abd3de5c 100644 --- a/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp +++ b/libs/full/parcelport_gasnet/include/hpx/parcelport_gasnet/sender.hpp @@ -37,8 +37,7 @@ namespace hpx::parcelset::policies::gasnet { // different versions of clang-format disagree // clang-format off sender() noexcept - : next_free_tag_request_(-1) - , next_free_tag_(-1) + : next_free_tag_(-1) { } // clang-format on @@ -157,7 +156,6 @@ namespace hpx::parcelset::policies::gasnet { connection_list connections_; hpx::spinlock next_free_tag_mtx_; - int next_free_tag_request_; int next_free_tag_; }; diff --git a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp index dee35f5af8af..a4f2377497bd 100644 --- a/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp +++ b/libs/full/parcelport_gasnet/src/parcelport_gasnet.cpp @@ -104,7 +104,7 @@ namespace hpx::parcelset { } static std::size_t background_threads( - util::runtime_configuration const& ini) + [[maybe_unused]] util::runtime_configuration const& ini) { /* return hpx::util::get_entry_as(ini, From 0f875bf67933bfe02247fdbc44fd219a8babcdb8 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 13 Oct 2023 10:31:00 -0400 Subject: [PATCH 328/333] fixed MPI_CC detection issue --- cmake/HPX_SetupGasnet.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index abe557f6c5b3..6f1c8420cf48 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -152,6 +152,12 @@ macro(hpx_setup_gasnet) message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") endif() + if(NOT ${MPI_C_COMPILER}) + set(CMAKE_C_COMPILER ${MPI_C_COMPILER}) + else() + message(FATAL_ERROR "GASNet MPI Conduit selected; $MPI_CC not found!") + endif() + execute_process( COMMAND bash -c From 4f1a3d971278c346d5c0bee508a646a9b120721c Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 13 Oct 2023 11:32:21 -0400 Subject: [PATCH 329/333] added maybe_unused --- libs/core/gasnet_base/src/gasnet_environment.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 1f7ca03cfa36..54b9055152fa 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -140,7 +140,8 @@ static void AM_signal_long([[maybe_unused]] gasnet_token_t token, // arg->dst (which is local to the caller of this AM). // nbytes is < gasnet_AMMaxLongReply here (see chpl_comm_get). // -static void AM_reply_put(gasnet_token_t token, void* buf, size_t nbytes) +static void AM_reply_put( + gasnet_token_t token, void* buf, [[maybe_unused]] size_t nbytes) { xfer_info_t* x = static_cast(buf); From cd8ed998f9f237fbab88e918f0069758b69f0301 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 13 Oct 2023 11:33:27 -0400 Subject: [PATCH 330/333] added maybe_unused --- libs/core/gasnet_base/src/gasnet_environment.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 54b9055152fa..5c09db35185a 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -170,7 +170,8 @@ static void AM_copy_payload(gasnet_token_t token, void* buf, size_t nbytes, GASNET_Safe(gasnet_AMReplyShort2(token, SIGNAL, ack0, ack1)); } -static gasnet_handlerentry_t ftable[] = {{SIGNAL, (void (*)()) & AM_signal}, +[[maybe_unused]] static gasnet_handlerentry_t ftable[] = { + {SIGNAL, (void (*)()) & AM_signal}, {SIGNAL_LONG, (void (*)()) & AM_signal_long}, {DO_REPLY_PUT, (void (*)()) & AM_reply_put}, {DO_COPY_PAYLOAD, (void (*)()) & AM_copy_payload}}; From e25d25a4f4c837958dcfdbc9a8b4a4acbca0cd99 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 13 Oct 2023 12:52:06 -0400 Subject: [PATCH 331/333] possible mpi_cc fix --- cmake/HPX_SetupGasnet.cmake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 6f1c8420cf48..2914e0130b60 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -152,8 +152,12 @@ macro(hpx_setup_gasnet) message(FATAL_ERROR "GASNet MPI Conduit selected; MPI not found!") endif() - if(NOT ${MPI_C_COMPILER}) + if(${MPI_C_COMPILER}) set(CMAKE_C_COMPILER ${MPI_C_COMPILER}) + elseif(${MPI_CC}) + set(CMAKE_C_COMPILER ${MPI_CC}) + elseif(ENV{MPI_CC}) + set(CMAKE_C_COMPILER ENV{MPI_CC}) else() message(FATAL_ERROR "GASNet MPI Conduit selected; $MPI_CC not found!") endif() @@ -161,7 +165,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${MPI_C_COMPILER} --with-mpi-cflags=${MPI_C_COMPILER_FLAGS} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} From 13d5de450d4addbecc7cd7ec0a5552966ac885f3 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 13 Oct 2023 15:09:51 -0400 Subject: [PATCH 332/333] possible testscript fix --- cmake/HPX_AddTest.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index abacd140f062..f666ec8c6adf 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -91,8 +91,14 @@ function(add_hpx_test category name) set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) endif() + set(ENV_VAR "") + if(HPX_WITH_PARCELPORT_GASNET) + set(ENV_VAR "GASNET_PSHM_NODES=2") + endif() + # cmake-format: off set(cmd + ${ENV_VAR} "${Python_EXECUTABLE}" "${_script_location}/bin/hpxrun.py" ${CMAKE_CROSSCOMPILING_EMULATOR} From 8e457ddb276fb6f5fd1f73e079df6ae0ab9a63ac Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 13 Oct 2023 15:37:11 -0400 Subject: [PATCH 333/333] cmake format issue resolved --- cmake/HPX_AddTest.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index f666ec8c6adf..863c003fea9e 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -93,7 +93,7 @@ function(add_hpx_test category name) set(ENV_VAR "") if(HPX_WITH_PARCELPORT_GASNET) - set(ENV_VAR "GASNET_PSHM_NODES=2") + set(ENV_VAR "GASNET_PSHM_NODES=2") endif() # cmake-format: off