Skip to content

Commit

Permalink
Merge #6363
Browse files Browse the repository at this point in the history 
6363: Update the lci parcelport to use LCI v1.7.6 r=hkaiser a=JiakunYan

The major changes include:
- Necessary changes to CMake files to accommodate changes due to the split of the original LCI library into two libraries (LCI and LCT).
- A new performance counters and logging infrastructure based on LCT (currently only applied to the lci pp).
  - Controlled by CMake variables `HPX_WITH_PARCELPORT_LCI_PCOUNTER` and `HPX_WITH_PARCELPORT_LCI_LOG`. The default is `OFF`.
- New LCI parcelport configurations:
  - `reg_mem`: whether to explicitly register memory buffers for long messages (value 1) or just let LCI register them on the fly (value 0). The default is 1.
  - `ndevices`: how many LCI devices (low-level network contexts) to use. The default is 2.
  - `ncomps`: how many completion managers to use. The default is 1.

Co-authored-by: Jiakun Yan <[email protected]>
  • Loading branch information
@JiakunYan
StellarBot and JiakunYan committed Oct 16, 2023
2 parents a0c6d49 + 67dd637 commit 53e9a95
Show file tree
Hide file tree
Showing 22 changed files with 695 additions and 277 deletions.
20 changes: 19 additions & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1147,10 +1147,28 @@ if(HPX_WITH_NETWORKING)
ADVANCED
)
hpx_option(
HPX_WITH_LCI_TAG STRING "LCI repository tag or branch" "v1.7.5"
HPX_WITH_LCI_TAG STRING "LCI repository tag or branch" "v1.7.6"
CATEGORY "Build Targets"
ADVANCED
)
hpx_option(
HPX_WITH_PARCELPORT_LCI_LOG STRING
"Enable the LCI-parcelport-specific logger" OFF
CATEGORY "Parcelport"
ADVANCED
)
if(HPX_WITH_PARCELPORT_LCI_LOG)
hpx_add_config_define(HPX_HAVE_PARCELPORT_LCI_LOG)
endif()
hpx_option(
HPX_WITH_PARCELPORT_LCI_PCOUNTER STRING
"Enable the LCI-parcelport-specific performance counter" OFF
CATEGORY "Parcelport"
ADVANCED
)
if(HPX_WITH_PARCELPORT_LCI_PCOUNTER)
hpx_add_config_define(HPX_HAVE_PARCELPORT_LCI_PCOUNTER)
endif()

hpx_option(
HPX_WITH_PARCELPORT_GASNET BOOL "Enable the GASNET based parcelport." OFF
Expand Down
7 changes: 4 additions & 3 deletions cmake/HPX_SetupLCI.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,23 +87,24 @@ macro(hpx_setup_lci)
endif()

install(
TARGETS LCI lci-ucx
TARGETS LCI LCT lci-ucx
EXPORT HPXLCITarget
COMPONENT core
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
)

install(
DIRECTORY ${lci_SOURCE_DIR}/src/api/ ${lci_BINARY_DIR}/src/api/
DIRECTORY ${lci_SOURCE_DIR}/lci/api/ ${lci_BINARY_DIR}/lci/api/
${lci_SOURCE_DIR}/lct/api/ ${lci_BINARY_DIR}/lct/api/
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
COMPONENT core
FILES_MATCHING
PATTERN "*.h"
)

export(
TARGETS LCI lci-ucx
TARGETS LCI LCT lci-ucx
NAMESPACE LCI::
FILE "${CMAKE_CURRENT_BINARY_DIR}/lib/cmake/${HPX_PACKAGE_NAME}/HPXLCITarget.cmake"
)
Expand Down
53 changes: 43 additions & 10 deletions libs/core/lci_base/include/hpx/lci_base/lci_environment.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,22 +42,55 @@ namespace hpx { namespace util {

static std::string get_processor_name();

// Configurations:
// Log level
// log
enum class log_level_t
{
none,
profile,
debug
debug,
};
static log_level_t log_level;
// Output filename of log
static FILE* log_outfile;
static void log(log_level_t level, const char* format, ...);

private:
static bool enabled_;
};
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
static LCT_log_ctx_t log_ctx;
#endif
static void log(
log_level_t level, const char* tag, const char* format, ...);
// performance counter
// clang-format off
#define HPX_LCI_PCOUNTER_NONE_FOR_EACH(_macro)

#define HPX_LCI_PCOUNTER_TREND_FOR_EACH(_macro) \
_macro(send_conn_start) \
_macro(send_conn_end) \
_macro(recv_conn_start) \
_macro(recv_conn_end)

#define HPX_LCI_PCOUNTER_TIMER_FOR_EACH(_macro) \
_macro(send_conn_timer) \
_macro(recv_conn_timer) \
_macro(async_write_timer) \
_macro(send_timer) \
_macro(handle_parcels) \
_macro(poll_comp) \
_macro(useful_bg_work)
// clang-format on

#define HPX_LCI_PCOUNTER_HANDLE_DECL(name) static LCT_pcounter_handle_t name;

HPX_LCI_PCOUNTER_NONE_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DECL)
HPX_LCI_PCOUNTER_TREND_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DECL)
HPX_LCI_PCOUNTER_TIMER_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DECL)

static LCT_pcounter_ctx_t pcounter_ctx;
static int64_t pcounter_now();
static int64_t pcounter_since(int64_t then);
static void pcounter_add(LCT_pcounter_handle_t handle, int64_t val);
static void pcounter_start(LCT_pcounter_handle_t handle);
static void pcounter_end(LCT_pcounter_handle_t handle);

private:
static bool enabled_;
};
}} // namespace hpx::util

#include <hpx/config/warnings_suffix.hpp>
Expand Down
152 changes: 95 additions & 57 deletions libs/core/lci_base/src/lci_environment.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,60 +91,19 @@ namespace hpx { namespace util {
defined(HPX_HAVE_MODULE_LCI_BASE)

namespace hpx { namespace util {

bool lci_environment::enabled_ = false;
lci_environment::log_level_t lci_environment::log_level = log_level_t::none;
FILE* lci_environment::log_outfile = nullptr;
lci_environment::log_level_t lci_environment::log_level;
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
LCT_log_ctx_t lci_environment::log_ctx;
#endif
LCT_pcounter_ctx_t lci_environment::pcounter_ctx;

///////////////////////////////////////////////////////////////////////////
void lci_environment::init_config(util::runtime_configuration& rtcfg)
{
// The default value here does not matter here
std::string log_level_str = get_entry_as<std::string>(
rtcfg, "hpx.parcel.lci.log_level", "" /* Does not matter*/);
if (log_level_str == "none")
log_level = log_level_t::none;
else if (log_level_str == "profile")
log_level = log_level_t::profile;
else if (log_level_str == "debug")
log_level = log_level_t::debug;
else
throw std::runtime_error("Unknown log level " + log_level_str);
std::string log_filename = get_entry_as<std::string>(
rtcfg, "hpx.parcel.lci.log_outfile", "" /* Does not matter*/);
if (log_filename == "stderr")
log_outfile = stderr;
else if (log_filename == "stdout")
log_outfile = stdout;
else
{
const int filename_max = 256;
char filename[filename_max];
char* p0_old = log_filename.data();
char* p0_new = strchr(log_filename.data(), '%');
char* p1 = filename;
while (p0_new)
{
long nbytes = p0_new - p0_old;
HPX_ASSERT(p1 + nbytes < filename + filename_max);
memcpy(p1, p0_old, nbytes);
p1 += nbytes;
nbytes =
snprintf(p1, filename + filename_max - p1, "%d", LCI_RANK);
p1 += nbytes;
p0_old = p0_new + 1;
p0_new = strchr(p0_old, '%');
}
strncat(p1, p0_old, filename + filename_max - p1 - 1);
log_outfile = fopen(filename, "w+");
if (log_outfile == nullptr)
{
throw std::runtime_error(
"Cannot open the logfile " + std::string(filename));
}
}
}
#define HPX_LCI_PCOUNTER_HANDLE_DEF(name) \
LCT_pcounter_handle_t lci_environment::name;

HPX_LCI_PCOUNTER_NONE_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DEF)
HPX_LCI_PCOUNTER_TREND_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DEF)
HPX_LCI_PCOUNTER_TIMER_FOR_EACH(HPX_LCI_PCOUNTER_HANDLE_DEF)
///////////////////////////////////////////////////////////////////////////
void lci_environment::init(
int*, char***, util::runtime_configuration& rtcfg)
Expand Down Expand Up @@ -186,7 +145,34 @@ namespace hpx { namespace util {

rtcfg.add_entry("hpx.parcel.bootstrap", "lci");
rtcfg.add_entry("hpx.parcel.lci.rank", std::to_string(this_rank));
init_config(rtcfg);
LCT_init();
// initialize the log context
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
const char* const log_levels[] = {"none", "profile", "debug"};
log_ctx = LCT_log_ctx_alloc(log_levels,
sizeof(log_levels) / sizeof(log_levels[0]), 0, "hpx_lci",
getenv("HPX_LCI_LOG_OUTFILE"), getenv("HPX_LCI_LOG_LEVEL"),
getenv("HPX_LCI_LOG_WHITELIST"), getenv("HPX_LCI_LOG_BLACKLIST"));
log_level = static_cast<log_level_t>(LCT_log_get_level(log_ctx));
#else
log_level = log_level_t::none;
#endif
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
// initialize the performance counters
pcounter_ctx = LCT_pcounter_ctx_alloc("hpx-lci");

#define HPX_LCI_PCOUNTER_NONE_REGISTER(name) \
name = LCT_pcounter_register(pcounter_ctx, #name, LCT_PCOUNTER_NONE);
HPX_LCI_PCOUNTER_NONE_FOR_EACH(HPX_LCI_PCOUNTER_NONE_REGISTER)

#define HPX_LCI_PCOUNTER_TREND_REGISTER(name) \
name = LCT_pcounter_register(pcounter_ctx, #name, LCT_PCOUNTER_TREND);
HPX_LCI_PCOUNTER_TREND_FOR_EACH(HPX_LCI_PCOUNTER_TREND_REGISTER)

#define HPX_LCI_PCOUNTER_TIMER_REGISTER(name) \
name = LCT_pcounter_register(pcounter_ctx, #name, LCT_PCOUNTER_TIMER);
HPX_LCI_PCOUNTER_TIMER_FOR_EACH(HPX_LCI_PCOUNTER_TIMER_REGISTER)
#endif
enabled_ = true;
}

Expand All @@ -200,7 +186,13 @@ namespace hpx { namespace util {
if (enabled())
{
enabled_ = false;
// for some reasons, this code block can be entered twice when HPX exits
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
LCT_pcounter_ctx_free(&pcounter_ctx);
#endif
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
LCT_log_ctx_free(&log_ctx);
#endif
LCT_fina();
int lci_init = 0;
LCI_initialized(&lci_init);
if (lci_init)
Expand Down Expand Up @@ -240,17 +232,63 @@ namespace hpx { namespace util {
return res;
}

void lci_environment::log(
lci_environment::log_level_t level, const char* format, ...)
void lci_environment::log([[maybe_unused]] log_level_t level,
[[maybe_unused]] const char* tag, [[maybe_unused]] const char* format,
...)
{
#ifdef HPX_HAVE_PARCELPORT_LCI_LOG
if (level > log_level)
return;
va_list args;
va_start(args, format);

if (level <= log_level)
vfprintf(log_outfile, format, args);
LCT_Logv(log_ctx, static_cast<int>(level), tag, format, args);

va_end(args);
#endif
}

int64_t lci_environment::pcounter_now()
{
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
return static_cast<int64_t>(LCT_now());
#endif
return 0;
}

int64_t lci_environment::pcounter_since([[maybe_unused]] int64_t then)
{
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
return static_cast<int64_t>(LCT_now()) - then;
#endif
return 0;
}

void lci_environment::pcounter_add(
[[maybe_unused]] LCT_pcounter_handle_t handle,
[[maybe_unused]] int64_t val)
{
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
LCT_pcounter_add(pcounter_ctx, handle, val);
#endif
}

void lci_environment::pcounter_start(
[[maybe_unused]] LCT_pcounter_handle_t handle)
{
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
LCT_pcounter_start(pcounter_ctx, handle);
#endif
}

void lci_environment::pcounter_end(
[[maybe_unused]] LCT_pcounter_handle_t handle)
{
#ifdef HPX_HAVE_PARCELPORT_LCI_PCOUNTER
LCT_pcounter_end(pcounter_ctx, handle);
#endif
}

}} // namespace hpx::util

#endif
5 changes: 4 additions & 1 deletion ...parcelport_lci/include/hpx/parcelport_lci/completion_manager/completion_manager_queue.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ namespace hpx::parcelset::policies::lci {
{
completion_manager_queue()
{
LCI_queue_create(LCI_UR_DEVICE, &queue);
// LCI_queue_create(LCI_UR_DEVICE, &queue);
// Hack for now
LCI_queue_createx(LCI_UR_DEVICE,
LCI_SERVER_NUM_PKTS * (size_t) config_t::ndevices, &queue);
}

~completion_manager_queue()
Expand Down
6 changes: 6 additions & 0 deletions libs/full/parcelport_lci/include/hpx/parcelport_lci/config.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ namespace hpx::parcelset::policies::lci {
// How many pre-posted receives for new messages
// (can only be applied to `sendrecv` protocol).
static int prepost_recv_num;
// Whether to register the buffer in HPX (or rely on LCI to register it)
static bool reg_mem;
// How many devices to use
static int ndevices;
// How many completion managers to use
static int ncomps;

static void init_config(util::runtime_configuration const& rtcfg);
};
Expand Down
30 changes: 21 additions & 9 deletions libs/full/parcelport_lci/include/hpx/parcelport_lci/header.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,25 @@ namespace hpx::parcelset::policies::lci {
{
// siguature for assert_valid
pos_signature = 0,
// device idx
pos_device_idx = 1 * sizeof(value_type),
// tag
pos_tag = 1 * sizeof(value_type),
pos_tag = 2 * sizeof(value_type),
// non-zero-copy chunk size
pos_numbytes_nonzero_copy = 2 * sizeof(value_type),
pos_numbytes_nonzero_copy = 3 * sizeof(value_type),
// transmission chunk size
pos_numbytes_tchunk = 3 * sizeof(value_type),
pos_numbytes_tchunk = 4 * sizeof(value_type),
// how many bytes in total (including zero-copy and non-zero-copy chunks)
pos_numbytes = 4 * sizeof(value_type),
pos_numbytes = 5 * sizeof(value_type),
// zero-copy chunk number
pos_numchunks_zero_copy = 5 * sizeof(value_type),
pos_numchunks_zero_copy = 6 * sizeof(value_type),
// non-zero-copy chunk number
pos_numchunks_nonzero_copy = 6 * sizeof(value_type),
pos_numchunks_nonzero_copy = 7 * sizeof(value_type),
// whether piggyback data
pos_piggy_back_flag_data = 7 * sizeof(value_type),
pos_piggy_back_flag_data = 8 * sizeof(value_type),
// whether piggyback transmission chunk
pos_piggy_back_flag_tchunk = 7 * sizeof(value_type) + 1,
pos_piggy_back_address = 7 * sizeof(value_type) + 2
pos_piggy_back_flag_tchunk = 8 * sizeof(value_type) + 1,
pos_piggy_back_address = 8 * sizeof(value_type) + 2
};

template <typename buffer_type, typename ChunkType>
Expand Down Expand Up @@ -133,6 +135,16 @@ namespace hpx::parcelset::policies::lci {
return get<pos_signature>();
}

void set_device_idx(int device_idx) noexcept
{
set<pos_device_idx>(static_cast<value_type>(device_idx));
}

value_type get_device_idx() const noexcept
{
return get<pos_device_idx>();
}

void set_tag(LCI_tag_t tag) noexcept
{
set<pos_tag>(static_cast<value_type>(tag));
Expand Down
Loading

0 comments on commit 53e9a95

Please sign in to comment.