From 962363df089e6adf41daf4df21c6bed8143c89cb Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Tue, 28 May 2024 15:29:37 -0400 Subject: [PATCH] Introduce JApplicationInspector --- src/libraries/JANA/CLI/JSignalHandler.cc | 17 +--- src/libraries/JANA/CMakeLists.txt | 2 + src/libraries/JANA/JApplication.cc | 36 +++++++ src/libraries/JANA/JApplicationFwd.h | 4 + .../JANA/Utils/JApplicationInspector.cc | 94 +++++++++++++++++++ .../JANA/Utils/JApplicationInspector.h | 9 ++ 6 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 src/libraries/JANA/Utils/JApplicationInspector.cc create mode 100644 src/libraries/JANA/Utils/JApplicationInspector.h diff --git a/src/libraries/JANA/CLI/JSignalHandler.cc b/src/libraries/JANA/CLI/JSignalHandler.cc index 3f8c5cce7..b664c8075 100644 --- a/src/libraries/JANA/CLI/JSignalHandler.cc +++ b/src/libraries/JANA/CLI/JSignalHandler.cc @@ -116,20 +116,7 @@ void send_overall_report_to_named_pipe() { /// The first 2 SIGINT signals received will tell JANA to shutdown gracefully. /// On the 3rd SIGINT, the program will try to exit immediately. void handle_sigint(int) { - g_sigint_count++; - switch (g_sigint_count) { - case 1: - LOG_FATAL(*g_logger) << "Exiting gracefully..." << LOG_END; - g_app->Quit(false); - break; - case 2: - LOG_FATAL(*g_logger) << "Exiting without waiting for threads to join..." << LOG_END; - japp->Quit(true); - break; - default: - LOG_FATAL(*g_logger) << "Exiting immediately." << LOG_END; - exit(-2); - } + g_app->HandleSigint(); } void handle_usr1(int) { @@ -170,7 +157,7 @@ void register_handlers(JApplication* app) { LOG_INFO(*g_logger) << "Setting signal handler USR1. Use to write status info to the named pipe." << LOG_END; signal(SIGUSR1, handle_usr1); signal(SIGUSR2, handle_usr2); - LOG_INFO(*g_logger) << "Setting signal handler SIGINT (Ctrl-C). Use a single SIGINT for a graceful shutdown, multiple SIGINTs for a hard shutdown." << LOG_END; + LOG_INFO(*g_logger) << "Setting signal handler SIGINT (Ctrl-C). Use a single SIGINT to enter the Inspector, or multiple SIGINTs for an immediate shutdown." << LOG_END; signal(SIGINT, handle_sigint); } diff --git a/src/libraries/JANA/CMakeLists.txt b/src/libraries/JANA/CMakeLists.txt index 45fb77e13..8f06c4a15 100644 --- a/src/libraries/JANA/CMakeLists.txt +++ b/src/libraries/JANA/CMakeLists.txt @@ -98,6 +98,8 @@ set(JANA2_SOURCES Utils/JCallGraphEntryMaker.h Utils/JInspector.cc Utils/JInspector.h + Utils/JApplicationInspector.cc + Utils/JApplicationInspector.h Calibrations/JCalibration.cc Calibrations/JCalibration.h diff --git a/src/libraries/JANA/JApplication.cc b/src/libraries/JANA/JApplication.cc index c58a3c43c..a8dd27a29 100644 --- a/src/libraries/JANA/JApplication.cc +++ b/src/libraries/JANA/JApplication.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include @@ -250,6 +251,10 @@ void JApplication::Run(bool wait_until_finished) { // We are going to throw the first exception and ignore the others. throw m_processing_controller->get_exceptions()[0]; } + + if (m_inspecting) { + Inspect(); + } } // Join all threads @@ -276,6 +281,14 @@ void JApplication::Scale(int nthreads) { m_processing_controller->scale(nthreads); } +void JApplication::Inspect() { + ::InspectApplication(this); + // While we are inside InspectApplication, any SIGINTs will lead to shutdown. + // Once we exit InspectApplication, one SIGINT will pause processing and reopen InspectApplication. + m_sigint_count = 0; + m_inspecting = false; +} + void JApplication::Stop(bool wait_until_idle) { if (!m_initialized) { // People might call Stop() during Initialize() rather than Run(). @@ -333,6 +346,29 @@ int JApplication::GetExitCode() { return m_exit_code; } +void JApplication::HandleSigint() { + m_sigint_count++; + switch (m_sigint_count) { + case 1: + LOG_WARN(m_logger) << "Entering Inspector..." << LOG_END; + m_inspecting = true; + m_processing_controller->request_pause(); + break; + case 2: + LOG_FATAL(m_logger) << "Exiting gracefully..." << LOG_END; + japp->Quit(false); + break; + case 3: + LOG_FATAL(m_logger) << "Exiting without waiting for threads to join..." << LOG_END; + japp->Quit(true); + break; + default: + LOG_FATAL(m_logger) << "Exiting immediately." << LOG_END; + exit(-2); + } + +} + JComponentSummary JApplication::GetComponentSummary() { /// Returns a data object describing all components currently running return m_component_manager->get_component_summary(); diff --git a/src/libraries/JANA/JApplicationFwd.h b/src/libraries/JANA/JApplicationFwd.h index 678d7fc3d..bcba49cb2 100644 --- a/src/libraries/JANA/JApplicationFwd.h +++ b/src/libraries/JANA/JApplicationFwd.h @@ -76,9 +76,11 @@ class JApplication { void Scale(int nthreads); void Stop(bool wait_until_idle = false); void Resume() {}; // TODO: Do we need this? + void Inspect(); void Quit(bool skip_join = false); void SetExitCode(int exitCode); int GetExitCode(); + void HandleSigint(); // Performance/status monitoring @@ -143,6 +145,7 @@ class JApplication { std::shared_ptr m_component_manager; std::shared_ptr m_processing_controller; + bool m_inspecting = false; bool m_quitting = false; bool m_draining_queues = false; bool m_skip_join = false; @@ -153,6 +156,7 @@ class JApplication { bool m_extended_report = false; int m_exit_code = (int) ExitCode::Success; int m_desired_nthreads; + std::atomic_int m_sigint_count = 0; std::mutex m_status_mutex; int m_ticker_interval_ms = 1000; diff --git a/src/libraries/JANA/Utils/JApplicationInspector.cc b/src/libraries/JANA/Utils/JApplicationInspector.cc new file mode 100644 index 000000000..2d1521467 --- /dev/null +++ b/src/libraries/JANA/Utils/JApplicationInspector.cc @@ -0,0 +1,94 @@ + +#include +#include + + +void PrintMenu() { + std::cout << " -----------------------------------------" << std::endl; + std::cout << " Available commands" << std::endl; + std::cout << " -----------------------------------------" << std::endl; + std::cout << " ic InspectComponents" << std::endl; + std::cout << " it InspectTopology" << std::endl; + std::cout << " ip InspectPlace arrow_id place_id" << std::endl; + std::cout << " ie InspectEvent arrow_id place_id slot_id" << std::endl; + std::cout << " f Fire arrow_id" << std::endl; + std::cout << " r Resume" << std::endl; + std::cout << " s Scale nthreads" << std::endl; + std::cout << " q Quit" << std::endl; + std::cout << " h Help" << std::endl; + std::cout << " -----------------------------------------" << std::endl; +} + +void InspectApplication(JApplication* app) { + auto engine = app->GetService(); + engine->request_pause(); + engine->wait_until_paused(); + app->SetTimeoutEnabled(false); + PrintMenu(); + + while (true) { + + std::string user_input; + std::cout << std::endl << "JANA: "; std::cout.flush(); + // obtain a single line + std::getline(std::cin, user_input); + // split into tokens + std::stringstream ss(user_input); + std::string token; + ss >> token; + std::vector args; + std::string arg; + try { + while (ss >> arg) { + args.push_back(std::stoi(arg)); + } + if (token == "InspectComponents" || token == "ic") { + // InspectComponents(); + } + else if ((token == "InspectTopology" || token == "it") && args.empty()) { + // InspectTopology(0); + } + else if ((token == "InspectPlace" || token == "ip") && args.size() == 2) { + // InspectPlace(std::stoi(args[0]), std::stoi(args[1])); + } + else if ((token == "InspectEvent" || token == "ie") && (args.size() == 3)) { + // InspectEvent(std::stoi(args[0]) + } + else if ((token == "Fire" || token == "f") && (args.size() == 1)) { + // Fire(args[0]); + } + else if (token == "Resume" || token == "r") { + app->Run(false); + break; + } + else if ((token == "Scale" || token == "s") && (args.size() == 1)) { + app->Scale(args[0]); + break; + } + else if (token == "Quit" || token == "q") { + app->Quit(true); + break; + } + else if (token == "Help" || token == "h") { + PrintMenu(); + } + else if (token == "") { + // Do nothing + } + else { + std::cout << "(Error: Unrecognized command, or wrong argument count)" << std::endl; + } + } + catch (JException& ex) { + std::cout << "(JException: " << ex.GetMessage() << ")" << std::endl; + } + catch (std::invalid_argument&) { + std::cout << "(Parse error: Maybe an argument needs to be an int)" << std::endl; + } + catch (...) { + std::cout << "(Unknown error)" << std::endl; + } + + } + +} diff --git a/src/libraries/JANA/Utils/JApplicationInspector.h b/src/libraries/JANA/Utils/JApplicationInspector.h new file mode 100644 index 000000000..b8399effa --- /dev/null +++ b/src/libraries/JANA/Utils/JApplicationInspector.h @@ -0,0 +1,9 @@ +// Copyright 2024, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. + +#pragma once + +class JApplication; +void InspectApplication(JApplication* app); + +