Introduce JApplicationInspector

src/libraries/JANA/CLI/JSignalHandler.cc

@@ -116,20 +116,7 @@ void send_overall_report_to_named_pipe() {
 /// The first 2 SIGINT signals received will tell JANA to shutdown gracefully.
 /// On the 3rd SIGINT, the program will try to exit immediately.
 void handle_sigint(int) {
-    g_sigint_count++;
-    switch (g_sigint_count) {
-        case 1:
-            LOG_FATAL(*g_logger) << "Exiting gracefully..." << LOG_END;
-            g_app->Quit(false);
-            break;
-        case 2:
-            LOG_FATAL(*g_logger) << "Exiting without waiting for threads to join..." << LOG_END;
-            japp->Quit(true);
-            break;
-        default:
-            LOG_FATAL(*g_logger) << "Exiting immediately." << LOG_END;
-            exit(-2);
-    }
+    g_app->HandleSigint();
 }
 
 void handle_usr1(int) {
@@ -170,7 +157,7 @@ void register_handlers(JApplication* app) {
     LOG_INFO(*g_logger) << "Setting signal handler USR1. Use to write status info to the named pipe." << LOG_END;
     signal(SIGUSR1, handle_usr1);
     signal(SIGUSR2, handle_usr2);
-    LOG_INFO(*g_logger) << "Setting signal handler SIGINT (Ctrl-C). Use a single SIGINT for a graceful shutdown, multiple SIGINTs for a hard shutdown." << LOG_END;
+    LOG_INFO(*g_logger) << "Setting signal handler SIGINT (Ctrl-C). Use a single SIGINT to enter the Inspector, or multiple SIGINTs for an immediate shutdown." << LOG_END;
     signal(SIGINT,  handle_sigint);
 }
 

src/libraries/JANA/CMakeLists.txt

@@ -98,6 +98,8 @@ set(JANA2_SOURCES
     Utils/JCallGraphEntryMaker.h
     Utils/JInspector.cc
     Utils/JInspector.h
+    Utils/JApplicationInspector.cc
+    Utils/JApplicationInspector.h
 
     Calibrations/JCalibration.cc
     Calibrations/JCalibration.h

src/libraries/JANA/JApplication.cc

@@ -15,6 +15,7 @@
 #include <JANA/Services/JLoggingService.h>
 #include <JANA/Services/JGlobalRootLock.h>
 #include <JANA/Engine/JArrowProcessingController.h>
+#include <JANA/Utils/JApplicationInspector.h>
 
 #include <unistd.h>
 
@@ -250,6 +251,10 @@ void JApplication::Run(bool wait_until_finished) {
             // We are going to throw the first exception and ignore the others.
             throw m_processing_controller->get_exceptions()[0];
         }
+
+        if (m_inspecting) {
+            Inspect();
+        }
     }
 
     // Join all threads
@@ -276,6 +281,14 @@ void JApplication::Scale(int nthreads) {
     m_processing_controller->scale(nthreads);
 }
 
+void JApplication::Inspect() {
+    ::InspectApplication(this);
+    // While we are inside InspectApplication, any SIGINTs will lead to shutdown.
+    // Once we exit InspectApplication, one SIGINT will pause processing and reopen InspectApplication.
+    m_sigint_count = 0; 
+    m_inspecting = false;
+}
+
 void JApplication::Stop(bool wait_until_idle) {
     if (!m_initialized) {
         // People might call Stop() during Initialize() rather than Run().
@@ -333,6 +346,29 @@ int JApplication::GetExitCode() {
     return m_exit_code;
 }
 
+void JApplication::HandleSigint() {
+    m_sigint_count++;
+    switch (m_sigint_count) {
+        case 1:
+            LOG_WARN(m_logger) << "Entering Inspector..." << LOG_END;
+            m_inspecting = true;
+            m_processing_controller->request_pause();
+            break;
+        case 2:
+            LOG_FATAL(m_logger) << "Exiting gracefully..." << LOG_END;
+            japp->Quit(false);
+            break;
+        case 3:
+            LOG_FATAL(m_logger) << "Exiting without waiting for threads to join..." << LOG_END;
+            japp->Quit(true);
+            break;
+        default:
+            LOG_FATAL(m_logger) << "Exiting immediately." << LOG_END;
+            exit(-2);
+    }
+
+}
+
 JComponentSummary JApplication::GetComponentSummary() {
     /// Returns a data object describing all components currently running
     return m_component_manager->get_component_summary();

src/libraries/JANA/JApplicationFwd.h

@@ -76,9 +76,11 @@ class JApplication {
     void Scale(int nthreads);
     void Stop(bool wait_until_idle = false);
     void Resume() {};  // TODO: Do we need this?
+    void Inspect();
     void Quit(bool skip_join = false);
     void SetExitCode(int exitCode);
     int GetExitCode();
+    void HandleSigint();
 
 
     // Performance/status monitoring
@@ -143,6 +145,7 @@ class JApplication {
     std::shared_ptr<JComponentManager> m_component_manager;
     std::shared_ptr<JArrowProcessingController> m_processing_controller;
 
+    bool m_inspecting = false;
     bool m_quitting = false;
     bool m_draining_queues = false;
     bool m_skip_join = false;
@@ -153,6 +156,7 @@ class JApplication {
     bool m_extended_report = false;
     int  m_exit_code = (int) ExitCode::Success;
     int  m_desired_nthreads;
+    std::atomic_int m_sigint_count = 0;
 
     std::mutex m_status_mutex;
     int m_ticker_interval_ms = 1000;

src/libraries/JANA/Utils/JApplicationInspector.cc

@@ -0,0 +1,94 @@
+
+#include <JANA/JApplication.h>
+#include <JANA/Engine/JArrowProcessingController.h>
+
+
+void PrintMenu() {
+    std::cout << "  -----------------------------------------" << std::endl;
+    std::cout << "  Available commands" << std::endl;
+    std::cout << "  -----------------------------------------" << std::endl;
+    std::cout << "  ic   InspectComponents" << std::endl;
+    std::cout << "  it   InspectTopology" << std::endl;
+    std::cout << "  ip   InspectPlace arrow_id place_id" << std::endl;
+    std::cout << "  ie   InspectEvent arrow_id place_id slot_id" << std::endl;
+    std::cout << "  f    Fire arrow_id" << std::endl;
+    std::cout << "  r    Resume" << std::endl;
+    std::cout << "  s    Scale nthreads" << std::endl;
+    std::cout << "  q    Quit" << std::endl;
+    std::cout << "  h    Help" << std::endl;
+    std::cout << "  -----------------------------------------" << std::endl;
+}
+
+void InspectApplication(JApplication* app) {
+    auto engine = app->GetService<JArrowProcessingController>();
+    engine->request_pause();
+    engine->wait_until_paused();
+    app->SetTimeoutEnabled(false);
+    PrintMenu();
+
+    while (true) {
+
+        std::string user_input;
+        std::cout << std::endl << "JANA: "; std::cout.flush();
+        // obtain a single line
+        std::getline(std::cin, user_input);
+        // split into tokens
+        std::stringstream ss(user_input);
+        std::string token;
+        ss >> token;
+        std::vector<int> args;
+        std::string arg;
+        try {
+            while (ss >> arg) {
+                args.push_back(std::stoi(arg));
+            }
+            if (token == "InspectComponents" || token == "ic") {
+                // InspectComponents();
+            }
+            else if ((token == "InspectTopology" || token == "it") && args.empty()) {
+                // InspectTopology(0);
+            }
+            else if ((token == "InspectPlace" || token == "ip") && args.size() == 2) {
+                // InspectPlace(std::stoi(args[0]), std::stoi(args[1]));
+            }
+            else if ((token == "InspectEvent" || token == "ie") && (args.size() == 3)) {
+                // InspectEvent(std::stoi(args[0])
+            }
+            else if ((token == "Fire" || token == "f") && (args.size() == 1)) {
+                // Fire(args[0]);
+            }
+            else if (token == "Resume" || token == "r") {
+                app->Run(false);
+                break;
+            }
+            else if ((token == "Scale" || token == "s") && (args.size() == 1)) {
+                app->Scale(args[0]);
+                break;
+            }
+            else if (token == "Quit" || token == "q") {
+                app->Quit(true);
+                break;
+            }
+            else if (token == "Help" || token == "h") {
+                PrintMenu();
+            }
+            else if (token == "") {
+                // Do nothing
+            }
+            else {
+            std::cout << "(Error: Unrecognized command, or wrong argument count)" << std::endl;
+            }
+        }
+        catch (JException& ex) {
+            std::cout << "(JException: " << ex.GetMessage() << ")" << std::endl;
+        }
+        catch (std::invalid_argument&) {
+            std::cout << "(Parse error: Maybe an argument needs to be an int)" << std::endl;
+        }
+        catch (...) {
+            std::cout << "(Unknown error)" << std::endl;
+        }
+
+    }
+
+}

src/libraries/JANA/Utils/JApplicationInspector.h

@@ -0,0 +1,9 @@
+// Copyright 2024, Jefferson Science Associates, LLC.
+// Subject to the terms in the LICENSE file found in the top-level directory.
+
+#pragma once
+
+class JApplication;
+void InspectApplication(JApplication* app);
+
+