From d2dbcf6ffd148ec307515a8c6629199a1895161d Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Thu, 27 Jun 2024 15:09:29 -0400 Subject: [PATCH 1/3] Factory parameters show up in parameters table --- src/libraries/JANA/JApplication.cc | 5 ++++- src/libraries/JANA/JFactory.cc | 14 ++++++++++---- src/libraries/JANA/JFactory.h | 7 +++++-- src/libraries/JANA/JFactorySet.cc | 11 +++++++++++ src/libraries/JANA/JFactorySet.h | 1 + src/libraries/JANA/JMultifactory.cc | 12 ++++++++++++ src/libraries/JANA/JMultifactory.h | 2 ++ src/libraries/JANA/Services/JComponentManager.cc | 9 ++++++++- src/libraries/JANA/Services/JComponentManager.h | 3 ++- .../unit_tests/Components/JMultiFactoryTests.cc | 2 ++ 10 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/libraries/JANA/JApplication.cc b/src/libraries/JANA/JApplication.cc index a8dd27a29..1ea00ead2 100644 --- a/src/libraries/JANA/JApplication.cc +++ b/src/libraries/JANA/JApplication.cc @@ -126,11 +126,14 @@ void JApplication::Initialize() { plugin_loader->attach_plugins(component_manager.get()); // Give all components a JApplication pointer and a logger - component_manager->configure_components(); + component_manager->preinitialize_components(); // Resolve all event sources now that all plugins have been loaded component_manager->resolve_event_sources(); + // Call Init() for all factories/multifactories. This populates the ParameterManager + component_manager->initialize_factories(); + // Set desired nthreads. We parse the 'nthreads' parameter two different ways for backwards compatibility. m_desired_nthreads = 1; m_params->SetDefaultParameter("nthreads", m_desired_nthreads, "Desired number of worker threads, or 'Ncores' to use all available cores."); diff --git a/src/libraries/JANA/JFactory.cc b/src/libraries/JANA/JFactory.cc index baa3f6865..54afca37c 100644 --- a/src/libraries/JANA/JFactory.cc +++ b/src/libraries/JANA/JFactory.cc @@ -9,10 +9,6 @@ void JFactory::Create(const std::shared_ptr& event) { - // We need this for JMultifactoryHelper. Eventually it should go away - auto app = event->GetJApplication(); - if (app != nullptr) SetApplication(app); - if (mStatus == Status::Uninitialized) { CallWithJExceptionWrapper("JFactory::Init", [&](){ Init(); }); mStatus = Status::Unprocessed; @@ -43,3 +39,13 @@ void JFactory::Create(const std::shared_ptr& event) { mCreationStatus = CreationStatus::Created; } } + +void JFactory::DoInit() { + if (GetApplication() == nullptr) { + throw JException("JFactory::DoInit(): Null JApplication pointer"); + } + if (mStatus == Status::Uninitialized) { + CallWithJExceptionWrapper("JFactory::Init", [&](){ Init(); }); + mStatus = Status::Unprocessed; + } +} diff --git a/src/libraries/JANA/JFactory.h b/src/libraries/JANA/JFactory.h index 02783c285..6e6c87003 100644 --- a/src/libraries/JANA/JFactory.h +++ b/src/libraries/JANA/JFactory.h @@ -168,10 +168,13 @@ class JFactory : public jana::omni::JComponent { template std::vector GetAs(); + + /// Create() calls JFactory::Init,BeginRun,Process in an invariant-preserving way without knowing the exact - /// type of object contained. It returns the number of objects created. In order to access said objects, - /// use JFactory::GetAs(). + /// type of object contained. In order to access these objects when all you have is a JFactory*, use JFactory::GetAs(). virtual void Create(const std::shared_ptr& event); + void DoInit(); + virtual void Set(const std::vector &data) = 0; virtual void Insert(JObject *data) = 0; diff --git a/src/libraries/JANA/JFactorySet.cc b/src/libraries/JANA/JFactorySet.cc index 00305788b..9a65ee9f4 100644 --- a/src/libraries/JANA/JFactorySet.cc +++ b/src/libraries/JANA/JFactorySet.cc @@ -105,6 +105,7 @@ bool JFactorySet::Add(JMultifactory *multifactory) { auto helpers = multifactory->GetHelpers(); for (auto fac : helpers->GetAllFactories()) { + fac->SetApplication(multifactory->GetApplication()); Add(fac); } helpers->mIsFactoryOwner = false; @@ -142,6 +143,16 @@ std::vector JFactorySet::GetAllFactories() const { return results; } +//--------------------------------- +// GetAllMultifactories +//--------------------------------- +std::vector JFactorySet::GetAllMultifactories() const { + std::vector results; + for (auto f : mMultifactories) { + results.push_back(f); + } + return results; +} //--------------------------------- // Merge diff --git a/src/libraries/JANA/JFactorySet.h b/src/libraries/JANA/JFactorySet.h index df3ff264c..774054c2f 100644 --- a/src/libraries/JANA/JFactorySet.h +++ b/src/libraries/JANA/JFactorySet.h @@ -35,6 +35,7 @@ class JFactorySet : public JResettable JFactory* GetFactory(const std::string& object_name, const std::string& tag="") const; template JFactoryT* GetFactory(const std::string& tag = "") const; std::vector GetAllFactories() const; + std::vector GetAllMultifactories() const; template std::vector*> GetAllFactories() const; std::vector Summarize() const; diff --git a/src/libraries/JANA/JMultifactory.cc b/src/libraries/JANA/JMultifactory.cc index 39c7d9f8e..39afe9e00 100644 --- a/src/libraries/JANA/JMultifactory.cc +++ b/src/libraries/JANA/JMultifactory.cc @@ -63,3 +63,15 @@ JFactorySet* JMultifactory::GetHelpers() { return &mHelpers; } +void JMultifactory::DoInit() { + + std::lock_guard lock(m_mutex); + if (m_status == Status::Uninitialized) { + CallWithJExceptionWrapper("JMultifactory::Init", [&](){ + Init(); + }); + m_status = Status::Initialized; + } +} + + diff --git a/src/libraries/JANA/JMultifactory.h b/src/libraries/JANA/JMultifactory.h index 43108a03e..b5fa627df 100644 --- a/src/libraries/JANA/JMultifactory.h +++ b/src/libraries/JANA/JMultifactory.h @@ -111,6 +111,8 @@ class JMultifactory : public jana::omni::JComponent, #endif /// CALLED BY JANA + + void DoInit(); void Execute(const std::shared_ptr&); // Should this be execute or create? Who is tracking that this is called at most once per event? diff --git a/src/libraries/JANA/Services/JComponentManager.cc b/src/libraries/JANA/Services/JComponentManager.cc index 5642e8415..0bb8f00e8 100644 --- a/src/libraries/JANA/Services/JComponentManager.cc +++ b/src/libraries/JANA/Services/JComponentManager.cc @@ -46,7 +46,7 @@ void JComponentManager::Init() { } } -void JComponentManager::configure_components() { +void JComponentManager::preinitialize_components() { for (auto* src : m_evt_srces) { src->SetApplication(GetApplication()); src->SetLogger(m_logging->get_logger(src->GetLoggerName())); @@ -69,6 +69,13 @@ void JComponentManager::configure_components() { } } +void JComponentManager::initialize_factories() { + JFactorySet dummy_fac_set(m_fac_gens); + for (auto* fac : dummy_fac_set.GetAllFactories()) { + fac->DoInit(); + } +} + void JComponentManager::next_plugin(std::string plugin_name) { // We defer resolving event sources until we have finished loading all plugins m_current_plugin_name = plugin_name; diff --git a/src/libraries/JANA/Services/JComponentManager.h b/src/libraries/JANA/Services/JComponentManager.h index ac00f2e48..ae07adefe 100644 --- a/src/libraries/JANA/Services/JComponentManager.h +++ b/src/libraries/JANA/Services/JComponentManager.h @@ -31,8 +31,9 @@ class JComponentManager : public JService { void add(JEventProcessor* processor); void add(JEventUnfolder* unfolder); - void configure_components(); + void preinitialize_components(); void resolve_event_sources(); + void initialize_factories(); JEventSourceGenerator* resolve_user_event_source_generator() const; JEventSourceGenerator* resolve_event_source(std::string source_name) const; diff --git a/src/programs/unit_tests/Components/JMultiFactoryTests.cc b/src/programs/unit_tests/Components/JMultiFactoryTests.cc index 8cc5402e5..bba559bbc 100644 --- a/src/programs/unit_tests/Components/JMultiFactoryTests.cc +++ b/src/programs/unit_tests/Components/JMultiFactoryTests.cc @@ -55,6 +55,7 @@ TEST_CASE("MultiFactoryTests") { SECTION("Calling from JEvent") { auto sut = new MyMultifactory(false); + sut->SetApplication(&app); auto event = std::make_shared(&app); auto fs = new JFactorySet; fs->Add(sut); @@ -70,6 +71,7 @@ TEST_CASE("MultiFactoryTests") { SECTION("Multifactory sets the wrong data") { auto sut = new MyMultifactory(true); + sut->SetApplication(&app); auto event = std::make_shared(&app); auto fs = new JFactorySet; fs->Add(sut); From 230496dc526a60ca6341e3d4d1dca1ff1dd23e02 Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Thu, 27 Jun 2024 22:30:51 -0400 Subject: [PATCH 2/3] EventSource parameters show up in parameters table --- src/libraries/JANA/JEventSource.h | 102 +++++++++++------- src/libraries/JANA/Omni/JComponentFwd.h | 2 +- .../JANA/Topology/JEventSourceArrow.cc | 10 +- .../unit_tests/Components/NEventNSkipTests.cc | 18 ++-- 4 files changed, 78 insertions(+), 54 deletions(-) diff --git a/src/libraries/JANA/JEventSource.h b/src/libraries/JANA/JEventSource.h index 834f6e556..584517243 100644 --- a/src/libraries/JANA/JEventSource.h +++ b/src/libraries/JANA/JEventSource.h @@ -42,6 +42,10 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu virtual ~JEventSource() = default; + // `Init` is where the user requests parameters and services. If the user requests all parameters and services here, + // JANA can report them back to the user without having to open the resource and run the topology. + + virtual void Init() {} // To be implemented by the user /// `Open` is called by JANA when it is ready to accept events from this event source. The implementor should open @@ -112,57 +116,73 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu } - // Wrappers for calling Open and GetEvent in a safe way + virtual void DoInit() { + if (m_status == Status::Uninitialized) { + CallWithJExceptionWrapper("JEventSource::Init", [&](){ Init();}); + m_status = Status::Initialized; + LOG_INFO(GetLogger()) << "Initialized JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + else { + throw JException("Attempted to initialize a JEventSource that is not uninitialized!"); + } + } - virtual void DoInitialize(bool with_lock=true) { + virtual void DoOpen(bool with_lock=true) { if (with_lock) { std::lock_guard lock(m_mutex); - if (m_status == Status::Uninitialized) { - CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); - if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Opened event source" << LOG_END; - } - else { - LOG_INFO(GetLogger()) << "Opened event source '" << GetResourceName() << "'" << LOG_END; - } - m_status = Status::Initialized; + if (m_status != Status::Initialized) { + throw JException("Attempted to open a JEventSource that hasn't been initialized!"); } + CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); + if (GetResourceName().empty()) { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + m_status = Status::Opened; } else { - if (m_status == Status::Uninitialized) { - CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); - if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Opened event source" << LOG_END; - } - else { - LOG_INFO(GetLogger()) << "Opened event source '" << GetResourceName() << "'" << LOG_END; - } - m_status = Status::Initialized; + if (m_status != Status::Initialized) { + throw JException("Attempted to open a JEventSource that hasn't been initialized!"); } + CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); + if (GetResourceName().empty()) { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + m_status = Status::Opened; } } - virtual void DoFinalize(bool with_lock=true) { + virtual void DoClose(bool with_lock=true) { if (with_lock) { std::lock_guard lock(m_mutex); + + if (m_status != JEventSource::Status::Opened) return; + CallWithJExceptionWrapper("JEventSource::Close", [&](){ Close();}); if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Closed event source" << LOG_END; + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "'" << LOG_END; } else { - LOG_INFO(GetLogger()) << "Closed event source '" << GetResourceName() << "'" << LOG_END; + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; } - m_status = Status::Finalized; + m_status = Status::Closed; } else { + if (m_status != JEventSource::Status::Opened) return; + CallWithJExceptionWrapper("JEventSource::Close", [&](){ Close();}); if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Closed event source" << LOG_END; + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "'" << LOG_END; } else { - LOG_INFO(GetLogger()) << "Closed event source '" << GetResourceName() << "'" << LOG_END; + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; } - m_status = Status::Finalized; + m_status = Status::Closed; } } @@ -170,6 +190,10 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu std::lock_guard lock(m_mutex); // In general, DoNext must be synchronized. + if (m_status == Status::Uninitialized) { + throw JException("JEventSource has not been initialized!"); + } + if (m_callback_style == CallbackStyle::LegacyMode) { return DoNextCompatibility(event); } @@ -177,13 +201,13 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu auto first_evt_nr = m_nskip; auto last_evt_nr = m_nevents + m_nskip; - if (m_status == Status::Uninitialized) { - DoInitialize(false); - } if (m_status == Status::Initialized) { + DoOpen(false); + } + if (m_status == Status::Opened) { if (m_nevents != 0 && (m_event_count == last_evt_nr)) { // We exit early (and recycle) because we hit our jana:nevents limit - DoFinalize(false); + DoClose(false); return Result::FailureFinished; } // If we reach this point, we will need to actually read an event @@ -219,7 +243,7 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu else if (result == Result::FailureFinished) { // We end up here if we tried to read an entry in a file, but found EOF // or if we received a message from a socket that contained no data and indicated no more data will be coming - DoFinalize(false); + DoClose(false); return Result::FailureFinished; } else if (result == Result::FailureTryAgain) { @@ -231,7 +255,7 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu throw JException("Invalid JEventSource::Result value!"); } } - else { // status == Finalized + else { // status == Closed return Result::FailureFinished; } } @@ -242,10 +266,10 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu auto last_evt_nr = m_nevents + m_nskip; try { - if (m_status == Status::Uninitialized) { - DoInitialize(false); - } if (m_status == Status::Initialized) { + DoOpen(false); + } + if (m_status == Status::Opened) { if (m_event_count < first_evt_nr) { // Skip these events due to nskip event->SetEventNumber(m_event_count); // Default event number to event count @@ -258,7 +282,7 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu return Result::FailureTryAgain; // Reject this event and recycle it } else if (m_nevents != 0 && (m_event_count == last_evt_nr)) { // Declare ourselves finished due to nevents - DoFinalize(false); // Close out the event source as soon as it declares itself finished + DoClose(false); // Close out the event source as soon as it declares itself finished return Result::FailureFinished; } else { // Actually emit an event. @@ -279,7 +303,7 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu m_event_count += 1; return Result::Success; // Don't reject this event! } - } else if (m_status == Status::Finalized) { + } else if (m_status == Status::Closed) { return Result::FailureFinished; } else { throw JException("Invalid m_status"); @@ -288,7 +312,7 @@ class JEventSource : public jana::omni::JComponent, public jana::omni::JHasOutpu catch (RETURN_STATUS rs) { if (rs == RETURN_STATUS::kNO_MORE_EVENTS) { - DoFinalize(false); + DoClose(false); return Result::FailureFinished; } else if (rs == RETURN_STATUS::kTRY_AGAIN || rs == RETURN_STATUS::kBUSY) { diff --git a/src/libraries/JANA/Omni/JComponentFwd.h b/src/libraries/JANA/Omni/JComponentFwd.h index f73bfe8f9..18b054233 100644 --- a/src/libraries/JANA/Omni/JComponentFwd.h +++ b/src/libraries/JANA/Omni/JComponentFwd.h @@ -20,7 +20,7 @@ namespace omni { struct JComponent { - enum class Status { Uninitialized, Initialized, Finalized }; + enum class Status { Uninitialized, Initialized, Opened, Closed, Finalized }; enum class CallbackStyle { LegacyMode, ExpertMode, DeclarativeMode }; struct ParameterBase; diff --git a/src/libraries/JANA/Topology/JEventSourceArrow.cc b/src/libraries/JANA/Topology/JEventSourceArrow.cc index 10af499ce..98d98153a 100644 --- a/src/libraries/JANA/Topology/JEventSourceArrow.cc +++ b/src/libraries/JANA/Topology/JEventSourceArrow.cc @@ -53,7 +53,10 @@ void JEventSourceArrow::process(Event* event, bool& success, JArrowMetrics::Stat } void JEventSourceArrow::initialize() { - // Initialization of individual sources happens on-demand, in order to keep us from having lots of open files + // We initialize everything immediately, but don't open any resources until we absolutely have to; see process(): source->DoNext() + for (JEventSource* source : m_sources) { + source->DoInit(); + } } void JEventSourceArrow::finalize() { @@ -61,9 +64,6 @@ void JEventSourceArrow::finalize() { // However, we can't rely on the JEventSources turning themselves off since execution can be externally paused. // Instead we leave everything open until we finalize the whole topology, and finalize remaining event sources then. for (JEventSource* source : m_sources) { - if (source->GetStatus() == JEventSource::Status::Initialized) { - LOG_INFO(m_logger) << "Finalizing JEventSource '" << source->GetTypeName() << "' (" << source->GetResourceName() << ")" << LOG_END; - source->DoFinalize(); - } + source->DoClose(); } } diff --git a/src/programs/unit_tests/Components/NEventNSkipTests.cc b/src/programs/unit_tests/Components/NEventNSkipTests.cc index 23d773eca..03ebbc690 100644 --- a/src/programs/unit_tests/Components/NEventNSkipTests.cc +++ b/src/programs/unit_tests/Components/NEventNSkipTests.cc @@ -102,9 +102,9 @@ TEST_CASE("JEventSourceArrow with multiple JEventSources") { app.Run(true); REQUIRE(app.GetExitCode() == (int) JApplication::ExitCode::Success); - REQUIRE(source1->GetStatus() == JEventSource::Status::Finalized); - REQUIRE(source2->GetStatus() == JEventSource::Status::Finalized); - REQUIRE(source3->GetStatus() == JEventSource::Status::Finalized); + REQUIRE(source1->GetStatus() == JEventSource::Status::Closed); + REQUIRE(source2->GetStatus() == JEventSource::Status::Closed); + REQUIRE(source3->GetStatus() == JEventSource::Status::Closed); REQUIRE(source1->open_count == 1); REQUIRE(source2->open_count == 1); REQUIRE(source3->open_count == 1); @@ -128,9 +128,9 @@ TEST_CASE("JEventSourceArrow with multiple JEventSources") { app.Run(true); REQUIRE(app.GetExitCode() == (int) JApplication::ExitCode::Success); - REQUIRE(source1->GetStatus() == JEventSource::Status::Finalized); - REQUIRE(source2->GetStatus() == JEventSource::Status::Finalized); - REQUIRE(source3->GetStatus() == JEventSource::Status::Finalized); + REQUIRE(source1->GetStatus() == JEventSource::Status::Closed); + REQUIRE(source2->GetStatus() == JEventSource::Status::Closed); + REQUIRE(source3->GetStatus() == JEventSource::Status::Closed); REQUIRE(source1->open_count == 1); REQUIRE(source2->open_count == 1); REQUIRE(source3->open_count == 1); @@ -156,9 +156,9 @@ TEST_CASE("JEventSourceArrow with multiple JEventSources") { app.Run(true); REQUIRE(app.GetExitCode() == (int) JApplication::ExitCode::Success); - REQUIRE(source1->GetStatus() == JEventSource::Status::Finalized); - REQUIRE(source2->GetStatus() == JEventSource::Status::Finalized); - REQUIRE(source3->GetStatus() == JEventSource::Status::Finalized); + REQUIRE(source1->GetStatus() == JEventSource::Status::Closed); + REQUIRE(source2->GetStatus() == JEventSource::Status::Closed); + REQUIRE(source3->GetStatus() == JEventSource::Status::Closed); REQUIRE(source1->open_count == 1); REQUIRE(source2->open_count == 1); REQUIRE(source3->open_count == 1); From e6845539c54d584d55037de058a57e1ebd2f51b5 Mon Sep 17 00:00:00 2001 From: Nathan Brei Date: Fri, 28 Jun 2024 12:29:34 -0400 Subject: [PATCH 3/3] Update JTest event source --- src/plugins/JTest/JTestParser.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/plugins/JTest/JTestParser.h b/src/plugins/JTest/JTestParser.h index 284ab9669..83e2a7951 100644 --- a/src/plugins/JTest/JTestParser.h +++ b/src/plugins/JTest/JTestParser.h @@ -37,7 +37,7 @@ class JTestParser : public JEventSource { return "JTest Fake Event Source"; } - void Open() { + void Init() override { auto app = GetApplication(); app->SetDefaultParameter("jtest:parser_ms", m_cputime_ms, "Time spent during parsing"); app->SetDefaultParameter("jtest:parser_spread", m_cputime_spread, "Spread of time spent during parsing"); @@ -45,7 +45,10 @@ class JTestParser : public JEventSource { app->SetDefaultParameter("jtest:parser_bytes_spread", m_write_spread, "Spread of bytes written during parsing"); } - Result Emit(JEvent& event) { + void Open() override { + } + + Result Emit(JEvent& event) override { if ((m_events_generated % 40) == 0) { // "Read" new entangled event every 40 events