diff --git a/CMakeLists.txt b/CMakeLists.txt index ad0705997..cd83257bc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.16) cmake_policy(SET CMP0074 NEW) # find_package() uses _ROOT implicit hints -project(jana2 VERSION 2.3.1) +project(jana2 VERSION 2.3.3) set(CMAKE_POSITION_INDEPENDENT_CODE ON) # Enable -fPIC for all targets @@ -220,6 +220,9 @@ add_subdirectory(src/python) install(DIRECTORY scripts/ DESTINATION bin FILES_MATCHING PATTERN "jana-*.py" PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_READ WORLD_EXECUTE) + install(FILES "scripts/jana-status.sh" RENAME "jana-status" DESTINATION "bin" + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_READ WORLD_EXECUTE) + include(${CMAKE_SOURCE_DIR}/cmake/MakeConfig.cmake) include(${CMAKE_SOURCE_DIR}/cmake/MakeJanaThis.cmake) include(${CMAKE_SOURCE_DIR}/cmake/MakeJVersionH.cmake) diff --git a/cmake/AddJanaPlugin.cmake b/cmake/AddJanaPlugin.cmake index c302f54e5..36705f38d 100644 --- a/cmake/AddJanaPlugin.cmake +++ b/cmake/AddJanaPlugin.cmake @@ -2,12 +2,18 @@ macro(add_jana_plugin plugin_name) # Parse remaining arguments - set(options) + set(options LINK_SHARED) set(oneValueArgs EXPORT) set(multiValueArgs SOURCES PUBLIC_HEADER TESTS) cmake_parse_arguments(PLUGIN "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if (LINK_SHARED) + set(PLUGIN_JANA_LIB jana2_shared_lib) + else() + set(PLUGIN_JANA_LIB jana2_static_lib) + endif() + if (NOT PLUGIN_SOURCES AND NOT PLUGIN_PUBLIC_HEADER AND NOT PLUGIN_TESTS) # If no arguments provided, glob everything file(GLOB HEADERS_IN_SUBDIR "include/*") @@ -64,7 +70,7 @@ macro(add_jana_plugin plugin_name) INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${CMAKE_INSTALL_PREFIX}/lib/${INSTALL_NAMESPACE}/plugins" ) - target_link_libraries(${plugin_name} PUBLIC "${JANA_NAMESPACE}jana2_static_lib") + target_link_libraries(${plugin_name} PUBLIC "${JANA_NAMESPACE}${PLUGIN_JANA_LIB}") # Handle public headers if (PLUGIN_PUBLIC_HEADER) diff --git a/cmake/JANAConfig.cmake.in b/cmake/JANAConfig.cmake.in index e1d395366..d62492012 100644 --- a/cmake/JANAConfig.cmake.in +++ b/cmake/JANAConfig.cmake.in @@ -32,4 +32,9 @@ set(JANA_INCLUDE_DIR ${JANA_INCLUDE_DIRS}) set(JANA_LIBRARY ${JANA_LIBRARIES}) set(JANA_LIB ${JANA_LIBRARIES}) +# Include JANA cmake helper macros +include("${CMAKE_CURRENT_LIST_DIR}/AddJanaPlugin.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/AddJanaLibrary.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/AddJanaTest.cmake") + diff --git a/cmake/MakeConfig.cmake b/cmake/MakeConfig.cmake index 8fafad0de..aba0927c9 100644 --- a/cmake/MakeConfig.cmake +++ b/cmake/MakeConfig.cmake @@ -27,5 +27,9 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/cmake/JANAConfigVersion.cmake" install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddJanaPlugin.cmake" DESTINATION "lib/JANA/cmake") +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddJanaLibrary.cmake" + DESTINATION "lib/JANA/cmake") +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddJanaTest.cmake" + DESTINATION "lib/JANA/cmake") diff --git a/docs/Download.md b/docs/Download.md index 8bcf47ef0..bd2d949f1 100644 --- a/docs/Download.md +++ b/docs/Download.md @@ -10,6 +10,62 @@ - [See online doxygen documentation](http://www.jlab.org/JANA/jana_doc_latest/index.html) - [Download doxygen documentation](http://www.jlab.org/JANA/jana_doc_latest.tar.gz) +### 2.3.3 + +#### Bugfixes +* Fix problem with user-defined factory generators (#366) +* JEventProcessor::Process() called before BeginRun() (#367) +* Lock overwrite in RootFillLock() (#369) +* JFactory::Finish() is called (#377) + +#### Features +* JTopologyBuilder supports topologies with arbitrarily nested levels (#346) +* Barrier events are back (#371) + +#### Refactoring +* Improved log output (#368) +* JTest uses new-style component interfaces (#374) +* JArrows now fire on individual events (#375, #378) + +- [See release on GitHub](https://github.com/JeffersonLab/JANA2/releases/tag/v2.3.3) + +### 2.3.2 +This release includes the following: + +#### Features +- Added a simple `JWiringService` which can be used to wire `JOmniFactories` via a TOML file. (#353, #363) +- Added `add_jana_plugin`, `add_jana_library`, and `add_jana_test` CMake macros (#364) + +#### Bugfixes +- A multithreading bug in `JEventProcessor` has been fixed. +- `JFactory::Create` now checks `JEventSource::GetObjects` (#361) +- `JPluginLoader` no longer loads plugins twice in certain cases (#343) +- `JParameterManager::FilterParameters` marks parameters as 'used', thereby avoiding spurious 'unused parameter' warnings. (#331) +- `JTypeInfo::to_string_with_si_prefix` generates the correct SI prefix in certain cases (#348) + +#### Refactoring +- Plugins and their headers are now installed to a directory that doesn't conflict with a system install (#330) +- `JPluginLoader` has been extensively rewritten (#339) +- `JCsvWriter` has been moved into `examples` (#350) +- JANA's internal performance testing RNG has been refactored to be more reproducible, and to avoid ASAN violations. (#315) +- `JPodioExample` has been split into several reusable examples. (#352) +- Code was moved from `Omni` and `Status` into `Components`, making the layered architecture clearer (#351) +- Documentation has been overhauled, including adding an extensive JANA1-to-JANA2 migration guide (#334, #336, #342, #354, #357, #359) +- CI testing has been extended (#332, #341) + +#### Behavior changes: +- JANA now has one internal logger, configurable via the `jana:loglevel` parameter. External loggers are now configurable via the `jana:global_loglevel` parameter. +- Log output has been streamlined: oversized tables are now YAML, and essential information is now logged at `WARN` level. (#362) +- `JPluginLoader` now stops when a plugin fails to load, rather than continuing searching for another plugin with the same name. +- `JPluginLoader` no longer accepts paths as part of a valid plugin name +- `JFactorySet` is no longer silent when the user attempts to include duplicates of the same factory (#343) +- `JMetadata` is deprecated, to be replaced with `JMultifactory`. (#345) +- All `JFactories` now call `JEventSource::GetObjects`, not just `JGetObjectsFactory`. (#361) + +- [See release on GitHub](https://github.com/JeffersonLab/JANA2/releases/tag/v2.3.2) +- [See online doxygen documentation](http://www.jlab.org/JANA/jana_doc_2.3.1/index.html) +- [Download doxygen documentation](http://www.jlab.org/JANA/jana_doc_2.3.1.tar.gz) + ### 2.3.1 This release fixes a bug which caused the `janadot` plugin to stop producing output. It also drops support for Podio <= 00-17 by replacing the user-provided `PodioTypeMap` with the built-in `PodioT::collection_type`. diff --git a/docs/doxygen/doxygen_index.md b/docs/doxygen/doxygen_index.md index 8760e35cc..e260cc468 100644 --- a/docs/doxygen/doxygen_index.md +++ b/docs/doxygen/doxygen_index.md @@ -30,14 +30,9 @@ This website provides documentation for JANA2 C++ API automatically generated by ## Internal services -* [JLoggingService](class_j_logging_service.html): Furnish the user with a logger already configured for that particular component * [JParameterManager](class_j_parameter_manager.html): Furnish the user with parameters extracted from command line flags and configuration files -## Parallelism engine +* [JExecutionEngine](class_j_execution_engine.html): Runs the topology. Manages the thread team and the topology status. -* [JProcessingController](class_j_processing_controller.html): The interface which any parallelism engine must adhere to -* [JArrowProcessingController](class_j_arrow_processing_controller.html): The entry point into the "Arrow" engine -* [JWorker](class_j_worker.html): Contains the loop for each worker thread, along with startup/shutdown logic and encapsulated worker state. -* [JScheduler](class_j_scheduler.html): Contains the logic for giving a worker a new assignment diff --git a/docs/howto/other-howtos.md b/docs/howto/other-howtos.md index 27e39283a..aaad3b7ae 100644 --- a/docs/howto/other-howtos.md +++ b/docs/howto/other-howtos.md @@ -68,43 +68,45 @@ The following configuration options are used most commonly: | Name | Type | Description | |:-----|:-----|:------------| -nthreads | int | Size of thread team (Defaults to the number of cores on your machine) -plugins | string | Comma-separated list of plugin filenames. JANA will look for these on the `$JANA_PLUGIN_PATH` -plugins_to_ignore | string | This removes plugins which had been specified in `plugins`. -event_source_type | string | Manually override JANA's decision about which JEventSource to use -jana:nevents | int | Limit the number of events each source may emit -jana:nskip | int | Skip processing the first n events from each event source -jana:extended_report | bool | The amount of status information to show while running -jana:status_fname | string | Named pipe for retrieving status information remotely - - -JANA has its own logger. You can control the verbosity of different components using -the parameters `log:off`, `log:fatal`, `log:error`, `log:warn`, `log:info`, `log:debug`, and `log:trace`. -The following example shows how you would increase the verbosity of JPluginLoader and JComponentManager: -``` -jana -Pplugins=JTest -Plog:debug=JPluginLoader,JComponentManager -``` +| nthreads | int | Size of thread team (Defaults to the number of cores on your machine) | +| plugins | string | Comma-separated list of plugin filenames. JANA will look for these on the `$JANA_PLUGIN_PATH` | +| plugins_to_ignore | string | This removes plugins which had been specified in `plugins`. | +| event_source_type | string | Manually specify which JEventSource to use | +| jana:nevents | int | Limit the number of events each source may emit | +| jana:nskip | int | Skip processing the first n events from each event source | +| jana:status_fname | string | Named pipe for retrieving status information remotely | +| jana:loglevel | string | Set the log level (trace,debug,info,warn,error,fatal,off) for loggers internal to JANA | +| jana:global_loglevel | string | Set the default log level (trace,debug,info,warn,error,fatal,off) for all loggers | +| jana:show_ticker | bool | Controls whether the status ticker is shown | +| jana:ticker_interval | int | Controls how often the status ticker updates (in ms) | +| jana:extended_report | bool | Controls whether to show extra details in the status ticker and final report | + +JANA automatically provides each component with its own logger. You can control the logging verbosity of individual components +just like any other parameter. For instance, if your component prefixes its parameters with `BCAL:tracking`, +you can set its log level to `DEBUG` by setting `BCAL:tracking:loglevel=debug`. You can set the parameter prefix by calling `SetPrefix()`, +and if you need the logger name to differ from the parameter prefix for any reason, you can override it by calling `SetLoggerName()`. + The `JTest` plugin lets you test JANA's performance for different workloads. It simulates a typical reconstruction pipeline with four stages: parsing, disentangling, tracking, and plotting. Parsing and plotting are sequential, whereas disentangling and tracking are parallel. Each stage reads all of the data written during the previous stage. The time spent and bytes written (and random variation thereof) are set using the following parameters: | Name | Type | Default | Description | |:-----|:-----|:------------|:--------| -jtest:parser_ms | int | 0 | Time spent during parsing -jtest:parser_spread | int | 0.25 | Spread of time spent during parsing -jtest:parser_bytes | int | 2000000 | Bytes written during parsing -jtest:parser_bytes_spread | double | 0.25 | Spread of bytes written during parsing -jtest:disentangler_ms | int | 20 | Time spent during disentangling -jtest:disentangler_spread | double | 0.25 | Spread of time spent during disentangling -jtest:disentangler_bytes | int | 500000 | Bytes written during disentangling -jtest:disentangler_bytes_spread | double | 0.25 | Spread of bytes written during disentangling -jtest:tracker_ms | int | 200 | Time spent during tracking -jtest:tracker_spread | double | 0.25 | Spread of time spent during tracking -jtest:tracker_bytes | int | 1000 | Bytes written during tracking -jtest:tracker_bytes_spread | double | 0.25 | Spread of bytes written during tracking -jtest:plotter_ms | int | 0 | Time spent during plotting -jtest:plotter_spread | double | 0.25 | Spread of time spent during plotting -jtest:plotter_bytes | int | 1000 | Bytes written during plotting -jtest:plotter_bytes_spread | double | 0.25 | Spread of bytes written during plotting +| jtest:parser:cputime_ms | int | 0 | Time spent during parsing | +| jtest:parser:cputime_spread | int | 0.25 | Spread of time spent during parsing | +| jtest:parser:bytes | int | 2000000 | Bytes written during parsing | +| jtest:parser:bytes_spread | double | 0.25 | Spread of bytes written during parsing | +| jtest:disentangler:cputime_ms | int | 20 | Time spent during disentangling | +| jtest:disentangler:cputime_spread | double | 0.25 | Spread of time spent during disentangling | +| jtest:disentangler:bytes | int | 500000 | Bytes written during disentangling | +| jtest:disentangler:bytes_spread | double | 0.25 | Spread of bytes written during disentangling | +| jtest:tracker:cputime_ms | int | 200 | Time spent during tracking | +| jtest:tracker:cputime_spread | double | 0.25 | Spread of time spent during tracking | +| jtest:tracker:bytes | int | 1000 | Bytes written during tracking | +| jtest:tracker:bytes_spread | double | 0.25 | Spread of bytes written during tracking | +| jtest:plotter:cputime_ms | int | 0 | Time spent during plotting | +| jtest:plotter:cputime_spread | double | 0.25 | Spread of time spent during plotting | +| jtest:plotter:bytes | int | 1000 | Bytes written during plotting | +| jtest:plotter:bytes_spread | double | 0.25 | Spread of bytes written during plotting | @@ -112,26 +114,21 @@ The following parameters are used for benchmarking: | Name | Type | Default | Description | |:-----|:-----|:------------|:--------| -benchmark:nsamples | int | 15 | Number of measurements made for each thread count -benchmark:minthreads | int | 1 | Minimum thread count -benchmark:maxthreads | int | ncores | Maximum thread count -benchmark:threadstep | int | 1 | Thread count increment -benchmark:resultsdir | string | JANA_Test_Results | Directory name for benchmark test results +| benchmark:nsamples | int | 15 | Number of measurements made for each thread count | +| benchmark:minthreads | int | 1 | Minimum thread count | +| benchmark:maxthreads | int | ncores | Maximum thread count | +| benchmark:threadstep | int | 1 | Thread count increment | +| benchmark:resultsdir | string | JANA_Test_Results | Directory name for benchmark test results | The following parameters are more advanced, but may come in handy when doing performance tuning: | Name | Type | Default | Description | |:-----|:-----|:------------|:--------| -jana:engine | int | 0 | Which parallelism engine to use. 0: JArrowProcessingController. 1: JDebugProcessingController. -jana:event_pool_size | int | nthreads | The number of events which may be in-flight at once -jana:limit_total_events_in_flight | bool | 1 | Whether the number of in-flight events should be limited -jana:affinity | int | 0 | Thread pinning strategy. 0: None. 1: Minimize number of memory localities. 2: Minimize number of hyperthreads. -jana:locality | int | 0 | Memory locality strategy. 0: Global. 1: Socket-local. 2: Numa-domain-local. 3. Core-local. 4. Cpu-local -jana:enable_stealing | bool | 0 | Allow threads to pick up work from a different memory location if their local mailbox is empty. -jana:event_queue_threshold | int | 80 | Mailbox buffer size -jana:event_source_chunksize | int | 40 | Reduce mailbox contention by chunking work assignments -jana:event_processor_chunksize | int | 1 | Reduce mailbox contention by chunking work assignments +| jana:max_inflight_events | int | nthreads | The number of events which may be in-flight at once. Should be at least `nthreads`, more gives better load balancing. | +| jana:affinity | int | 0 | Thread pinning strategy. 0: None. 1: Minimize number of memory localities. 2: Minimize number of hyperthreads. | +| jana:locality | int | 0 | Memory locality strategy. 0: Global. 1: Socket-local. 2: Numa-domain-local. 3. Core-local. 4. Cpu-local | +| jana:enable_stealing | bool | 0 | Allow threads to pick up work from a different memory location if their local mailbox is empty. | Creating code skeletons diff --git a/docs/jana1to2/developers-transition-guide.md b/docs/jana1to2/developers-transition-guide.md index a446e36e5..45cf5fcbe 100644 --- a/docs/jana1to2/developers-transition-guide.md +++ b/docs/jana1to2/developers-transition-guide.md @@ -686,6 +686,26 @@ for (auto hit: results){ Set(results) ``` +### Forcing Factory to Regenerate Objects + +##### **JANA1** +In JANA1, adding `use_factory` in the constructor forces the factory to regenerate objects, regardless of whether they already exist in the input file. This prevents JANA from searching the input file for those objects. For example: + +```cpp +DEventWriterROOT_factory_ReactionEfficiency() { + use_factory = 1; // prevents JANA from searching the input file for these objects +}; +``` + +##### **JANA2** +In JANA2, the `use_factory` flag is no longer used. To achieve the same functionality, the `SetRegenerateFlag` function must be called with `true` as the parameter. This will prevent JANA from searching the input file for these objects, similar to the behavior in JANA1. For example: + +```cpp +DEventWriterROOT_factory_ReactionEfficiency() { + SetRegenerateFlag(true); // prevents JANA from searching the input file for these objects +}; +``` + ## JEvent ### Transition from JEventLoop to JEvent ##### **JANA1** diff --git a/docs/jana1to2/parameter-changes.md b/docs/jana1to2/parameter-changes.md index 80d1f6a26..c11083e52 100644 --- a/docs/jana1to2/parameter-changes.md +++ b/docs/jana1to2/parameter-changes.md @@ -10,9 +10,9 @@ The following table compares commonly used JANA parameters between JANA1 and JAN |-----------------------|------------------------|------------------------------|--------------------------------------------------| | **PLUGINS** | `- ` | `-` | `-` | | **EVENTS_TO_KEEP** | `EVENTS_TO_KEEP` | `jana:nevents` | `-` | -| **EVENTS_TO_SKIP** | `EVENTS_TO_SKIP` | `jana:nskips` | `-` | +| **EVENTS_TO_SKIP** | `EVENTS_TO_SKIP` | `jana:nskip` | `-` | | **NTHREADS** | `-` | `-` | `-` | -| **JANA:BATCH_MODE** | `JANA:BATCH_MODE` | `log:global` | `TRACE`, `DEBUG`, `INFO`, `WARN`, `FATAL`, `OFF` | +| **JANA:BATCH_MODE** | `JANA:BATCH_MODE` | `jana:global_loglevel` | `TRACE`, `DEBUG`, `INFO`, `WARN`, `FATAL`, `OFF` | | **JANA_CALIB_CONTEXT**| `JANA_CALIB_CONTEXT` | `jana:calib_context` | `-` | ## Changes in `halld_recon` Parameters @@ -22,4 +22,4 @@ The following table compares commonly used JANA parameters between JANA1 and JAN - **JANA1:** The `-b` option was used for printing event status bits in `hd_dump`. - **JANA2:** The `-b` option is used for benchmarking in JANA2. Thus, `hd_dump` parameter for event status bits (`-b`) has been changed to `-B`. Update your usage accordingly. -For additional assistance or questions, please contact [rasool@jlab.org](mailto:rasool@jlab.org). \ No newline at end of file +For additional assistance or questions, please contact [rasool@jlab.org](mailto:rasool@jlab.org). diff --git a/scripts/jana-generate.py b/scripts/jana-generate.py index f84597cbc..ffa3d119b 100755 --- a/scripts/jana-generate.py +++ b/scripts/jana-generate.py @@ -187,8 +187,6 @@ class {name} : public JEventSource {{ public: {name}(); - {name}(std::string resource_name, JApplication* app); - virtual ~{name}() = default; void Open() override; @@ -230,12 +228,6 @@ class {name} : public JEventSource {{ SetCallbackStyle(CallbackStyle::ExpertMode); }} -{name}::{name}(std::string resource_name, JApplication* app) : JEventSource(resource_name, app) {{ - SetTypeName(NAME_OF_THIS); // Provide JANA with class name - SetCallbackStyle(CallbackStyle::ExpertMode); -}} - - void {name}::Open() {{ /// Open is called exactly once when processing begins. diff --git a/scripts/jana-plot-utilization.py b/scripts/jana-plot-utilization.py new file mode 100644 index 000000000..80dc9ef03 --- /dev/null +++ b/scripts/jana-plot-utilization.py @@ -0,0 +1,188 @@ + +import re +import svgwrite +from datetime import datetime, timedelta +from collections import defaultdict + + +def parse_logfile(): + + # Parse the logs and store intervals + thread_history = defaultdict(list) # Key: thread_id, Value: list of (start_time, end_time, processor_name, event_nr, result) + barrier_history = [] # [(released_timestamp, finished_timestamp)] + + start_times = {} # Key: (thread_id, processor_name), Value: start_time + + # Define a regular expression to parse the log lines + source_start_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) Executing arrow (\w+)$") + source_finish_noemit_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) Executed arrow (\w+) with result (\w+)$") + source_finish_emit_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) Executed arrow (\w+) with result (\w+), emitting event# (\d+)$") + source_finish_pending_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) Executed arrow (\w+) with result (\w+), holding back barrier event# (\d+)$") + processor_start_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) Executing arrow (\w+) for event# (\d+)$") + processor_finish_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) Executed arrow (\w+) for event# (\d+)$") + barrier_inflight_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) JEventSourceArrow: Barrier event is in-flight$") + barrier_finished_pattern = re.compile(r"^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) \[debug\] (\d+) JEventSourceArrow: Barrier event finished, returning to normal operation$") + + with open("log.txt", "r") as log_file: + for line in log_file: + + match = re.match(source_start_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id, processor_name = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_times[(thread_id, processor_name)] = millis + continue + + match = re.match(source_finish_noemit_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id, processor_name, result = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_time = start_times.pop((thread_id, processor_name), None) + if start_time: + thread_history[thread_id].append((start_time, millis, processor_name, None, result)) + continue + + match = re.match(source_finish_emit_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id, processor_name, result, event_nr = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_time = start_times.pop((thread_id, processor_name), None) + if start_time: + thread_history[thread_id].append((start_time, millis, processor_name, event_nr, result)) + continue + + match = re.match(source_finish_pending_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id, processor_name, result, event_nr = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_time = start_times.pop((thread_id, processor_name), None) + if start_time: + thread_history[thread_id].append((start_time, millis, processor_name, event_nr, result)) + continue + + match = re.match(processor_start_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id, processor_name, event_nr = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_times[(thread_id, processor_name)] = millis + continue + + match = re.match(processor_finish_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id, processor_name, event_nr = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_time = start_times.pop((thread_id, processor_name), None) + if start_time: + thread_history[thread_id].append((start_time, millis, processor_name, event_nr, result)) + continue + + match = re.match(barrier_inflight_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_times[()] = millis + continue + + match = re.match(barrier_finished_pattern, line.strip()) + if match: + hours_str, mins_str, secs_str, millis_str, thread_id = match.groups() + millis = (((int(hours_str) * 60) + int(mins_str) * 60) + int(secs_str)) * 1000 + int(millis_str) + start_time = start_times.pop((), None) + if start_time: + barrier_history.append((start_time, millis)) + continue + + return (thread_history, barrier_history) + + + +def create_svg(all_thread_history, barrier_history): + # Assign colors to processors + processor_colors = {} + color_palette = ['#004E64', '#00A5CF', '#9FFFCB', '#25A18E', '#7AE582', '#FF69B4'] + color_index = 0 + + # Figure out drawing coordinate system + overall_start_time = min(start for history in all_thread_history.values() for (start,_,_,_,_) in history) + overall_end_time = max(end for history in all_thread_history.values() for (_,end,_,_,_) in history) + thread_count = len(all_thread_history) + width=1000 + x_scale = width/(overall_end_time-overall_start_time) + thread_height=40 + thread_y_padding=20 + height=thread_count * thread_height + (thread_count+1) * thread_y_padding + + + # Create the SVG drawing + dwg = svgwrite.Drawing("timeline.svg", profile='tiny', size=(width, height)) + #dwg.add(dwg.rect(insert=(0,0),size=(width,height),stroke="red",fill="white")) + + # Draw a rectangle for each processor run on each thread's timeline + y_position = thread_y_padding + + for barrier_start,barrier_end in barrier_history: + rect_start = (barrier_start-overall_start_time)*x_scale + if (barrier_end == barrier_start): + rect_width=1 + else: + rect_width = (barrier_end-barrier_start)*x_scale + + rect = dwg.rect(insert=(rect_start, 0), + size=(rect_width, height), + fill="red", + stroke="none", + stroke_width=1) + dwg.add(rect) + + for thread_id, intervals in all_thread_history.items(): + dwg.add(dwg.rect(insert=(0,y_position),size=(1000,thread_height),stroke="lightgray",fill="lightgray")) + for start_time, end_time, processor_name, event_nr, result in intervals: + # Calculate the position and width of the rectangle + # Assign a unique color to each processor name + if processor_name not in processor_colors: + processor_colors[processor_name] = color_palette[color_index % len(color_palette)] + color_index += 1 + + # Draw the rectangle + rect_start = (start_time-overall_start_time)*x_scale + if (end_time == start_time): + rect_width=1 + else: + rect_width = (end_time-start_time)*x_scale + + rect_stroke_color = "black" + if (result == "ComeBackLater" and event_nr is None): + rect_stroke_color = "gray" + + + rect = dwg.rect(insert=(rect_start, y_position), + size=(rect_width, thread_height), + fill=processor_colors[processor_name], + stroke=rect_stroke_color, + stroke_width=1) + mouseover = "Arrow: " + processor_name + "\nEvent nr: " + str(event_nr) + "\nResult: " + result + "\nTime: "+ str(end_time-start_time) + "ms" + rect.add(svgwrite.base.Title(mouseover)) + dwg.add(rect) + if (event_nr is not None): + text = dwg.text(str(event_nr), insert=(rect_start+1, y_position+thread_height-1), fill="white", font_size=8) + dwg.add(text) + + # Move the y position for the next thread + y_position += (thread_y_padding + thread_height) + + + # Save the SVG file + dwg.save() + + + +if __name__ == "__main__": + thread_history,barrier_history = parse_logfile() + #thread_history = { + # 1103:[(0,1,"a"), (2,5,"b"), (6,8,"a")], + # 219:[(0,3,"b"), (3,6,"c"), (9,10,"d")], + # 3:[(2,7,"a")] + #} + create_svg(thread_history, barrier_history) + + diff --git a/scripts/jana-status.sh b/scripts/jana-status.sh new file mode 100644 index 000000000..4ebbc57ad --- /dev/null +++ b/scripts/jana-status.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Default named pipe path +DEFAULT_PIPE="/tmp/jana_status" + +# Check for required arguments +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [jana_status_pipe]" + exit 1 +fi + +# Assign variables +PID="$1" +PIPE="${2:-$DEFAULT_PIPE}" # Use the second argument if provided, otherwise use the default + +# Validate the PID +if ! kill -0 "$PID" 2>/dev/null; then + echo "Error: Process with PID $PID does not exist." + exit 1 +fi + +# Check if the pipe exists +if [[ ! -p "$PIPE" ]]; then + echo "Error: Named pipe '$PIPE' does not exist." + exit 1 +fi + +# Send the USR1 signal to the process +if ! kill -USR1 "$PID"; then + echo "Error: Failed to send SIGUSR1 to PID $PID." + exit 1 +fi + +# Cat the named pipe +cat "$PIPE" diff --git a/src/examples/CMakeLists.txt b/src/examples/CMakeLists.txt index 63b0487c4..c47828d93 100644 --- a/src/examples/CMakeLists.txt +++ b/src/examples/CMakeLists.txt @@ -4,8 +4,8 @@ add_subdirectory(DstExample) add_subdirectory(Tutorial) add_subdirectory(StreamingExample) add_subdirectory(EventGroupExample) -add_subdirectory(SubeventExample) -add_subdirectory(SubeventCUDAExample) +#add_subdirectory(SubeventExample) +#add_subdirectory(SubeventCUDAExample) add_subdirectory(UnitTestingExample) add_subdirectory(PodioDatamodel) add_subdirectory(PodioFileReader) diff --git a/src/examples/DstExample/DstExampleSource.cc b/src/examples/DstExample/DstExampleSource.cc index 518c49545..efc07f2f2 100644 --- a/src/examples/DstExample/DstExampleSource.cc +++ b/src/examples/DstExample/DstExampleSource.cc @@ -15,11 +15,6 @@ DstExampleSource::DstExampleSource() : JEventSource() { SetCallbackStyle(CallbackStyle::ExpertMode); } -DstExampleSource::DstExampleSource(std::string resource_name, JApplication* app) : JEventSource(resource_name, app) { - SetTypeName(NAME_OF_THIS); // Provide JANA with class name - SetCallbackStyle(CallbackStyle::ExpertMode); -} - void DstExampleSource::Open() { /// Open is called exactly once when processing begins. diff --git a/src/examples/DstExample/DstExampleSource.h b/src/examples/DstExample/DstExampleSource.h index ef0a9226e..7d75201af 100644 --- a/src/examples/DstExample/DstExampleSource.h +++ b/src/examples/DstExample/DstExampleSource.h @@ -17,8 +17,6 @@ class DstExampleSource : public JEventSource { public: DstExampleSource(); - DstExampleSource(std::string resource_name, JApplication* app); - virtual ~DstExampleSource() = default; void Open() override; diff --git a/src/examples/EventGroupExample/BlockingGroupedEventSource.h b/src/examples/EventGroupExample/BlockingGroupedEventSource.h index 917877950..5d82e00e8 100644 --- a/src/examples/EventGroupExample/BlockingGroupedEventSource.h +++ b/src/examples/EventGroupExample/BlockingGroupedEventSource.h @@ -24,7 +24,7 @@ class BlockingGroupedEventSource : public JEventSource { public: - BlockingGroupedEventSource(std::string res_name, JApplication* app) : JEventSource(std::move(res_name), app) { + BlockingGroupedEventSource() { // TODO: Get EventGroupManager from ServiceLocator instead SetCallbackStyle(CallbackStyle::ExpertMode); m_pending_group_id = 1; diff --git a/src/examples/EventGroupExample/EventGroupExamplePlugin.cc b/src/examples/EventGroupExample/EventGroupExamplePlugin.cc index 76f635740..9323ea54f 100644 --- a/src/examples/EventGroupExample/EventGroupExamplePlugin.cc +++ b/src/examples/EventGroupExample/EventGroupExamplePlugin.cc @@ -67,7 +67,7 @@ void InitPlugin(JApplication *app) { app->Add(new GroupedEventProcessor()); - auto evt_src = new BlockingGroupedEventSource("blocking_source", app); + auto evt_src = new BlockingGroupedEventSource(); app->Add(evt_src); diff --git a/src/examples/EventGroupExample/GroupedEventSource.h b/src/examples/EventGroupExample/GroupedEventSource.h index 6933ef935..a46474759 100644 --- a/src/examples/EventGroupExample/GroupedEventSource.h +++ b/src/examples/EventGroupExample/GroupedEventSource.h @@ -19,7 +19,7 @@ class GroupedEventSource : public JEventSource { int m_current_event_number; public: - GroupedEventSource(std::string res_name, JApplication* app) : JEventSource(std::move(res_name), app) { + GroupedEventSource() { // TODO: Get EventGroupManager from ServiceLocator instead SetCallbackStyle(CallbackStyle::ExpertMode); m_remaining_events_in_group = 5; diff --git a/src/examples/InteractiveStreamingExample/InteractiveStreamingExample.cc b/src/examples/InteractiveStreamingExample/InteractiveStreamingExample.cc index d9235524b..ab6cfcf0b 100644 --- a/src/examples/InteractiveStreamingExample/InteractiveStreamingExample.cc +++ b/src/examples/InteractiveStreamingExample/InteractiveStreamingExample.cc @@ -18,9 +18,11 @@ void dummy_publisher_loop(JApplication* app) { + auto params = app->GetJParameterManager(); + JBenchUtils bench_utils = JBenchUtils(); size_t delay_ms = 1; - auto logger = app->GetService()->get_logger("dummy_publisher_loop"); + auto logger = params->GetLogger("dummy_publisher_loop"); bench_utils.set_seed(7, "InteractiveStreamingExample.cc:dummy_publisher_loop"); std::this_thread::yield(); @@ -71,12 +73,11 @@ extern "C" { void InitPlugin(JApplication* app) { InitJANAPlugin(app); - auto logger = app->GetService()->get_logger("streamDet"); + auto logger = app->GetJParameterManager()->GetLogger("streamDet"); app->SetParameterValue("nthreads", 4); app->SetParameterValue("jana:extended_report", true); - app->SetParameterValue("jana:limit_total_events_in_flight", true); - app->SetParameterValue("jana:event_pool_size", 16); + app->SetParameterValue("jana:max_inflight_events", 16); // TODO: Consider making streamDet:sub_socket be the 'source_name', and use JESG to switch between JSES and DecodeDASSource // TODO: Improve parametermanager interface diff --git a/src/examples/SubeventCUDAExample/SubeventCUDAExample.cu b/src/examples/SubeventCUDAExample/SubeventCUDAExample.cu index 1bc4287d0..cc2b2cd2b 100644 --- a/src/examples/SubeventCUDAExample/SubeventCUDAExample.cu +++ b/src/examples/SubeventCUDAExample/SubeventCUDAExample.cu @@ -4,10 +4,10 @@ #include #include -#include #include #include #include "JANA/Engine/JTopologyBuilder.h" +#include struct MyInput : public JObject { @@ -128,12 +128,10 @@ int main() { JMailbox > subevents_out; auto split_arrow = new JSplitArrow("split", &processor, &events_in, &subevents_in); - auto subprocess_arrow = new JSubeventArrow("subprocess", &processor, &subevents_in, - &subevents_out); + auto subprocess_arrow = new JSubeventArrow("subprocess", &processor, &subevents_in, &subevents_out); auto merge_arrow = new JMergeArrow("merge", &processor, &subevents_out, &events_out); JApplication app; - app.SetParameterValue("log:info", "JWorker,JScheduler,JArrowProcessingController,JEventProcessorArrow"); app.SetTimeoutEnabled(false); app.SetTicker(false); @@ -141,12 +139,14 @@ int main() { source->SetNEvents(10); // limit ourselves to 10 events. Note that the 'jana:nevents' param won't work // here because we aren't using JComponentManager to manage the EventSource - auto topology = app.GetService()->create_empty(); - auto source_arrow = new JEventSourceArrow("simpleSource", - {source}, - &events_in, - topology->event_pool); - auto proc_arrow = new JEventProcessorArrow("simpleProcessor", &events_out, nullptr, topology->event_pool); + auto topology = app.GetService(); + auto source_arrow = new JEventSourceArrow("simpleSource", {source}); + source_arrow->set_input(topology->event_pool); + source_arrow->set_output(&events_in); + + auto proc_arrow = new JEventMapArrow("simpleProcessor"); + proc_arrow->set_input(&events_out); + proc_arrow->set_output(topology->event_pool); proc_arrow->add_processor(new SimpleProcessor); topology->arrows.push_back(source_arrow); diff --git a/src/examples/SubeventExample/SubeventExample.cc b/src/examples/SubeventExample/SubeventExample.cc index 71f5f6b25..d99feccb9 100644 --- a/src/examples/SubeventExample/SubeventExample.cc +++ b/src/examples/SubeventExample/SubeventExample.cc @@ -8,9 +8,9 @@ #include #include -#include +#include #include -#include "JANA/Topology/JTopologyBuilder.h" +#include struct MyInput : public JObject { @@ -86,7 +86,7 @@ struct SimpleProcessor : public JEventProcessor { int main() { JApplication app; - app.SetParameterValue("log:info", "JWorker,JScheduler,JArrowProcessingController,JEventProcessorArrow"); + app.SetParameterValue("jana:loglevel", "info"); app.SetTimeoutEnabled(false); app.SetTicker(false); @@ -98,8 +98,8 @@ int main() { auto topology = app.GetService(); topology->set_configure_fn([&](JTopologyBuilder& builder) { - JMailbox*> events_in; - JMailbox*> events_out; + JMailbox events_in; + JMailbox events_out; JMailbox> subevents_in; JMailbox> subevents_out; @@ -107,11 +107,13 @@ int main() { auto subprocess_arrow = new JSubeventArrow("subprocess", &processor, &subevents_in, &subevents_out); auto merge_arrow = new JMergeArrow("merge", &processor, &subevents_out, &events_out); - auto source_arrow = new JEventSourceArrow("simpleSource", - {source}, - &events_in, - topology->event_pool); - auto proc_arrow = new JEventProcessorArrow("simpleProcessor", &events_out, nullptr, topology->event_pool); + auto source_arrow = new JEventSourceArrow("simpleSource", {source}); + source_arrow->attach(topology->event_pool, 0); + source_arrow->attach(&events_in, 1); + + auto proc_arrow = new JEventMapArrow("simpleProcessor"); + proc_arrow->attach(&events_out, 0); + proc_arrow->attach(topology->event_pool, 1); proc_arrow->add_processor(new SimpleProcessor); builder.arrows.push_back(source_arrow); diff --git a/src/examples/Tutorial/RandomSource.cc b/src/examples/Tutorial/RandomSource.cc index 70a8d4268..a5881cafb 100644 --- a/src/examples/Tutorial/RandomSource.cc +++ b/src/examples/Tutorial/RandomSource.cc @@ -23,11 +23,6 @@ RandomSource::RandomSource() : JEventSource() { SetCallbackStyle(CallbackStyle::ExpertMode); } -RandomSource::RandomSource(std::string resource_name, JApplication* app) : JEventSource(resource_name, app) { - SetTypeName(NAME_OF_THIS); // Provide JANA with class name - SetCallbackStyle(CallbackStyle::ExpertMode); -} - void RandomSource::Open() { /// Open is called exactly once when processing begins. diff --git a/src/examples/Tutorial/RandomSource.h b/src/examples/Tutorial/RandomSource.h index 87fb80c61..11e1d2c4e 100644 --- a/src/examples/Tutorial/RandomSource.h +++ b/src/examples/Tutorial/RandomSource.h @@ -16,8 +16,6 @@ class RandomSource : public JEventSource { public: RandomSource(); - RandomSource(std::string resource_name, JApplication* app); - virtual ~RandomSource() = default; void Open() override; diff --git a/src/examples/Tutorial/Tutorial.cc b/src/examples/Tutorial/Tutorial.cc index 1823332ba..0c31b78b0 100644 --- a/src/examples/Tutorial/Tutorial.cc +++ b/src/examples/Tutorial/Tutorial.cc @@ -20,7 +20,7 @@ void InitPlugin(JApplication* app) { app->Add(new JFactoryGeneratorT); // Always use RandomSource - app->Add(new RandomSource("random", app)); + app->Add(new RandomSource()); // Only use RandomSource when 'random' specified on cmd line // app->Add(new JEventSourceGeneratorT); diff --git a/src/examples/UnitTestingExample/SimpleClusterFactoryTests.cc b/src/examples/UnitTestingExample/SimpleClusterFactoryTests.cc index d7ca22701..154338a90 100644 --- a/src/examples/UnitTestingExample/SimpleClusterFactoryTests.cc +++ b/src/examples/UnitTestingExample/SimpleClusterFactoryTests.cc @@ -13,8 +13,7 @@ TEST_CASE("SimpleClusterFactoryTests") { // We need to fire up the JApplication so that our Factory can access all of its JServices. // However, for unit testing, we don't need (or want!) to set up an event source or actually call JApplication::Run(). JApplication app; - app.SetParameterValue("log:debug", "SimpleClusterFactory"); - app.SetParameterValue("log:off", "JApplication,JParameterManager,JArrowProcessingController,JArrow"); + app.SetParameterValue("jana:loglevel", "off"); // Add any plugins you need here // app.AddPlugin("myPlugin"); app.Initialize(); // Load the plugins diff --git a/src/libraries/JANA/CLI/JBenchmarker.cc b/src/libraries/JANA/CLI/JBenchmarker.cc index 6df57d73f..4e1ef7f85 100644 --- a/src/libraries/JANA/CLI/JBenchmarker.cc +++ b/src/libraries/JANA/CLI/JBenchmarker.cc @@ -5,7 +5,6 @@ #include "JBenchmarker.h" #include -#include #include #include @@ -15,10 +14,11 @@ JBenchmarker::JBenchmarker(JApplication* app) : m_app(app) { m_max_threads = JCpuInfo::GetNumCpus(); - m_logger = app->GetService()->get_logger("JBenchmarker"); auto params = app->GetJParameterManager(); + m_logger = params->GetLogger("JBenchmarker"); + params->SetParameter("jana:nevents", 0); // Prevent users' choice of nevents from interfering with everything @@ -159,7 +159,7 @@ void JBenchmarker::RunUntilFinished() { << " cd " << m_output_dir << "\n" << " $JANA_HOME/bin/jana-plot-scaletest.py\n" << LOG_END; } - m_app->Stop(); + m_app->Stop(true); } diff --git a/src/libraries/JANA/CLI/JMain.cc b/src/libraries/JANA/CLI/JMain.cc index bd335b3b1..f9c70954b 100644 --- a/src/libraries/JANA/CLI/JMain.cc +++ b/src/libraries/JANA/CLI/JMain.cc @@ -153,7 +153,7 @@ int Execute(JApplication* app, UserOptions &options) { else { // Run JANA in normal mode try { - app->Run(); + app->Run(true, true); } catch (JException& e) { std::cout << "----------------------------------------------------------" << std::endl; diff --git a/src/libraries/JANA/CLI/JSignalHandler.cc b/src/libraries/JANA/CLI/JSignalHandler.cc index b664c8075..2ef87f4d8 100644 --- a/src/libraries/JANA/CLI/JSignalHandler.cc +++ b/src/libraries/JANA/CLI/JSignalHandler.cc @@ -4,110 +4,19 @@ #include "JSignalHandler.h" -#include -#include -#include -#include -#include - #include +#include +#include + +#include +#include /// JSignalHandler bundles together the logic for querying a JApplication /// about its JStatus with signal handlers for USR1, USR2, and CTRL-C. namespace JSignalHandler { JApplication* g_app; -int g_sigint_count = 0; JLogger* g_logger; -std::string g_path_to_named_pipe = "/tmp/jana_status"; -std::map g_thread_reports; -std::atomic_int g_thread_report_count; - - -void create_named_pipe(const std::string& path_to_named_pipe) { - - LOG_INFO(*g_logger) << "Creating pipe named \"" << g_path_to_named_pipe - << "\" for status info." << LOG_END; - - mkfifo(path_to_named_pipe.c_str(), 0666); -} - - -void send_to_named_pipe(const std::string& path_to_named_pipe, const std::string& data) { - - int fd = open(path_to_named_pipe.c_str(), O_WRONLY); - if (fd >=0) { - write(fd, data.c_str(), data.length()+1); - close(fd); - } - else { - LOG_WARN(*g_logger) << "Unable to open named pipe '" << g_path_to_named_pipe << "' for writing. \n" - << " You can use a different named pipe for status info by setting the parameter `jana:status_fname`.\n" - << " The status report will still show up in the log." << LOG_END; - } -} - -void produce_thread_report() { - std::stringstream bt_str; - make_backtrace(bt_str); - g_thread_reports[pthread_self()] = bt_str.str(); -} - -/// If something goes wrong, we want to signal all threads to assemble a report -/// Whereas USR1 is meant to be triggered externally and is caught by one thread, -/// produce_overall_report triggers USR2 and is caught by all threads. -std::string produce_overall_report() { - std::stringstream ss; - - // Include detailed report from JApplication - auto t = time(nullptr); - ss << "JANA status report: " << ctime(&t) << std::endl; - ss << g_app->GetComponentSummary() << std::endl; - - // Include backtraces from each individual thread - if( typeid(std::thread::native_handle_type) == typeid(pthread_t) ){ - ss << "Thread model: pthreads" << std::endl; - - // Send every worker thread (but not self) the USR2 signal - auto main_thread_id = pthread_self(); - std::vector threads; // TODO: Populate this - g_thread_report_count = threads.size(); - for (auto& thread_id : threads) { - if (main_thread_id == thread_id) { - pthread_kill(thread_id, SIGUSR2); - } - } - - // Assemble backtrace for own thread - std::stringstream bt_str; - make_backtrace(bt_str); - g_thread_reports[main_thread_id] = bt_str.str(); - - // Wait for all other threads to finish handling USR2 - for(int i=0; i<1000; i++){ - std::this_thread::sleep_for(std::chrono::microseconds(1000)); - if (g_thread_report_count == 0) break; - } - - // Assemble overall backtrace - for (const auto& thread_report : g_thread_reports) { - ss << thread_report.first << ": " << std::endl << thread_report.second << std::endl; - } - - // Clear backtrace - g_thread_reports.clear(); - // TODO: Backtrace memory use is unsafe - } - else { - ss << "Thread model: unknown" << std::endl; - } - return ss.str(); -} - -void send_overall_report_to_named_pipe() { - LOG_WARN(*g_logger) << "Caught USR1 signal! Sending status report to named pipe. `cat " << g_path_to_named_pipe << "` to view." << LOG_END; - send_to_named_pipe(g_path_to_named_pipe, produce_overall_report()); -} /// Handle SIGINT signals (e.g. from hitting Ctrl-C). When a SIGINT @@ -116,23 +25,39 @@ void send_overall_report_to_named_pipe() { /// The first 2 SIGINT signals received will tell JANA to shutdown gracefully. /// On the 3rd SIGINT, the program will try to exit immediately. void handle_sigint(int) { - g_app->HandleSigint(); + if (g_app->IsInitialized()) { + g_app->GetService()->HandleSIGINT(); + } + else { + exit(-2); + } } void handle_usr1(int) { - std::thread th(send_overall_report_to_named_pipe); - th.detach(); + if (g_app->IsInitialized()) { + g_app->GetService()->HandleSIGUSR1(); + } } void handle_usr2(int) { - produce_thread_report(); + if (g_app->IsInitialized()) { + g_app->GetService()->HandleSIGUSR2(); + } +} + +void handle_tstp(int) { + if (g_app->IsInitialized()) { + g_app->GetService()->HandleSIGTSTP(); + } } void handle_sigsegv(int /*signal_number*/, siginfo_t* /*signal_info*/, void* /*context*/) { - LOG_FATAL(*g_logger) << "Segfault detected! Printing backtraces and exiting." << LOG_END; - auto report = produce_overall_report(); - LOG_INFO(*g_logger) << report << LOG_END; - exit(static_cast(JApplication::ExitCode::Segfault)); + LOG_FATAL(*g_logger) << "Segfault detected!" << LOG_END; + JBacktrace backtrace; + backtrace.Capture(3); + LOG_FATAL(*g_logger) << "Hard exit due to segmentation fault! Backtrace:\n\n" << backtrace.ToString() << LOG_END; + _exit(static_cast(JApplication::ExitCode::Segfault)); + // _exit() is async-signal-safe, whereas exit() is not } @@ -141,9 +66,10 @@ void register_handlers(JApplication* app) { assert (app != nullptr); g_app = app; g_logger = &default_cout_logger; - g_app->GetJParameterManager()->SetDefaultParameter("jana:status_fname", g_path_to_named_pipe, - "Filename of named pipe for retrieving instantaneous status info"); - create_named_pipe(g_path_to_named_pipe); + *g_logger = app->GetJParameterManager()->GetLogger("jana"); + // Note that this updates the static default_cout_logger to match the user-provided jana:loglevel. + // It would be nice to do this in a less unexpected place, and hopefully that will naturally + // emerge from future refactorings. //Define signal action struct sigaction sSignalAction; @@ -154,10 +80,11 @@ void register_handlers(JApplication* app) { sigemptyset(&sSignalAction.sa_mask); sigaction(SIGSEGV, &sSignalAction, nullptr); - LOG_INFO(*g_logger) << "Setting signal handler USR1. Use to write status info to the named pipe." << LOG_END; + LOG_WARN(*g_logger) << "Setting signal handler USR1. Use to write status info to the named pipe." << LOG_END; signal(SIGUSR1, handle_usr1); signal(SIGUSR2, handle_usr2); - LOG_INFO(*g_logger) << "Setting signal handler SIGINT (Ctrl-C). Use a single SIGINT to enter the Inspector, or multiple SIGINTs for an immediate shutdown." << LOG_END; + signal(SIGTSTP, handle_tstp); + LOG_WARN(*g_logger) << "Setting signal handler SIGINT (Ctrl-C). Use a single SIGINT to enter the Inspector, or multiple SIGINTs for an immediate shutdown." << LOG_END; signal(SIGINT, handle_sigint); } diff --git a/src/libraries/JANA/CLI/JSignalHandler.h b/src/libraries/JANA/CLI/JSignalHandler.h index 8dba6cbe2..f485360d2 100644 --- a/src/libraries/JANA/CLI/JSignalHandler.h +++ b/src/libraries/JANA/CLI/JSignalHandler.h @@ -4,35 +4,11 @@ #pragma once -#include -#include -#include -#include -#include +class JApplication; -#include - -/// JSignalHandler bundles together the logic for querying a JApplication -/// about its JStatus with signal handlers for USR1, USR2, and CTRL-C. +/// JSignalHandler attaches signal handlers for USR1, USR2, and CTRL-C to a given JApplication instance. namespace JSignalHandler { -extern JApplication* g_app; -extern int g_sigint_count; -extern JLogger* g_logger; -extern std::string g_path_to_named_pipe; -extern std::map g_thread_reports; -extern std::atomic_int g_thread_report_count; - - -void create_named_pipe(const std::string& path_to_named_pipe); -void send_to_named_pipe(const std::string& path_to_named_pipe, const std::string& data); -void produce_thread_report(); -std::string produce_overall_report(); -void send_overall_report_to_named_pipe(); -void handle_sigint(int); -void handle_usr1(int); -void handle_usr2(int); -void handle_sigsegv(int /*signal_number*/, siginfo_t* /*signal_info*/, void* /*context*/); void register_handlers(JApplication* app); }; // namespace JSignalHandler diff --git a/src/libraries/JANA/CMakeLists.txt b/src/libraries/JANA/CMakeLists.txt index 1ca7a8800..08f5e9f17 100644 --- a/src/libraries/JANA/CMakeLists.txt +++ b/src/libraries/JANA/CMakeLists.txt @@ -4,25 +4,24 @@ set(JANA2_SOURCES JApplication.cc + JEvent.cc + JEventSource.cc JFactory.cc JFactorySet.cc JMultifactory.cc JService.cc JVersion.cc + JEvent.cc - Engine/JArrowProcessingController.cc - Engine/JScheduler.cc - Engine/JWorker.cc - Engine/JPerfMetrics.cc - Engine/JPerfSummary.cc + Engine/JExecutionEngine.cc - Topology/JEventProcessorArrow.cc + Topology/JArrow.cc Topology/JEventSourceArrow.cc Topology/JEventMapArrow.cc + Topology/JEventTapArrow.cc Topology/JTopologyBuilder.cc Services/JComponentManager.cc - Services/JLoggingService.cc Services/JParameterManager.cc Services/JPluginLoader.cc Services/JWiringService.cc @@ -38,6 +37,7 @@ set(JANA2_SOURCES Utils/JCallGraphRecorder.cc Utils/JInspector.cc Utils/JApplicationInspector.cc + Utils/JBacktrace.cc Calibrations/JCalibration.cc Calibrations/JCalibrationFile.cc diff --git a/src/libraries/JANA/Calibrations/JCalibrationManager.h b/src/libraries/JANA/Calibrations/JCalibrationManager.h index b5fe0e144..2002af403 100644 --- a/src/libraries/JANA/Calibrations/JCalibrationManager.h +++ b/src/libraries/JANA/Calibrations/JCalibrationManager.h @@ -6,10 +6,10 @@ #include #include -#include -#include +#include #include +#include "JANA/Services/JParameterManager.h" #include "JLargeCalibration.h" class JCalibrationManager : public JService { @@ -22,17 +22,20 @@ class JCalibrationManager : public JService { pthread_mutex_t m_resource_manager_mutex; std::shared_ptr m_params; - JLogger m_logger; + std::string m_url = "file://./"; std::string m_context = "default"; public: - void acquire_services(JServiceLocator *service_locator) { - // Configure our logger - m_logger = service_locator->get()->get_logger("JCalibrationManager"); + JCalibrationManager() { + pthread_mutex_init(&m_calibration_mutex, nullptr); + pthread_mutex_init(&m_resource_manager_mutex, nullptr); + + SetPrefix("jana"); + } - m_params = service_locator->get(); + void acquire_services(JServiceLocator* sl) { // Url and context may be passed in either as environment variables // or configuration parameters. Default values are used if neither is available. @@ -40,6 +43,7 @@ class JCalibrationManager : public JService { if (getenv("JANA_CALIB_URL") != nullptr) m_url = getenv("JANA_CALIB_URL"); if (getenv("JANA_CALIB_CONTEXT") != nullptr) m_context = getenv("JANA_CALIB_CONTEXT"); + m_params = sl->get(); m_params->SetDefaultParameter("JANA:CALIB_URL", m_url, "URL used to access calibration constants"); m_params->SetDefaultParameter("JANA:CALIB_CONTEXT", m_context, "Calibration context to pass on to concrete JCalibration derived class"); diff --git a/src/libraries/JANA/Compatibility/JLockService.h b/src/libraries/JANA/Compatibility/JLockService.h index 44122875b..96072b692 100644 --- a/src/libraries/JANA/Compatibility/JLockService.h +++ b/src/libraries/JANA/Compatibility/JLockService.h @@ -227,17 +227,22 @@ inline pthread_rwlock_t *JLockService::RootFillLock(JEventProcessor *proc) { /// are in use and all contending for the same root lock. You should /// only use this when filling a histogram and not for creating. Use /// RootWriteLock and RootUnLock for that. - pthread_rwlock_t *lock; pthread_rwlock_rdlock(&m_root_fill_locks_lock); auto iter = m_root_fill_rw_lock.find(proc); if (iter == m_root_fill_rw_lock.end()) { pthread_rwlock_unlock(&m_root_fill_locks_lock); - lock = new pthread_rwlock_t; pthread_rwlock_wrlock(&m_root_fill_locks_lock); - pthread_rwlock_init(lock, nullptr); - m_root_fill_rw_lock[proc] = lock; + auto iter_now = m_root_fill_rw_lock.find(proc); + if(iter_now == m_root_fill_rw_lock.end()){ + lock = new pthread_rwlock_t; + pthread_rwlock_init(lock, nullptr); + m_root_fill_rw_lock[proc] = lock; + } + else{ + lock = iter_now->second; + } } else { lock = iter->second; @@ -265,7 +270,7 @@ inline pthread_rwlock_t *JLockService::RootFillUnLock(JEventProcessor *proc) { } pthread_rwlock_t *lock = iter->second; pthread_rwlock_unlock(&m_root_fill_locks_lock); - pthread_rwlock_unlock(m_root_fill_rw_lock[proc]); + pthread_rwlock_unlock(lock); return lock; } diff --git a/src/libraries/JANA/Compatibility/JStreamLog.h b/src/libraries/JANA/Compatibility/JStreamLog.h index 2d9bc7eb7..a84be9fdf 100644 --- a/src/libraries/JANA/Compatibility/JStreamLog.h +++ b/src/libraries/JANA/Compatibility/JStreamLog.h @@ -62,3 +62,7 @@ extern JStreamLog jout; extern JStreamLog jerr; #define jendl std::endl +#define _DBG_ std::cerr<<__FILE__<<":"<<__LINE__<<" " +#define _DBG__ std::cerr<<__FILE__<<":"<<__LINE__<GetService(); } }; diff --git a/src/libraries/JANA/Components/JComponentFwd.h b/src/libraries/JANA/Components/JComponentFwd.h index 6dca0272b..2471932d2 100644 --- a/src/libraries/JANA/Components/JComponentFwd.h +++ b/src/libraries/JANA/Components/JComponentFwd.h @@ -34,6 +34,7 @@ struct JComponent { CallbackStyle m_callback_style = CallbackStyle::LegacyMode; std::string m_prefix; std::string m_plugin_name; + std::string m_logger_name; std::string m_type_name; Status m_status = Status::Uninitialized; mutable std::mutex m_mutex; @@ -72,10 +73,17 @@ struct JComponent { JEventLevel GetLevel() const { return m_level; } - std::string GetLoggerName() const { return m_prefix.empty() ? m_type_name : m_prefix; } + std::string GetLoggerName() const { + if (!m_logger_name.empty()) return m_logger_name; + if (!m_prefix.empty()) return m_prefix; + if (!m_type_name.empty()) return m_type_name; + return ""; + } std::string GetPluginName() const { return m_plugin_name; } + void SetLoggerName(std::string logger_name) { m_logger_name = std::move(logger_name); } + void SetPluginName(std::string plugin_name) { m_plugin_name = std::move(plugin_name); }; std::string GetTypeName() const { return m_type_name; } @@ -119,7 +127,7 @@ struct JComponent { class Parameter; struct ServiceBase { - virtual void Init(JApplication* app) = 0; + virtual void Fetch(JApplication* app) = 0; }; template diff --git a/src/libraries/JANA/Components/JComponentSummary.cc b/src/libraries/JANA/Components/JComponentSummary.cc index 80277cefd..7476a6cc9 100644 --- a/src/libraries/JANA/Components/JComponentSummary.cc +++ b/src/libraries/JANA/Components/JComponentSummary.cc @@ -128,10 +128,23 @@ void PrintComponentTable(std::ostream& os, const JComponentSummary& cs) { os << comp_table; } +void PrintComponentYaml(std::ostream& os, const JComponentSummary& cs) { + os << "Component Summary" << std::endl; + + for (const auto* comp : cs.GetAllComponents()) { + os << std::endl; + os << " - base: " << "\"" << comp->GetBaseName() << "\"" << std::endl; + os << " type: " << "\"" << comp->GetTypeName() << "\"" << std::endl; + os << " prefix: " << "\"" << comp->GetPrefix() << "\"" << std::endl; + os << " level: " << "\"" << comp->GetLevel() << "\"" << std::endl; + os << " plugin: " << "\"" << comp->GetPluginName() << "\"" << std::endl; + } +} + std::ostream& operator<<(std::ostream& os, JComponentSummary const& cs) { - PrintComponentTable(os,cs); + PrintComponentYaml(os,cs); return os; } diff --git a/src/libraries/JANA/Components/JComponentSummary.h b/src/libraries/JANA/Components/JComponentSummary.h index 86c09a82b..d61d596e9 100644 --- a/src/libraries/JANA/Components/JComponentSummary.h +++ b/src/libraries/JANA/Components/JComponentSummary.h @@ -160,6 +160,7 @@ std::ostream& operator<<(std::ostream& os, const JComponentSummary& cs); std::ostream& operator<<(std::ostream& os, JComponentSummary::Collection const&); std::ostream& operator<<(std::ostream& os, JComponentSummary::Component const&); void PrintComponentTable(std::ostream& os, const JComponentSummary&); +void PrintComponentYaml(std::ostream& os, const JComponentSummary&); void PrintCollectionTable(std::ostream& os, const JComponentSummary&); inline bool operator==(const JComponentSummary::Collection& lhs, const JComponentSummary::Collection& rhs) { diff --git a/src/libraries/JANA/Components/JHasInputs.h b/src/libraries/JANA/Components/JHasInputs.h index ed0836abd..c6c644cfa 100644 --- a/src/libraries/JANA/Components/JHasInputs.h +++ b/src/libraries/JANA/Components/JHasInputs.h @@ -87,6 +87,9 @@ struct JHasInputs { Input(JHasInputs* owner) { owner->RegisterInput(this); this->type_name = JTypeInfo::demangle(); + this->names.push_back(""); + // For non-PODIO inputs, these are technically tags for now, not names + this->levels.push_back(JEventLevel::None); } Input(JHasInputs* owner, const InputOptions& options) { @@ -96,6 +99,8 @@ struct JHasInputs { } const std::vector& operator()() { return m_data; } + const std::vector& operator*() { return m_data; } + const std::vector* operator->() { return &m_data; } private: @@ -103,23 +108,24 @@ struct JHasInputs { void GetCollection(const JEvent& event) { auto& level = this->levels[0]; + m_data.clear(); if (level == event.GetLevel() || level == JEventLevel::None) { - m_data = event.Get(this->names[0], !this->is_optional); + event.Get(m_data, this->names[0], !this->is_optional); } else { if (this->is_optional && !event.HasParent(level)) return; - m_data = event.GetParent(level).template Get(this->names[0], !this->is_optional); + event.GetParent(level).template Get(m_data, this->names[0], !this->is_optional); } } void PrefetchCollection(const JEvent& event) { auto& level = this->levels[0]; auto& name = this->names[0]; if (level == event.GetLevel() || level == JEventLevel::None) { - event.Get(name, !this->is_optional); + event.GetFactory(name, !this->is_optional)->Create(event.shared_from_this()); } else { if (this->is_optional && !event.HasParent(level)) return; - event.GetParent(level).template Get(name, !this->is_optional); + event.GetParent(level).template GetFactory(name, !this->is_optional)->Create(event.shared_from_this()); } } }; @@ -135,6 +141,8 @@ struct JHasInputs { PodioInput(JHasInputs* owner) { owner->RegisterInput(this); this->type_name = JTypeInfo::demangle(); + this->names.push_back(this->type_name); + this->levels.push_back(JEventLevel::None); } PodioInput(JHasInputs* owner, const InputOptions& options) { @@ -146,6 +154,12 @@ struct JHasInputs { const typename PodioT::collection_type* operator()() { return m_data; } + const typename PodioT::collection_type& operator*() { + return *m_data; + } + const typename PodioT::collection_type* operator->() { + return m_data; + } void GetCollection(const JEvent& event) { auto& level = this->levels[0]; diff --git a/src/libraries/JANA/Components/JHasOutputs.h b/src/libraries/JANA/Components/JHasOutputs.h index 461eb549a..4f84d72a1 100644 --- a/src/libraries/JANA/Components/JHasOutputs.h +++ b/src/libraries/JANA/Components/JHasOutputs.h @@ -31,6 +31,7 @@ struct JHasOutputs { template class Output : public OutputBase { std::vector m_data; + bool is_not_owner = false; public: Output(JHasOutputs* owner, std::string default_tag_name="") { @@ -43,9 +44,12 @@ struct JHasOutputs { protected: void InsertCollection(JEvent& event) override { - event.Insert(m_data, this->collection_names[0]); + auto fac = event.Insert(m_data, this->collection_names[0]); + fac->SetIsNotOwnerFlag(is_not_owner); } void Reset() override { } + + void SetIsNotOwnerFlag(bool not_owner=true) { is_not_owner = not_owner; } }; diff --git a/src/libraries/JANA/Components/JOmniFactory.h b/src/libraries/JANA/Components/JOmniFactory.h index 81da2ef6f..8f7093c45 100644 --- a/src/libraries/JANA/Components/JOmniFactory.h +++ b/src/libraries/JANA/Components/JOmniFactory.h @@ -10,14 +10,12 @@ * which might be changed by user parameters. */ +#include "JANA/Services/JParameterManager.h" #include #include #include #include -#include -#include - #include #include @@ -271,8 +269,7 @@ class JOmniFactory : public JMultifactory, public jana::components::JHasInputs { } // Obtain logger - //m_logger = m_app->GetService()->logger(m_prefix); - m_logger = m_app->GetService()->get_logger(m_prefix); + m_logger = m_app->GetService()->GetLogger(m_prefix); // Configure logger. Priority = [JParameterManager, system log level] // std::string default_log_level = eicrecon::LogLevelToString(m_logger->level()); @@ -285,7 +282,7 @@ class JOmniFactory : public JMultifactory, public jana::components::JHasInputs { parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); } for (auto* service : m_services) { - service->Init(m_app); + service->Fetch(m_app); } static_cast(this)->Configure(); } diff --git a/src/libraries/JANA/Engine/JArrowProcessingController.cc b/src/libraries/JANA/Engine/JArrowProcessingController.cc deleted file mode 100644 index 2d71611b5..000000000 --- a/src/libraries/JANA/Engine/JArrowProcessingController.cc +++ /dev/null @@ -1,296 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#include -#include -#include -#include -#include - -#include - -using millisecs = std::chrono::duration; -using secs = std::chrono::duration; - -void JArrowProcessingController::acquire_services(JServiceLocator * sl) { - auto ls = sl->get(); - m_logger = ls->get_logger("JArrowProcessingController"); - m_worker_logger = ls->get_logger("JWorker"); - m_scheduler_logger = ls->get_logger("JScheduler"); - - m_topology = sl->get(); - - // Obtain timeouts from parameter manager - auto params = sl->get(); - params->SetDefaultParameter("jana:timeout", m_timeout_s, "Max time (in seconds) JANA will wait for a thread to update its heartbeat before hard-exiting. 0 to disable timeout completely."); - params->SetDefaultParameter("jana:warmup_timeout", m_warmup_timeout_s, "Max time (in seconds) JANA will wait for 'initial' events to complete before hard-exiting."); - // Originally "THREAD_TIMEOUT" and "THREAD_TIMEOUT_FIRST_EVENT" -} - -void JArrowProcessingController::initialize() { - - m_scheduler = new JScheduler(m_topology); - m_scheduler->logger = m_scheduler_logger; - LOG_INFO(m_logger) << m_topology->mapping << LOG_END; - - m_scheduler->initialize_topology(); - -} - -/// @brief This will run the JArrowTopology and create N JWorker objects, each with thier own threads. -/// -/// This will first call JTopology::run(size_t) to initialize the topology and -/// prepare it for workers. It will then create `nthreads` JWorker objects and call -/// their JWorker::start() methods causing them to create their own threads and -/// start running their JWorker::loop() methods. This will return after the worker -/// threads are launched. -/// -/// @param [in] nthreads The number of worker threads to start -void JArrowProcessingController::run(size_t nthreads) { - LOG_INFO(m_logger) << "run(): Launching " << nthreads << " workers" << LOG_END; - // run_topology needs to happen _before_ threads are started so that threads don't quit due to lack of assignments - m_scheduler->run_topology(nthreads); - - bool pin_to_cpu = (m_topology->mapping.get_affinity() != JProcessorMapping::AffinityStrategy::None); - - size_t next_worker_id = 0; - while (next_worker_id < nthreads) { - size_t next_cpu_id = m_topology->mapping.get_cpu_id(next_worker_id); - size_t next_loc_id = m_topology->mapping.get_loc_id(next_worker_id); - auto worker = new JWorker(this, m_scheduler, next_worker_id, next_cpu_id, next_loc_id, pin_to_cpu); - worker->logger = m_worker_logger; - m_workers.push_back(worker); - next_worker_id++; - } - for (size_t i=0; istart(); - }; - // It's tempting to put a barrier here so that JAPC::run() blocks until all workers have entered loop(). - // The reason it doesn't work is that the topology might exit immediately (or close to immediately), leaving - // the supervisor thread waiting forever for workers to reach RunState::Running when they've already Stopped. -} - -void JArrowProcessingController::scale(size_t nthreads) { - - LOG_INFO(m_logger) << "scale(): Stopping all running workers" << LOG_END; - m_scheduler->request_topology_pause(); - for (JWorker* worker : m_workers) { - worker->wait_for_stop(); - } - m_scheduler->achieve_topology_pause(); - - LOG_INFO(m_logger) << "scale(): All workers are stopped" << LOG_END; - bool pin_to_cpu = (m_topology->mapping.get_affinity() != JProcessorMapping::AffinityStrategy::None); - size_t next_worker_id = m_workers.size(); - - while (next_worker_id < nthreads) { - - size_t next_cpu_id = m_topology->mapping.get_cpu_id(next_worker_id); - size_t next_loc_id = m_topology->mapping.get_loc_id(next_worker_id); - - auto worker = new JWorker(this, m_scheduler, next_worker_id, next_cpu_id, next_loc_id, pin_to_cpu); - worker->logger = m_worker_logger; - m_workers.push_back(worker); - next_worker_id++; - } - - LOG_INFO(m_logger) << "scale(): Restarting " << nthreads << " workers" << LOG_END; - // topology->run needs to happen _before_ threads are started so that threads don't quit due to lack of assignments - m_scheduler->run_topology(nthreads); - - for (size_t i=0; istart(); - }; -} - -void JArrowProcessingController::request_pause() { - m_scheduler->request_topology_pause(); - // Or: - // for (JWorker* worker : m_workers) { - // worker->request_stop(); - // } -} - -void JArrowProcessingController::wait_until_paused() { - for (JWorker* worker : m_workers) { - worker->wait_for_stop(); - } - // Join all the worker threads. - // Do not trigger the pause (we may want the pause to come internally, e.g. from an event source running out.) - // Do NOT finish() the topology (we want the ability to be able to restart it) - m_scheduler->achieve_topology_pause(); -} - -void JArrowProcessingController::request_stop() { - // Shut off the sources; the workers will stop on their own once they run out of assignments. - // Unlike request_pause, this drains all queues. - // Conceivably, a user could request_stop() followed by wait_until_paused(), which would drain all queues but - // leave the topology in a restartable state. This suggests we might want to rename some of these things. - // e.g. request_stop => drain OR drain_then_pause - // request_pause => pause - // wait_until_stop => join_then_finish - // wait_until_pause => join - m_scheduler->drain_topology(); -} - -void JArrowProcessingController::wait_until_stopped() { - // Join all workers - for (JWorker* worker : m_workers) { - worker->wait_for_stop(); - } - // finish out the topology - // (note some arrows might have already finished e.g. event sources, but that's fine, finish() is idempotent) - m_scheduler->achieve_topology_pause(); - m_scheduler->finish_topology(); -} - -bool JArrowProcessingController::is_stopped() { - return m_scheduler->get_topology_status() == JScheduler::TopologyStatus::Paused; -} - -bool JArrowProcessingController::is_finished() { - return m_scheduler->get_topology_status() == JScheduler::TopologyStatus::Finalized; -} - -bool JArrowProcessingController::is_timed_out() { - if (m_timeout_s == 0) return false; - - // Note that this makes its own (redundant) call to measure_internal_performance(). - // Probably want to refactor so that we only make one such call per ticker iteration. - // Since we are storing our metrics summary anyway, we could call measure_performance() - // and have print_report(), print_final_report(), is_timed_out(), etc use the cached version - auto metrics = measure_performance(); - - int timeout_s; - if (metrics->total_uptime_s < m_warmup_timeout_s * m_topology->m_event_pool_size / metrics->thread_count) { - // We are at the beginning and not all events have necessarily had a chance to warm up - timeout_s = m_warmup_timeout_s; - } - else if (!m_topology->m_limit_total_events_in_flight) { - // New events are constantly emitted, each of which may contain jfactorysets which need to be warmed up - timeout_s = m_warmup_timeout_s; - } - else { - timeout_s = m_timeout_s; - } - - // Find all workers whose last heartbeat exceeds timeout - bool found_timeout = false; - for (size_t i=0; iworkers.size(); ++i) { - if (metrics->workers[i].last_heartbeat_ms > (timeout_s * 1000)) { - found_timeout = true; - m_workers[i]->declare_timeout(); - // This assumes the workers and their summaries are ordered the same. - // Which is true, but I don't like it. - } - } - return found_timeout; -} - -bool JArrowProcessingController::is_excepted() { - for (auto worker : m_workers) { - if (worker->get_runstate() == JWorker::RunState::Excepted) { - return true; - } - } - return false; -} - -std::vector JArrowProcessingController::get_exceptions() const { - std::vector exceptions; - for (auto worker : m_workers) { - if (worker->get_runstate() == JWorker::RunState::Excepted) { - exceptions.push_back(worker->get_exception()); - } - } - return exceptions; -} - -JArrowProcessingController::~JArrowProcessingController() { - - for (JWorker* worker : m_workers) { - worker->request_stop(); - } - for (JWorker* worker : m_workers) { - worker->wait_for_stop(); - } - for (JWorker* worker : m_workers) { - delete worker; - } - delete m_scheduler; -} - -JArrowMetrics::Status JArrowProcessingController::execute_arrow(int arrow_index) { - auto arrow = m_scheduler->checkout(arrow_index); - if (arrow == nullptr) return JArrowMetrics::Status::Error; - JArrowMetrics metrics; - arrow->execute(metrics, 0); - m_scheduler->last_assignment(0, arrow, metrics.get_last_status()); - return metrics.get_last_status(); -} - -void JArrowProcessingController::print_report() { - auto metrics = measure_performance(); - LOG_INFO(m_logger) << "Running" << *metrics << LOG_END; -} - -void JArrowProcessingController::print_final_report() { - auto metrics = measure_performance(); - LOG_INFO(m_logger) << "Final Report" << *metrics << LOG_END; -} - -std::unique_ptr JArrowProcessingController::measure_performance() { - - // Measure perf on all Workers first, as this will prompt them to publish - // any ArrowMetrics they have collected - if (m_perf_summary.workers.size() != m_workers.size()) { - m_perf_summary.workers = std::vector(m_workers.size()); - } - for (size_t i=0; imeasure_perf(m_perf_summary.workers[i]); - } - - size_t monotonic_event_count = 0; - for (JArrow* arrow : m_topology->arrows) { - if (arrow->is_sink()) { - monotonic_event_count += arrow->get_metrics().get_total_message_count(); - } - } - - // Uptime - m_topology->metrics.split(monotonic_event_count); - m_topology->metrics.summarize(m_perf_summary); - - double worst_seq_latency = 0; - double worst_par_latency = 0; - - m_scheduler->summarize_arrows(m_perf_summary.arrows); - - - // Figure out what the bottlenecks in this topology are - for (const ArrowSummary& summary : m_perf_summary.arrows) { - if (summary.is_parallel) { - worst_par_latency = std::max(worst_par_latency, summary.avg_latency_ms); - } else { - worst_seq_latency = std::max(worst_seq_latency, summary.avg_latency_ms); - } - } - - // bottlenecks - m_perf_summary.avg_seq_bottleneck_hz = 1e3 / worst_seq_latency; - m_perf_summary.avg_par_bottleneck_hz = 1e3 * m_perf_summary.thread_count / worst_par_latency; - - auto tighter_bottleneck = std::min(m_perf_summary.avg_seq_bottleneck_hz, m_perf_summary.avg_par_bottleneck_hz); - - m_perf_summary.avg_efficiency_frac = (tighter_bottleneck == 0) - ? std::numeric_limits::infinity() - : m_perf_summary.avg_throughput_hz / tighter_bottleneck; - - return std::unique_ptr(new JPerfSummary(m_perf_summary)); -} - - - - - diff --git a/src/libraries/JANA/Engine/JArrowProcessingController.h b/src/libraries/JANA/Engine/JArrowProcessingController.h deleted file mode 100644 index 05ef6fb22..000000000 --- a/src/libraries/JANA/Engine/JArrowProcessingController.h +++ /dev/null @@ -1,61 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once - -#include -#include -#include - -#include - -class JArrowProcessingController : public JService { -public: - - ~JArrowProcessingController() override; - void acquire_services(JServiceLocator *) override; - - void initialize(); - void run(size_t nthreads); - void scale(size_t nthreads); - void request_pause(); - void wait_until_paused(); - void request_stop(); - void wait_until_stopped(); - - bool is_stopped(); - bool is_finished(); - bool is_timed_out(); - bool is_excepted(); - - JArrowMetrics::Status execute_arrow(int); - - std::vector get_exceptions() const; - - std::unique_ptr measure_performance(); - - void print_report(); - void print_final_report(); - - // This is so we can test - inline JScheduler* get_scheduler() { return m_scheduler; } - - -private: - std::shared_ptr m_topology; - - using jclock_t = std::chrono::steady_clock; - int m_timeout_s = 8; - int m_warmup_timeout_s = 30; - - JPerfSummary m_perf_summary; - JScheduler* m_scheduler = nullptr; - - std::vector m_workers; - JLogger m_logger; - JLogger m_worker_logger; - JLogger m_scheduler_logger; - -}; - diff --git a/src/libraries/JANA/Engine/JExecutionEngine.cc b/src/libraries/JANA/Engine/JExecutionEngine.cc new file mode 100644 index 000000000..c9ae36edf --- /dev/null +++ b/src/libraries/JANA/Engine/JExecutionEngine.cc @@ -0,0 +1,783 @@ + +#include "JExecutionEngine.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +thread_local int jana2_worker_id = -1; +thread_local JBacktrace* jana2_worker_backtrace = nullptr; + +void JExecutionEngine::Init() { + auto params = GetApplication()->GetJParameterManager(); + + params->SetDefaultParameter("jana:timeout", m_timeout_s, + "Max time (in seconds) JANA will wait for a thread to update its heartbeat before hard-exiting. 0 to disable timeout completely."); + + params->SetDefaultParameter("jana:warmup_timeout", m_warmup_timeout_s, + "Max time (in seconds) JANA will wait for 'initial' events to complete before hard-exiting."); + + params->SetDefaultParameter("jana:backoff_interval", m_backoff_ms, + "Max time (in seconds) JANA will wait for 'initial' events to complete before hard-exiting."); + + params->SetDefaultParameter("jana:show_ticker", m_show_ticker, "Controls whether the ticker is visible"); + + params->SetDefaultParameter("jana:ticker_interval", m_ticker_ms, "Controls the ticker interval (in ms)"); + + auto p = params->SetDefaultParameter("jana:status_fname", m_path_to_named_pipe, + "Filename of named pipe for retrieving instantaneous status info"); + + size_t pid = getpid(); + mkfifo(m_path_to_named_pipe.c_str(), 0666); + + LOG_WARN(GetLogger()) << "To pause processing and inspect, press Ctrl-C." << LOG_END; + LOG_WARN(GetLogger()) << "For a clean shutdown, press Ctrl-C twice." << LOG_END; + LOG_WARN(GetLogger()) << "For a hard shutdown, press Ctrl-C three times." << LOG_END; + + if (p->IsDefault()) { + LOG_WARN(GetLogger()) << "For worker status information, press Ctrl-Z, or run `jana-status " << pid << "`" << LOG_END; + } + else { + LOG_WARN(GetLogger()) << "For worker status information, press Ctrl-Z, or run `jana-status " << pid << " " << m_path_to_named_pipe << "`" << LOG_END; + } + + + // Not sure how I feel about putting this here yet, but I think it will at least work in both cases it needs to. + // The reason this works is because JTopologyBuilder::create_topology() has already been called before + // JApplication::ProvideService(). + for (JArrow* arrow : m_topology->arrows) { + + arrow->initialize(); + + m_arrow_states.emplace_back(); + auto& arrow_state = m_arrow_states.back(); + arrow_state.is_source = arrow->is_source(); + arrow_state.is_sink = arrow->is_sink(); + arrow_state.is_parallel = arrow->is_parallel(); + arrow_state.next_input = arrow->get_next_port_index(); + } +} + + +void JExecutionEngine::RunTopology() { + std::unique_lock lock(m_mutex); + + if (m_runstatus == RunStatus::Failed) { + throw JException("Cannot switch topology runstatus to Running because it is already Failed"); + } + if (m_runstatus == RunStatus::Finished) { + throw JException("Cannot switch topology runstatus to Running because it is already Finished"); + } + + // Set start time and event count + m_time_at_start = clock_t::now(); + m_event_count_at_start = m_event_count_at_finish; + + // Reactivate topology + for (auto& arrow: m_arrow_states) { + if (arrow.status == ArrowState::Status::Paused) { + arrow.status = ArrowState::Status::Running; + } + } + + m_runstatus = RunStatus::Running; + + lock.unlock(); + m_condvar.notify_one(); +} + +void JExecutionEngine::ScaleWorkers(size_t nthreads) { + // We both create and destroy the pool of workers here. They all sleep until they + // receive work from the scheduler, which won't happen until the runstatus <- {Running, + // Pausing, Draining} and there is a task ready to execute. This way worker creation/destruction + // is decoupled from topology execution. + + // If we scale to zero, no workers will run. This is useful for testing, and also for using + // an external thread team, should the need arise. + + std::unique_lock lock(m_mutex); + + auto prev_nthreads = m_worker_states.size(); + + if (prev_nthreads < nthreads) { + // We are launching additional worker threads + LOG_DEBUG(GetLogger()) << "Scaling up to " << nthreads << " worker threads" << LOG_END; + for (size_t worker_id=prev_nthreads; worker_id < nthreads; ++worker_id) { + auto worker = std::make_unique(); + worker->worker_id = worker_id; + worker->is_stop_requested = false; + worker->cpu_id = m_topology->mapping.get_cpu_id(worker_id); + worker->location_id = m_topology->mapping.get_loc_id(worker_id); + worker->thread = new std::thread(&JExecutionEngine::RunWorker, this, Worker{worker_id, &worker->backtrace}); + LOG_DEBUG(GetLogger()) << "Launching worker thread " << worker_id << " on cpu=" << worker->cpu_id << ", location=" << worker->location_id << LOG_END; + m_worker_states.push_back(std::move(worker)); + + bool pin_to_cpu = (m_topology->mapping.get_affinity() != JProcessorMapping::AffinityStrategy::None); + if (pin_to_cpu) { + JCpuInfo::PinThreadToCpu(worker->thread, worker->cpu_id); + } + } + } + + else if (prev_nthreads > nthreads) { + // We are destroying existing worker threads + LOG_DEBUG(GetLogger()) << "Scaling down to " << nthreads << " worker threads" << LOG_END; + + // Signal to threads that they need to terminate. + for (int worker_id=prev_nthreads-1; worker_id >= (int)nthreads; --worker_id) { + LOG_DEBUG(GetLogger()) << "Stopping worker " << worker_id << LOG_END; + m_worker_states[worker_id]->is_stop_requested = true; + } + lock.unlock(); + + m_condvar.notify_all(); // Wake up all threads so that they can exit the condvar wait loop + + // We join all (eligible) threads _outside_ of the mutex + for (int worker_id=prev_nthreads-1; worker_id >= (int) nthreads; --worker_id) { + if (m_worker_states[worker_id]->thread != nullptr) { + if (m_worker_states[worker_id]->is_timed_out) { + // Thread has timed out. Rather than non-cooperatively killing it, + // we relinquish ownership of it but remember that it was ours once and + // is still out there, somewhere, biding its time + m_worker_states[worker_id]->thread->detach(); + LOG_DEBUG(GetLogger()) << "Detached worker " << worker_id << LOG_END; + } + else { + LOG_DEBUG(GetLogger()) << "Joining worker " << worker_id << LOG_END; + m_worker_states[worker_id]->thread->join(); + LOG_DEBUG(GetLogger()) << "Joined worker " << worker_id << LOG_END; + } + } + else { + LOG_DEBUG(GetLogger()) << "Skipping worker " << worker_id << LOG_END; + } + } + + lock.lock(); + // We retake the mutex so we can safely modify m_worker_states + for (int worker_id=prev_nthreads-1; worker_id >= (int)nthreads; --worker_id) { + if (m_worker_states.back()->thread != nullptr) { + delete m_worker_states.back()->thread; + } + m_worker_states.pop_back(); + } + } +} + +void JExecutionEngine::PauseTopology() { + std::unique_lock lock(m_mutex); + if (m_runstatus != RunStatus::Running) return; + m_runstatus = RunStatus::Pausing; + for (auto& arrow: m_arrow_states) { + if (arrow.status == ArrowState::Status::Running) { + arrow.status = ArrowState::Status::Paused; + } + } + LOG_WARN(GetLogger()) << "Requested pause" << LOG_END; + lock.unlock(); + m_condvar.notify_all(); +} + +void JExecutionEngine::DrainTopology() { + std::unique_lock lock(m_mutex); + if (m_runstatus != RunStatus::Running) return; + m_runstatus = RunStatus::Draining; + for (auto& arrow: m_arrow_states) { + if (arrow.is_source) { + if (arrow.status == ArrowState::Status::Running) { + arrow.status = ArrowState::Status::Paused; + } + } + } + LOG_WARN(GetLogger()) << "Requested drain" << LOG_END; + lock.unlock(); + m_condvar.notify_all(); +} + +void JExecutionEngine::RunSupervisor() { + + m_interrupt_status = InterruptStatus::NoInterruptsSupervised; + size_t last_event_count = 0; + clock_t::time_point last_measurement_time = clock_t::now(); + + Perf perf; + while (true) { + + if (m_enable_timeout) { + CheckTimeout(); + } + + if (m_print_worker_report_requested) { + PrintWorkerReport(false); + m_print_worker_report_requested = false; + } + + if (m_send_worker_report_requested) { + PrintWorkerReport(true); + m_send_worker_report_requested = false; + } + + perf = GetPerf(); + if ((perf.runstatus == RunStatus::Paused && m_interrupt_status != InterruptStatus::InspectRequested) || + perf.runstatus == RunStatus::Finished || + perf.runstatus == RunStatus::Failed) { + break; + } + + if (m_interrupt_status == InterruptStatus::InspectRequested) { + if (perf.runstatus == RunStatus::Paused) { + PrintFinalReport(); + LOG_INFO(GetLogger()) << "Entering inspector" << LOG_END; + m_enable_timeout = false; + m_interrupt_status = InterruptStatus::InspectInProgress; + InspectApplication(GetApplication()); + m_interrupt_status = InterruptStatus::NoInterruptsSupervised; + } + else { + PauseTopology(); + } + } + else if (m_interrupt_status == InterruptStatus::PauseAndQuit) { + PauseTopology(); + } + + if (m_show_ticker) { + auto last_measurement_duration_ms = std::chrono::duration_cast(clock_t::now() - last_measurement_time).count(); + float latest_throughput_hz = (last_measurement_duration_ms == 0) ? 0 : (perf.event_count - last_event_count) * 1000.0 / last_measurement_duration_ms; + last_measurement_time = clock_t::now(); + last_event_count = perf.event_count; + + // Print rates + LOG_WARN(m_logger) << "Status: " << perf.event_count << " events processed at " + << JTypeInfo::to_string_with_si_prefix(latest_throughput_hz) << "Hz (" + << JTypeInfo::to_string_with_si_prefix(perf.throughput_hz) << "Hz avg)" << LOG_END; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(m_ticker_ms)); + } + LOG_INFO(GetLogger()) << "Processing paused." << LOG_END; + + if (perf.runstatus == RunStatus::Failed) { + HandleFailures(); + } + + PrintFinalReport(); +} + +bool JExecutionEngine::CheckTimeout() { + std::unique_lock lock(m_mutex); + auto now = clock_t::now(); + bool timeout_detected = false; + for (auto& worker: m_worker_states) { + auto timeout_s = (worker->is_event_warmed_up) ? m_timeout_s : m_warmup_timeout_s; + auto duration_s = std::chrono::duration_cast(now - worker->last_checkout_time).count(); + if (duration_s > timeout_s) { + worker->is_timed_out = true; + timeout_detected = true; + m_runstatus = RunStatus::Failed; + } + } + return timeout_detected; +} + +void JExecutionEngine::HandleFailures() { + + std::unique_lock lock(m_mutex); + + // First, we log all of the failures we've found + for (auto& worker: m_worker_states) { + const auto& arrow_name = m_topology->arrows[worker->last_arrow_id]->get_name(); + if (worker->is_timed_out) { + LOG_FATAL(GetLogger()) << "Timeout in worker thread " << worker->worker_id << " while executing " << arrow_name << " on event #" << worker->last_event_nr << LOG_END; + pthread_kill(worker->thread->native_handle(), SIGUSR2); + LOG_INFO(GetLogger()) << "Worker thread signalled; waiting for backtrace capture." << LOG_END; + worker->backtrace.WaitForCapture(); + } + if (worker->stored_exception != nullptr) { + LOG_FATAL(GetLogger()) << "Exception in worker thread " << worker->worker_id << " while executing " << arrow_name << " on event #" << worker->last_event_nr << LOG_END; + } + } + + // Now we throw each of these exceptions in order, in case the caller is going to attempt to catch them. + // In reality all callers are going to print everything they can about the exception and exit. + for (auto& worker: m_worker_states) { + if (worker->stored_exception != nullptr) { + GetApplication()->SetExitCode((int) JApplication::ExitCode::UnhandledException); + std::rethrow_exception(worker->stored_exception); + } + if (worker->is_timed_out) { + GetApplication()->SetExitCode((int) JApplication::ExitCode::Timeout); + auto ex = JException("Timeout in worker thread"); + ex.stacktrace = worker->backtrace.ToString(); + throw ex; + } + } +} + +void JExecutionEngine::FinishTopology() { + std::unique_lock lock(m_mutex); + assert(m_runstatus == RunStatus::Paused); + + LOG_DEBUG(GetLogger()) << "Finishing processing..." << LOG_END; + for (auto* arrow : m_topology->arrows) { + arrow->finalize(); + } + for (auto* pool: m_topology->pools) { + pool->Finalize(); + } + m_runstatus = RunStatus::Finished; + LOG_INFO(GetLogger()) << "Finished processing." << LOG_END; +} + +JExecutionEngine::RunStatus JExecutionEngine::GetRunStatus() { + std::unique_lock lock(m_mutex); + return m_runstatus; +} + +JExecutionEngine::Perf JExecutionEngine::GetPerf() { + std::unique_lock lock(m_mutex); + Perf result; + if (m_runstatus == RunStatus::Paused || m_runstatus == RunStatus::Failed) { + result.event_count = m_event_count_at_finish - m_event_count_at_start; + result.uptime_ms = std::chrono::duration_cast(m_time_at_finish - m_time_at_start).count(); + } + else { + // Obtain current event count + size_t current_event_count = 0; + for (auto& state : m_arrow_states) { + if (state.is_sink) { + current_event_count += state.events_processed; + } + } + result.event_count = current_event_count - m_event_count_at_start; + result.uptime_ms = std::chrono::duration_cast(clock_t::now() - m_time_at_start).count(); + } + result.runstatus = m_runstatus; + result.thread_count = m_worker_states.size(); + result.throughput_hz = (result.uptime_ms == 0) ? 0 : (result.event_count * 1000.0) / result.uptime_ms; + result.event_level = JEventLevel::PhysicsEvent; + return result; +} + +JExecutionEngine::Worker JExecutionEngine::RegisterWorker() { + std::unique_lock lock(m_mutex); + auto worker_id = m_worker_states.size(); + auto worker = std::make_unique(); + worker->worker_id = worker_id; + worker->is_stop_requested = false; + worker->cpu_id = m_topology->mapping.get_cpu_id(worker_id); + worker->location_id = m_topology->mapping.get_loc_id(worker_id); + worker->thread = nullptr; + m_worker_states.push_back(std::move(worker)); + + bool pin_to_cpu = (m_topology->mapping.get_affinity() != JProcessorMapping::AffinityStrategy::None); + if (pin_to_cpu) { + JCpuInfo::PinThreadToCpu(worker->thread, worker->cpu_id); + } + return {worker_id, &worker->backtrace}; + +} + + +void JExecutionEngine::RunWorker(Worker worker) { + + LOG_DEBUG(GetLogger()) << "Launched worker thread " << worker.worker_id << LOG_END; + jana2_worker_id = worker.worker_id; + jana2_worker_backtrace = worker.backtrace; + try { + Task task; + while (true) { + ExchangeTask(task, worker.worker_id); + if (task.arrow == nullptr) break; // Exit as soon as ExchangeTask() stops blocking + task.arrow->fire(task.input_event, task.outputs, task.output_count, task.status); + } + LOG_DEBUG(GetLogger()) << "Stopped worker thread " << worker.worker_id << LOG_END; + } + catch (...) { + LOG_ERROR(GetLogger()) << "Exception on worker thread " << worker.worker_id << LOG_END; + std::unique_lock lock(m_mutex); + m_runstatus = RunStatus::Failed; + m_worker_states.at(worker.worker_id)->stored_exception = std::current_exception(); + } +} + + +void JExecutionEngine::ExchangeTask(Task& task, size_t worker_id, bool nonblocking) { + + auto checkin_time = std::chrono::steady_clock::now(); + // It's important to start measuring this _before_ acquiring the lock because acquiring the lock + // may be a big part of the scheduler overhead + + std::unique_lock lock(m_mutex); + + auto& worker = *m_worker_states.at(worker_id); + + if (task.arrow != nullptr) { + CheckinCompletedTask_Unsafe(task, worker, checkin_time); + } + + if (worker.is_stop_requested) { + return; + } + + FindNextReadyTask_Unsafe(task, worker); + + if (nonblocking) { return; } + auto idle_time_start = clock_t::now(); + m_total_scheduler_duration += (idle_time_start - checkin_time); + + while (task.arrow == nullptr && !worker.is_stop_requested) { + m_condvar.wait(lock); + FindNextReadyTask_Unsafe(task, worker); + } + worker.last_checkout_time = clock_t::now(); + + if (task.input_event != nullptr) { + worker.last_event_nr = task.input_event->GetEventNumber(); + } + else { + worker.last_event_nr = 0; + } + m_total_idle_duration += (worker.last_checkout_time - idle_time_start); + + lock.unlock(); + // Notify one worker, who will notify the next, etc, as long as FindNextReadyTaskUnsafe() succeeds. + // After FindNextReadyTaskUnsafe fails, all threads block until the next returning worker reactivates the + // notification chain. + m_condvar.notify_one(); +} + + +void JExecutionEngine::CheckinCompletedTask_Unsafe(Task& task, WorkerState& worker, clock_t::time_point checkin_time) { + + auto processing_duration = checkin_time - worker.last_checkout_time; + + ArrowState& arrow_state = m_arrow_states.at(worker.last_arrow_id); + + arrow_state.active_tasks -= 1; + arrow_state.total_processing_duration += processing_duration; + + for (size_t output=0; outputget_port(task.outputs[output].second).is_input) { + arrow_state.events_processed++; + } + } + + // Put each output in its correct queue or pool + task.arrow->push(task.outputs, task.output_count, worker.location_id); + + if (task.status == JArrow::FireResult::Finished) { + // If this is an eventsource self-terminating (the only thing that returns Status::Finished right now) it will + // have already called DoClose(). I'm tempted to always call DoClose() as part of JExecutionEngine::Finish() instead, however. + + // Mark arrow as finished + arrow_state.status = ArrowState::Status::Finished; + + // Check if this switches the topology to Draining() + if (m_runstatus == RunStatus::Running) { + bool draining = true; + for (auto& arrow: m_arrow_states) { + if (arrow.is_source && arrow.status == ArrowState::Status::Running) { + draining = false; + } + } + if (draining) { + m_runstatus = RunStatus::Draining; + } + } + } + worker.last_arrow_id = -1; + worker.last_event_nr = 0; + + task.arrow = nullptr; + task.input_event = nullptr; + task.output_count = 0; + task.status = JArrow::FireResult::NotRunYet; +}; + + +void JExecutionEngine::FindNextReadyTask_Unsafe(Task& task, WorkerState& worker) { + + if (m_runstatus == RunStatus::Running || m_runstatus == RunStatus::Draining) { + // We only pick up a new task if the topology is running or draining. + + for (size_t arrow_id=0; arrow_idarrows[arrow_id]; + // TODO: consider setting state.next_input, retrieving via fire() + auto port = arrow->get_next_port_index(); + JEvent* event = (port == -1) ? nullptr : arrow->pull(port, worker.location_id); + if (event != nullptr || port == -1) { + // We've found a task that is ready! + state.active_tasks += 1; + + task.arrow = arrow; + task.input_port = port; + task.input_event = event; + task.output_count = 0; + task.status = JArrow::FireResult::NotRunYet; + + worker.last_arrow_id = arrow_id; + if (event != nullptr) { + worker.is_event_warmed_up = event->IsWarmedUp(); + worker.last_event_nr = event->GetEventNumber(); + } + else { + worker.is_event_warmed_up = true; // Use shorter timeout + worker.last_event_nr = 0; + } + return; + } + } + } + + // Because we reached this point, we know that there aren't any tasks ready, + // so we check whether more are potentially coming. If not, we can pause the topology. + // Note that our worker threads will still wait at ExchangeTask() until they get + // shut down separately during Scale(). + + bool any_active_source_found = false; + bool any_active_task_found = false; + + LOG_DEBUG(GetLogger()) << "Scheduler: No tasks ready" << LOG_END; + + for (size_t arrow_id = 0; arrow_id < m_arrow_states.size(); ++arrow_id) { + auto& state = m_arrow_states[arrow_id]; + auto* arrow = m_topology->arrows[arrow_id]; + LOG_TRACE(GetLogger()) << "Scheduler: arrow=" << arrow->get_name() << ", is_source=" << state.is_source << ", active_tasks=" << state.active_tasks << ", is_parallel=" << state.is_parallel << LOG_END; + any_active_source_found |= (state.status == ArrowState::Status::Running && state.is_source); + any_active_task_found |= (state.active_tasks != 0); + // A source might have been deactivated by RequestPause, Ctrl-C, etc, and might be inactive even though it still has active tasks + } + + if (!any_active_source_found && !any_active_task_found) { + // Pause the topology + m_time_at_finish = clock_t::now(); + m_event_count_at_finish = 0; + for (auto& arrow_state : m_arrow_states) { + if (arrow_state.is_sink) { + m_event_count_at_finish += arrow_state.events_processed; + } + } + LOG_DEBUG(GetLogger()) << "Processing paused" << LOG_END; + m_runstatus = RunStatus::Paused; + // I think this is the ONLY site where the topology gets paused. Verify this? + } + + worker.last_arrow_id = -1; + + task.arrow = nullptr; + task.input_port = -1; + task.input_event = nullptr; + task.output_count = 0; + task.status = JArrow::FireResult::NotRunYet; +} + + +void JExecutionEngine::PrintFinalReport() { + + std::unique_lock lock(m_mutex); + auto event_count = m_event_count_at_finish - m_event_count_at_start; + auto uptime_ms = std::chrono::duration_cast(m_time_at_finish - m_time_at_start).count(); + auto thread_count = m_worker_states.size(); + auto throughput_hz = (event_count * 1000.0) / uptime_ms; + + LOG_INFO(GetLogger()) << "Detailed report:" << LOG_END; + LOG_INFO(GetLogger()) << LOG_END; + LOG_INFO(GetLogger()) << " Avg throughput [Hz]: " << std::setprecision(3) << throughput_hz << LOG_END; + LOG_INFO(GetLogger()) << " Completed events [count]: " << event_count << LOG_END; + LOG_INFO(GetLogger()) << " Total uptime [s]: " << std::setprecision(4) << uptime_ms/1000.0 << LOG_END; + LOG_INFO(GetLogger()) << " Thread team size [count]: " << thread_count << LOG_END; + LOG_INFO(GetLogger()) << LOG_END; + LOG_INFO(GetLogger()) << " Arrow-level metrics:" << LOG_END; + LOG_INFO(GetLogger()) << LOG_END; + + size_t total_useful_ms = 0; + + for (size_t arrow_id=0; arrow_id < m_arrow_states.size(); ++arrow_id) { + auto* arrow = m_topology->arrows[arrow_id]; + auto& arrow_state = m_arrow_states[arrow_id]; + auto useful_ms = std::chrono::duration_cast(arrow_state.total_processing_duration).count(); + total_useful_ms += useful_ms; + auto avg_latency = useful_ms*1.0/arrow_state.events_processed; + auto throughput_bottleneck = 1000.0 / avg_latency; + if (arrow->is_parallel()) { + throughput_bottleneck *= thread_count; + } + + LOG_INFO(GetLogger()) << " - Arrow name: " << arrow->get_name() << LOG_END; + LOG_INFO(GetLogger()) << " Parallel: " << arrow->is_parallel() << LOG_END; + LOG_INFO(GetLogger()) << " Events completed: " << arrow_state.events_processed << LOG_END; + LOG_INFO(GetLogger()) << " Avg latency [ms/event]: " << avg_latency << LOG_END; + LOG_INFO(GetLogger()) << " Throughput bottleneck [Hz]: " << throughput_bottleneck << LOG_END; + LOG_INFO(GetLogger()) << LOG_END; + } + + auto total_scheduler_ms = std::chrono::duration_cast(m_total_scheduler_duration).count(); + auto total_idle_ms = std::chrono::duration_cast(m_total_idle_duration).count(); + + LOG_INFO(GetLogger()) << " Total useful time [s]: " << std::setprecision(6) << total_useful_ms/1000.0 << LOG_END; + LOG_INFO(GetLogger()) << " Total scheduler time [s]: " << std::setprecision(6) << total_scheduler_ms/1000.0 << LOG_END; + LOG_INFO(GetLogger()) << " Total idle time [s]: " << std::setprecision(6) << total_idle_ms/1000.0 << LOG_END; + + LOG_INFO(GetLogger()) << LOG_END; + + LOG_WARN(GetLogger()) << "Final report: " << event_count << " events processed at " + << JTypeInfo::to_string_with_si_prefix(throughput_hz) << "Hz" << LOG_END; + +} + +void JExecutionEngine::SetTickerEnabled(bool show_ticker) { + m_show_ticker = show_ticker; +} + +bool JExecutionEngine::IsTickerEnabled() const { + return m_show_ticker; +} + +void JExecutionEngine::SetTimeoutEnabled(bool timeout_enabled) { + m_enable_timeout = timeout_enabled; +} + +bool JExecutionEngine::IsTimeoutEnabled() const { + return m_enable_timeout; +} + +JArrow::FireResult JExecutionEngine::Fire(size_t arrow_id, size_t location_id) { + + std::unique_lock lock(m_mutex); + JArrow* arrow = m_topology->arrows[arrow_id]; + + ArrowState& arrow_state = m_arrow_states[arrow_id]; + if (arrow_state.status == ArrowState::Status::Finished) { + return JArrow::FireResult::Finished; + } + if (!arrow_state.is_parallel && arrow_state.active_tasks != 0) { + return JArrow::FireResult::NotRunYet; + } + arrow_state.active_tasks += 1; + + auto port = arrow->get_next_port_index(); + JEvent* event = nullptr; + if (port != -1) { + event = arrow->pull(port, location_id); + } + lock.unlock(); + + size_t output_count; + JArrow::OutputData outputs; + JArrow::FireResult result = JArrow::FireResult::NotRunYet; + + if (event != nullptr || port == -1) { + arrow->fire(event, outputs, output_count, result); + lock.lock(); + arrow->push(outputs, output_count, location_id); + arrow_state.active_tasks -= 1; + lock.unlock(); + } + + return result; +} + + +void JExecutionEngine::HandleSIGINT() { + InterruptStatus status = m_interrupt_status; + std::cout << std::endl; + switch (status) { + case InterruptStatus::NoInterruptsSupervised: m_interrupt_status = InterruptStatus::InspectRequested; break; + case InterruptStatus::InspectRequested: m_interrupt_status = InterruptStatus::PauseAndQuit; break; + case InterruptStatus::NoInterruptsUnsupervised: + case InterruptStatus::PauseAndQuit: + case InterruptStatus::InspectInProgress: + _exit(-2); + } +} + +void JExecutionEngine::HandleSIGUSR1() { + m_send_worker_report_requested = true; +} + +void JExecutionEngine::HandleSIGUSR2() { + if (jana2_worker_backtrace != nullptr) { + jana2_worker_backtrace->Capture(3); + } +} + +void JExecutionEngine::HandleSIGTSTP() { + std::cout << std::endl; + m_print_worker_report_requested = true; +} + +void JExecutionEngine::PrintWorkerReport(bool send_to_pipe) { + + std::unique_lock lock(m_mutex); + LOG_INFO(GetLogger()) << "Generating worker report. It may take some time to retrieve each symbol's debug information." << LOG_END; + for (auto& worker: m_worker_states) { + worker->backtrace.Reset(); + pthread_kill(worker->thread->native_handle(), SIGUSR2); + } + for (auto& worker: m_worker_states) { + worker->backtrace.WaitForCapture(); + } + std::ostringstream oss; + oss << "Worker report" << std::endl; + for (auto& worker: m_worker_states) { + oss << "------------------------------" << std::endl + << " Worker: " << worker->worker_id << std::endl + << " Current arrow: " << worker->last_arrow_id << std::endl + << " Current event: " << worker->last_event_nr << std::endl + << " Backtrace:" << std::endl << std::endl + << worker->backtrace.ToString(); + } + auto s = oss.str(); + LOG_WARN(GetLogger()) << s << LOG_END; + + if (send_to_pipe) { + + int fd = open(m_path_to_named_pipe.c_str(), O_WRONLY); + if (fd >= 0) { + write(fd, s.c_str(), s.length()+1); + close(fd); + } + else { + LOG_ERROR(GetLogger()) << "Unable to open named pipe '" << m_path_to_named_pipe << "' for writing. \n" + << " You can use a different named pipe for status info by setting the parameter `jana:status_fname`.\n" + << " The status report will still show up in the log." << LOG_END; + } + } +} + + +std::string ToString(JExecutionEngine::RunStatus runstatus) { + switch(runstatus) { + case JExecutionEngine::RunStatus::Running: return "Running"; + case JExecutionEngine::RunStatus::Paused: return "Paused"; + case JExecutionEngine::RunStatus::Failed: return "Failed"; + case JExecutionEngine::RunStatus::Pausing: return "Pausing"; + case JExecutionEngine::RunStatus::Draining: return "Draining"; + case JExecutionEngine::RunStatus::Finished: return "Finished"; + } +} + diff --git a/src/libraries/JANA/Engine/JExecutionEngine.h b/src/libraries/JANA/Engine/JExecutionEngine.h new file mode 100644 index 000000000..f0e2ffd66 --- /dev/null +++ b/src/libraries/JANA/Engine/JExecutionEngine.h @@ -0,0 +1,170 @@ + +// Copyright 2024, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +extern thread_local int jana2_worker_id; + +class JExecutionEngine : public JService { + +public: + using clock_t = std::chrono::steady_clock; + + enum class RunStatus { Paused, Running, Pausing, Draining, Failed, Finished }; + enum class InterruptStatus { NoInterruptsSupervised, NoInterruptsUnsupervised, InspectRequested, InspectInProgress, PauseAndQuit }; + + struct Perf { + RunStatus runstatus; + size_t thread_count; + size_t event_count; + size_t uptime_ms; + double throughput_hz; + JEventLevel event_level; + }; + + struct Worker { + size_t worker_id; + JBacktrace* backtrace; + }; + +#ifndef JANA2_TESTCASE +private: +#endif + + struct Task { + JArrow* arrow = nullptr; + JEvent* input_event = nullptr; + int input_port = -1; + JArrow::OutputData outputs; + size_t output_count = 0; + JArrow::FireResult status = JArrow::FireResult::NotRunYet; + }; + + struct ArrowState { + enum class Status { Paused, Running, Finished }; + Status status = Status::Paused; + bool is_parallel = false; + bool is_source = false; + bool is_sink = false; + size_t next_input = 0; + size_t active_tasks = 0; + size_t events_processed; + clock_t::duration total_processing_duration; + }; + + struct WorkerState { + std::thread* thread = nullptr; + size_t worker_id = 0; + size_t cpu_id = 0; + size_t location_id = 0; + clock_t::time_point last_checkout_time = clock_t::now(); + std::exception_ptr stored_exception = nullptr; + bool is_stop_requested = false; + bool is_event_warmed_up = false; + bool is_timed_out = false; + uint64_t last_event_nr = 0; + size_t last_arrow_id = 0; + JBacktrace backtrace; + }; + + +#ifndef JANA2_TESTCASE +private: +#endif + // Services + Service m_topology {this}; + + // Parameters + bool m_show_ticker = true; + bool m_enable_timeout = true; + int m_backoff_ms = 10; + int m_ticker_ms = 500; + int m_timeout_s = 8; + int m_warmup_timeout_s = 30; + std::string m_path_to_named_pipe = "/tmp/jana_status"; + + // Concurrency + std::mutex m_mutex; + std::condition_variable m_condvar; + std::vector> m_worker_states; + std::vector m_arrow_states; + RunStatus m_runstatus = RunStatus::Paused; + std::atomic m_interrupt_status { InterruptStatus::NoInterruptsUnsupervised }; + std::atomic_bool m_print_worker_report_requested {false}; + std::atomic_bool m_send_worker_report_requested {false}; + + // Metrics + size_t m_event_count_at_start = 0; + size_t m_event_count_at_finish = 0; + clock_t::time_point m_time_at_start; + clock_t::time_point m_time_at_finish; + clock_t::duration m_total_idle_duration = clock_t::duration::zero(); + clock_t::duration m_total_scheduler_duration = clock_t::duration::zero(); + + +public: + + JExecutionEngine() { + SetLoggerName("jana"); + } + + ~JExecutionEngine() { + ScaleWorkers(0); + // If we don't shut down the thread team, the condition variable will hang during destruction + } + + void Init() override; + + void RunTopology(); + void PauseTopology(); + void DrainTopology(); + void FinishTopology(); + + void ScaleWorkers(size_t nthreads); + Worker RegisterWorker(); + void RunWorker(Worker); + void RunSupervisor(); + + JArrow::FireResult Fire(size_t arrow_id, size_t location_id=0); + + Perf GetPerf(); + RunStatus GetRunStatus(); + void SetTickerEnabled(bool ticker_on); + bool IsTickerEnabled() const; + void SetTimeoutEnabled(bool timeout_on); + bool IsTimeoutEnabled() const; + + void HandleSIGINT(); + void HandleSIGUSR1(); + void HandleSIGUSR2(); + void HandleSIGTSTP(); + +#ifndef JANA2_TESTCASE +private: +#endif + + void PrintWorkerReport(bool); + void PrintFinalReport(); + bool CheckTimeout(); + void HandleFailures(); + void ExchangeTask(Task& task, size_t worker_id, bool nonblocking=false); + void CheckinCompletedTask_Unsafe(Task& task, WorkerState& worker, clock_t::time_point checkin_time); + void FindNextReadyTask_Unsafe(Task& task, WorkerState& worker); + +}; + + +std::string ToString(JExecutionEngine::RunStatus status); + + diff --git a/src/libraries/JANA/Engine/JPerfMetrics.cc b/src/libraries/JANA/Engine/JPerfMetrics.cc deleted file mode 100644 index 549369603..000000000 --- a/src/libraries/JANA/Engine/JPerfMetrics.cc +++ /dev/null @@ -1,131 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#include "JPerfMetrics.h" -#include - - -/// Reset is meant to be called between separate run()s or scale()s. -/// It sets everything in the metric back to zero or now(), except the start_event_count, -/// which increases monotonically throughout the life of the program. -void JPerfMetrics::reset() { - std::lock_guard lock(mutex_); - auto now = Clock::now(); - mode_ = Mode::Reset; - start_time_ = now; - prev_time_ = now; - last_time_ = now; - start_event_count_ = last_event_count_; - prev_event_count_ = last_event_count_; - last_event_count_ = 0; - thread_count_ = 0; -} - -/// This implementation of start() assumes that no processing has -/// occurred while the stopwatch wasn't running. Thus it can reuse -/// the event count from the previous run. This is convenient -/// because that value can be annoying to calculate. -void JPerfMetrics::start(size_t current_thread_count) { - std::lock_guard lock(mutex_); - auto now = Clock::now(); - if (mode_ == Mode::Reset) { - mode_ = Mode::Ticking; - start_time_ = now; - last_time_ = now; - prev_time_ = now; - thread_count_ = current_thread_count; - } - else { - throw JException("Stopwatch must be reset before it can be started!"); - } -} - -/// This is the preferred implementation of start(), which doesn't make -/// dangerous assumptions about nothing happening in the meantime -void JPerfMetrics::start(size_t current_event_count, size_t current_thread_count) { - - std::lock_guard lock(mutex_); - auto now = Clock::now(); - if (mode_ == Mode::Reset) { - mode_ = Mode::Ticking; - start_time_ = now; - last_time_ = now; - prev_time_ = now; - start_event_count_ = current_event_count; - prev_event_count_ = current_event_count; - last_event_count_ = current_event_count; - thread_count_ = current_thread_count; - } - else { - throw JException("Stopwatch must be reset before it can be started!"); - } -} - - -/// Split accumulates another (monotonic) event count measurement. - -/// If the stopwatch has already been stopped, split will -/// update the event count without updating the time. -/// This is convenient because we sometimes don't have access -/// to the current event count but still wish to stop the timer, -/// but it is dangerous because it makes the caller responsible for ensuring -/// that further events don't get processed after the stopwatch stops. -void JPerfMetrics::split(size_t current_event_count) { - - std::lock_guard lock(mutex_); - if (mode_ == Mode::Ticking) { - prev_time_ = last_time_; - last_time_ = Clock::now(); - prev_event_count_ = last_event_count_; - last_event_count_ = current_event_count; - } - else { - // TODO: Reconsider this - last_event_count_ = current_event_count; - } -} - - -/// Stop without recording a final measurement. This can be provided -/// in a later call to split(). `void stop(current_event_count)` is preferred. -void JPerfMetrics::stop() { - std::lock_guard lock(mutex_); - auto now = Clock::now(); - if (mode_ == Mode::Ticking) { - mode_ = Mode::Stopped; - prev_time_ = last_time_; - last_time_ = now; - } -} - -/// Stop, and record the final measurement at that time. This should be used -/// instead of `void stop()` whenever possible. -void JPerfMetrics::stop(size_t current_event_count) { - - std::lock_guard lock(mutex_); - auto now = Clock::now(); - if (mode_ == Mode::Ticking) { - mode_ = Mode::Stopped; - prev_time_ = last_time_; - last_time_ = now; - prev_event_count_ = last_event_count_; - last_event_count_ = current_event_count; - } -} - -void JPerfMetrics::summarize(JPerfSummary& summary) { - - std::lock_guard lock(mutex_); - summary.monotonic_events_completed = last_event_count_; - summary.total_events_completed = last_event_count_ - start_event_count_; - summary.latest_events_completed = last_event_count_ - prev_event_count_; - summary.total_uptime_s = secs(last_time_ - start_time_).count(); - summary.latest_uptime_s = secs(last_time_ - prev_time_).count(); - summary.avg_throughput_hz = summary.total_events_completed / summary.total_uptime_s; - summary.latest_throughput_hz = summary.latest_events_completed / summary.latest_uptime_s; - summary.thread_count = thread_count_; -} - - - diff --git a/src/libraries/JANA/Engine/JPerfMetrics.h b/src/libraries/JANA/Engine/JPerfMetrics.h deleted file mode 100644 index e0730cd66..000000000 --- a/src/libraries/JANA/Engine/JPerfMetrics.h +++ /dev/null @@ -1,55 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#pragma once -#include -#include -#include "JPerfSummary.h" - -/// JPerfMetrics represents the highest-level metrics we can collect from a running JProcessingTopology. -/// Conceptually, it resembles a stopwatch, which is meant to be reset when the thread count changes, -/// started when processing starts, split every time a performance measurement should be made, and -/// stopped when processing finished. At each interaction the caller provides the current event count. -/// Timers are all managed internally. -class JPerfMetrics { - -public: - - enum class Mode {Reset, Ticking, Stopped}; - - Mode get_mode() { return mode_;}; - - void reset(); - - void start(size_t current_thread_count); - - void start(size_t current_event_count, size_t current_thread_count); - - void split(size_t current_event_count); - - void stop(size_t current_event_count); - - void stop(); - - void summarize(JPerfSummary& summary); - -private: - - using Clock = std::chrono::steady_clock; - using secs = std::chrono::duration; - - Mode mode_ = Mode::Reset; - - Clock::time_point start_time_ = Clock::now(); - Clock::time_point prev_time_ = Clock::now(); - Clock::time_point last_time_ = Clock::now(); - size_t start_event_count_ = 0; - size_t prev_event_count_ = 0; - size_t last_event_count_ = 0; - size_t thread_count_ = 0; - std::mutex mutex_; -}; - - diff --git a/src/libraries/JANA/Engine/JPerfSummary.cc b/src/libraries/JANA/Engine/JPerfSummary.cc deleted file mode 100644 index da33dc4ce..000000000 --- a/src/libraries/JANA/Engine/JPerfSummary.cc +++ /dev/null @@ -1,91 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#include "JPerfSummary.h" - -#include -#include - -std::ostream& operator<<(std::ostream& os, const JPerfSummary& s) { - - os << std::endl; - os << " Thread team size [count]: " << s.thread_count << std::endl; - os << " Total uptime [s]: " << std::setprecision(4) << s.total_uptime_s << std::endl; - os << " Uptime delta [s]: " << std::setprecision(4) << s.latest_uptime_s << std::endl; - os << " Completed events [count]: " << s.total_events_completed << std::endl; - os << " Inst throughput [Hz]: " << std::setprecision(3) << s.latest_throughput_hz << std::endl; - os << " Avg throughput [Hz]: " << std::setprecision(3) << s.avg_throughput_hz << std::endl; - os << " Sequential bottleneck [Hz]: " << std::setprecision(3) << s.avg_seq_bottleneck_hz << std::endl; - os << " Parallel bottleneck [Hz]: " << std::setprecision(3) << s.avg_par_bottleneck_hz << std::endl; - os << " Efficiency [0..1]: " << std::setprecision(3) << s.avg_efficiency_frac << std::endl; - os << std::endl; - - os << " +--------------------------+--------+-----+---------+-------+--------+---------+-------------+" << std::endl; - os << " | Name | Type | Par | Threads | Chunk | Thresh | Pending | Completed |" << std::endl; - os << " +--------------------------+--------+-----+---------+-------+--------+---------+-------------+" << std::endl; - - for (auto as : s.arrows) { - os << " | " - << std::setw(24) << std::left << as.arrow_name << " | " - << std::setw(6) << std::left << (as.is_source ? "Src" : (as.is_sink ? "Sink" : "")) << " | " - << std::setw(3) << std::right << (as.is_parallel ? " T " : " F ") << " | " - << std::setw(7) << as.thread_count << " |" - << std::setw(6) << as.chunksize << " |"; - - if (!as.is_source) { - - os << std::setw(7) << as.threshold << " |" - << std::setw(8) << as.messages_pending << " |"; - } - else { - - os << " - | - |"; - } - os << std::setw(12) << as.total_messages_completed << " |" - << std::endl; - } - os << " +--------------------------+--------+-----+---------+-------+--------+---------+-------------+" << std::endl; - - - os << " +--------------------------+-------------+--------------+----------------+--------------+----------------+" << std::endl; - os << " | Name | Avg latency | Inst latency | Queue latency | Queue visits | Queue overhead | " << std::endl; - os << " | | [ms/event] | [ms/event] | [ms/visit] | [count] | [0..1] | " << std::endl; - os << " +--------------------------+-------------+--------------+----------------+--------------+----------------+" << std::endl; - - for (auto as : s.arrows) { - os << " | " << std::setprecision(3) - << std::setw(24) << std::left << as.arrow_name << " | " - << std::setw(11) << std::right << as.avg_latency_ms << " |" - << std::setw(13) << as.last_latency_ms << " |" - << std::setw(15) << as.avg_queue_latency_ms << " |" - << std::setw(13) << as.queue_visit_count << " |" - << std::setw(15) << as.avg_queue_overhead_frac << " |" - << std::endl; - } - os << " +--------------------------+-------------+--------------+----------------+--------------+----------------+" << std::endl; - - - os << " +----+----------------------+-------------+------------+-----------+----------------+------------------+" << std::endl; - os << " | ID | Last arrow name | Useful time | Retry time | Idle time | Scheduler time | Scheduler visits |" << std::endl; - os << " | | | [ms] | [ms] | [ms] | [ms] | [count] |" << std::endl; - os << " +----+----------------------+-------------+------------+-----------+----------------+------------------+" << std::endl; - - for (auto ws : s.workers) { - os << " |" - << std::setw(3) << std::right << ws.worker_id << " | " - << std::setw(20) << std::left << ws.last_arrow_name << " |" - << std::setw(12) << std::right << ws.last_useful_time_ms << " |" - << std::setw(11) << ws.last_retry_time_ms << " |" - << std::setw(10) << ws.last_idle_time_ms << " |" - << std::setw(15) << ws.last_scheduler_time_ms << " |" - << std::setw(17) << ws.scheduler_visit_count << " |" - << std::endl; - } - os << " +----+----------------------+-------------+------------+-----------+----------------+------------------+" << std::endl; - return os; -} - - - diff --git a/src/libraries/JANA/Engine/JPerfSummary.h b/src/libraries/JANA/Engine/JPerfSummary.h deleted file mode 100644 index e2840edd9..000000000 --- a/src/libraries/JANA/Engine/JPerfSummary.h +++ /dev/null @@ -1,76 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#pragma once -#include -#include - -struct ArrowSummary { - std::string arrow_name; - bool is_parallel; - bool is_source; - bool is_sink; - size_t thread_count; - int running_upstreams; - bool has_backpressure; - size_t messages_pending; - size_t threshold; - size_t chunksize; - - size_t total_messages_completed; - size_t last_messages_completed; - double avg_latency_ms; - double avg_queue_latency_ms; - double last_latency_ms; - double last_queue_latency_ms; - double avg_queue_overhead_frac; - size_t queue_visit_count; -}; - -struct WorkerSummary { - int worker_id; - int cpu_id; - bool is_pinned; - double last_heartbeat_ms; - double total_useful_time_ms; - double total_retry_time_ms; - double total_idle_time_ms; - double total_scheduler_time_ms; - double last_useful_time_ms; - double last_retry_time_ms; - double last_idle_time_ms; - double last_scheduler_time_ms; - long scheduler_visit_count; - std::string last_arrow_name; - double last_arrow_avg_latency_ms; - double last_arrow_avg_queue_latency_ms; - double last_arrow_last_latency_ms; - double last_arrow_last_queue_latency_ms; - size_t last_arrow_queue_visit_count; -}; - -struct JPerfSummary { - - size_t monotonic_events_completed = 0; // Since program started - size_t total_events_completed = 0; // Since run or rescale started - size_t latest_events_completed = 0; // Since previous measurement - size_t thread_count = 0; - double total_uptime_s = 0; - double latest_uptime_s = 0; - double avg_throughput_hz = 0; - double latest_throughput_hz = 0; - - double avg_seq_bottleneck_hz; - double avg_par_bottleneck_hz; - double avg_efficiency_frac; - - std::vector workers; - std::vector arrows; - -}; - -std::ostream& operator<<(std::ostream& stream, const JPerfSummary& data); - - diff --git a/src/libraries/JANA/Engine/JScheduler.cc b/src/libraries/JANA/Engine/JScheduler.cc deleted file mode 100644 index 4c0786df2..000000000 --- a/src/libraries/JANA/Engine/JScheduler.cc +++ /dev/null @@ -1,471 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#include - -#include "JScheduler.h" -#include -#include -#include - - -JScheduler::JScheduler(std::shared_ptr topology) - : m_topology(topology) - { - m_topology_state.next_arrow_index = 0; - - // Keep track of downstream arrows - std::map arrow_map; - size_t i=0; - for (auto* arrow : topology->arrows) { - arrow_map[arrow] = i++; - } - m_topology_state.arrow_states = std::vector(topology->arrows.size()); - - for (i=0; iarrows.size(); ++i) { - auto& as = m_topology_state.arrow_states[i]; - JArrow* arrow = topology->arrows[i]; - as.arrow = arrow; - for (JArrow* downstream : arrow->m_listeners) { - as.downstream_arrow_indices.push_back(arrow_map[downstream]); - } - } - } - - -JArrow* JScheduler::next_assignment(uint32_t worker_id, JArrow* assignment, JArrowMetrics::Status last_result) { - - std::lock_guard lock(m_mutex); - - LOG_DEBUG(logger) << "Worker " << worker_id << " checking in: " - << ((assignment == nullptr) ? "idle" : assignment->get_name()) << " -> " << to_string(last_result) << LOG_END; - - // Check latest arrow back in - if (assignment != nullptr) { - checkin_unprotected(assignment, last_result); - } - - JArrow* next = checkout_unprotected(); - - LOG_DEBUG(logger) << "Worker " << worker_id << " assigned: " - << ((next == nullptr) ? "idle" : next->get_name()) << LOG_END; - return next; - -} - - -void JScheduler::last_assignment(uint32_t worker_id, JArrow* assignment, JArrowMetrics::Status last_result) { - - std::lock_guard lock(m_mutex); - - LOG_DEBUG(logger) << "Worker " << worker_id << " checking in: " - << ((assignment == nullptr) ? "idle" : assignment->get_name()) - << " -> " << to_string(last_result) << "). Shutting down!" << LOG_END; - - if (assignment != nullptr) { - checkin_unprotected(assignment, last_result); - } -} - - - -void JScheduler::checkin_unprotected(JArrow* assignment, JArrowMetrics::Status last_result) { - // Find index of arrow - size_t index = 0; - while (m_topology_state.arrow_states[index].arrow != assignment) index++; - assert(index < m_topology_state.arrow_states.size()); - - // Decrement arrow's thread count - ArrowState& as = m_topology_state.arrow_states[index]; - as.thread_count -= 1; - - bool found_inactive_source = - assignment->is_source() && - last_result == JArrowMetrics::Status::Finished && // Only Sources get to declare themselves finished! - as.status == ArrowStatus::Active; // We only want to deactivate once - - - bool found_draining_stage_or_sink = - !assignment->is_source() && // We aren't a source - as.active_or_draining_upstream_arrow_count == 0 && // All upstreams arrows are inactive - assignment->get_pending() == 0 && // All upstream queues are empty - as.thread_count > 0 && // There are other workers still assigned to this arrow - as.status == ArrowStatus::Active; // We only want to deactivate once - - - bool found_inactive_stage_or_sink = - !assignment->is_source() && // We aren't a source - as.active_or_draining_upstream_arrow_count == 0 && // All upstreams arrows are inactive - assignment->get_pending() == 0 && // All upstream queues are empty - as.thread_count == 0 && // There are NO other workers still assigned to this arrow - (as.status == ArrowStatus::Draining || // We only want to deactivate once - as.status == ArrowStatus::Active); - - - if (found_inactive_source || found_inactive_stage_or_sink) { - - // Deactivate arrow - // Because we are deactivating it from Active state, we know the topology has not been paused. Hence we can finalize immediately. - assignment->finalize(); - as.status = ArrowStatus::Finalized; - m_topology_state.active_or_draining_arrow_count--; - - LOG_DEBUG(logger) << "Deactivated arrow '" << assignment->get_name() << "' (" << m_topology_state.active_or_draining_arrow_count << " remaining)" << LOG_END; - - for (size_t downstream: m_topology_state.arrow_states[index].downstream_arrow_indices) { - m_topology_state.arrow_states[downstream].active_or_draining_upstream_arrow_count--; - } - } - else if (found_draining_stage_or_sink) { - // Drain arrow - as.status = ArrowStatus::Draining; - LOG_DEBUG(logger) << "Draining arrow '" << assignment->get_name() << "' (" << m_topology_state.active_or_draining_arrow_count << " remaining)" << LOG_END; - } - - // Test if this was the last arrow running - if (m_topology_state.active_or_draining_arrow_count == 0) { - LOG_DEBUG(logger) << "All arrows are inactive. Deactivating topology." << LOG_END; - achieve_topology_pause_unprotected(); - } -} - - -JArrow* JScheduler::checkout(size_t arrow_index) { - // Note that this lets us check out Inactive arrows, whereas checkout_unprotected() does not. This because we are called by JApplicationInspector - // whereas checkout_unprotected is called by JWorker. This is because JArrowProcessingController::request_pause shuts off the topology - // instead of shutting off the workers, which in hindsight might have been the wrong choice. - - std::lock_guard lock(m_mutex); - - if (arrow_index >= m_topology_state.arrow_states.size()) return nullptr; - - ArrowState& candidate = m_topology_state.arrow_states[arrow_index]; - - if ((candidate.status == ArrowStatus::Active || candidate.status == ArrowStatus::Inactive) && // This excludes Draining arrows - (candidate.arrow->is_parallel() || candidate.thread_count == 0)) { // This excludes non-parallel arrows that are already assigned to a worker - - m_topology_state.arrow_states[arrow_index].thread_count += 1; - return candidate.arrow; - - } - return nullptr; -} - - -JArrow* JScheduler::checkout_unprotected() { - - // Choose a new arrow. Loop over all arrows, starting at where we last left off, and pick the first arrow that works - size_t current_idx = m_topology_state.next_arrow_index; - do { - ArrowState& candidate = m_topology_state.arrow_states[current_idx]; - current_idx += 1; - current_idx %= m_topology->arrows.size(); - - if (candidate.status == ArrowStatus::Active && // This excludes Draining arrows - (candidate.arrow->is_parallel() || candidate.thread_count == 0)) { // This excludes non-parallel arrows that are already assigned to a worker - - m_topology_state.next_arrow_index = current_idx; // Next time, continue right where we left off - candidate.thread_count += 1; - return candidate.arrow; - - } - } while (current_idx != m_topology_state.next_arrow_index); - return nullptr; // We've looped through everything with no luck -} - - -void JScheduler::initialize_topology() { - - std::lock_guard lock(m_mutex); - - assert(m_topology_state.current_topology_status == TopologyStatus::Uninitialized); - for (JArrow* arrow : m_topology->arrows) { - arrow->initialize(); - } - m_topology_state.current_topology_status = TopologyStatus::Paused; -} - - -void JScheduler::drain_topology() { - std::lock_guard lock(m_mutex); - if (m_topology_state.current_topology_status == TopologyStatus::Finalized) { - LOG_DEBUG(logger) << "JScheduler: drain(): Skipping because topology is already Finalized" << LOG_END; - return; - } - LOG_DEBUG(logger) << "JScheduler: drain_topology()" << LOG_END; - - // We pause (as opposed to finish) for two reasons: - // 1. There might be workers in the middle of calling eventSource->GetEvent. - // 2. drain() might be called from a signal handler. It isn't safe to make syscalls during signal handlers - // due to risk of deadlock. (We technically shouldn't even do logging!) - // - for (size_t i=0; iis_source()) { - pause_arrow_unprotected(i); - } - } - m_topology_state.current_topology_status = TopologyStatus::Draining; - -} - -void JScheduler::run_topology(int nthreads) { - std::lock_guard lock(m_mutex); - TopologyStatus current_status = m_topology_state.current_topology_status; - if (current_status == TopologyStatus::Running || current_status == TopologyStatus::Finalized) { - LOG_DEBUG(logger) << "JScheduler: run_topology() : " << current_status << " => " << current_status << LOG_END; - return; - } - LOG_DEBUG(logger) << "JScheduler: run_topology() : " << current_status << " => Running" << LOG_END; - - bool source_found = false; - for (JArrow* arrow : m_topology->arrows) { - if (arrow->is_source()) { - source_found = true; - } - } - if (!source_found) { - throw JException("No event sources found!"); - } - for (size_t i=0; iis_source()) { - run_arrow_unprotected(i); - } - } - // Note that we activate workers AFTER we activate the topology, so no actual processing will have happened - // by this point when we start up the metrics. - m_topology->metrics.reset(); - m_topology->metrics.start(nthreads); - m_topology_state.current_topology_status = TopologyStatus::Running; -} - -void JScheduler::request_topology_pause() { - std::lock_guard lock(m_mutex); - // This sets all Running arrows to Paused, which prevents Workers from picking up any additional assignments - // Once all Workers have completed their remaining assignments, the scheduler will call achieve_pause(). - TopologyStatus current_status = m_topology_state.current_topology_status; - if (current_status == TopologyStatus::Running) { - LOG_DEBUG(logger) << "JScheduler: request_pause() : " << current_status << " => Pausing" << LOG_END; - for (size_t i=0; i " << current_status << LOG_END; - } -} - -void JScheduler::achieve_topology_pause() { - - std::lock_guard lock(m_mutex); - achieve_topology_pause_unprotected(); -} - -void JScheduler::achieve_topology_pause_unprotected() { - - // This is meant to be used by the scheduler to tell us when all workers have stopped, so it is safe to finish(), etc - TopologyStatus current_status = m_topology_state.current_topology_status; - if (current_status == TopologyStatus::Running || current_status == TopologyStatus::Pausing || current_status == TopologyStatus::Draining) { - LOG_DEBUG(logger) << "JScheduler: achieve_topology_pause() : " << current_status << " => " << TopologyStatus::Paused << LOG_END; - m_topology->metrics.stop(); - m_topology_state.current_topology_status = TopologyStatus::Paused; - } - else { - LOG_DEBUG(logger) << "JScheduler: achieve_topology_pause() : " << current_status << " => " << current_status << LOG_END; - } -} - -void JScheduler::finish_topology() { - std::lock_guard lock(m_mutex); - // This finalizes all arrows. Once this happens, we cannot restart the topology. - // assert(m_topology_state.current_topology_status == TopologyStatus::Inactive); - for (ArrowState& as : m_topology_state.arrow_states) { - - if (as.status != ArrowStatus::Finalized) { - as.arrow->finalize(); - } - } - m_topology_state.current_topology_status = TopologyStatus::Finalized; -} - -JScheduler::TopologyStatus JScheduler::get_topology_status() { - std::lock_guard lock(m_mutex); - return m_topology_state.current_topology_status; -} - -JScheduler::TopologyState JScheduler::get_topology_state() { - std::lock_guard lock(m_mutex); - return m_topology_state; -} - - -void JScheduler::run_arrow_unprotected(size_t index) { - auto& as = m_topology_state.arrow_states[index]; - auto name = as.arrow->get_name(); - ArrowStatus status = as.status; - - // if (status == ArrowStatus::Unopened) { - // LOG_DEBUG(logger) << "Arrow '" << name << "' run(): Not initialized!" << LOG_END; - // throw rException("Arrow %s has not been initialized!", name.c_str()); - // } - // if (status == ArrowStatus::Active || m_status == ArrowStatus::Finalized) { - // LOG_DEBUG(logger) << "Arrow '" << name << "' run() : " << status << " => " << status << LOG_END; - // return; - // } - LOG_DEBUG(logger) << "Arrow '" << name << "' run() : " << status << " => Active" << LOG_END; - - m_topology_state.active_or_draining_arrow_count++; - for (size_t downstream: m_topology_state.arrow_states[index].downstream_arrow_indices) { - m_topology_state.arrow_states[downstream].active_or_draining_upstream_arrow_count++; - run_arrow_unprotected(downstream); // Activating something recursively activates everything downstream. - } - m_topology_state.arrow_states[index].status = ArrowStatus::Active; -} - -void JScheduler::pause_arrow_unprotected(size_t index) { - auto& as = m_topology_state.arrow_states[index]; - auto name = as.arrow->get_name(); - ArrowStatus status = as.status; - - if (status != ArrowStatus::Active) { - LOG_DEBUG(logger) << "JArrow '" << name << "' pause() : " << status << " => " << status << LOG_END; - return; // pause() is a no-op unless running - } - LOG_DEBUG(logger) << "JArrow '" << name << "' pause() : " << status << " => Inactive" << LOG_END; - m_topology_state.active_or_draining_arrow_count--; - for (size_t downstream: m_topology_state.arrow_states[index].downstream_arrow_indices) { - m_topology_state.arrow_states[downstream].active_or_draining_upstream_arrow_count--; - } - m_topology_state.arrow_states[index].status = ArrowStatus::Inactive; -} - -void JScheduler::finish_arrow_unprotected(size_t index) { - auto& as = m_topology_state.arrow_states[index]; - auto name = as.arrow->get_name(); - ArrowStatus status = as.status; - - LOG_DEBUG(logger) << "JArrow '" << name << "' finish() : " << status << " => Finalized" << LOG_END; - ArrowStatus old_status = as.status; - // if (old_status == ArrowStatus::Unopened) { - // LOG_DEBUG(logger) << "JArrow '" << name << "': Uninitialized!" << LOG_END; - // throw JException("JArrow::finish(): Arrow %s has not been initialized!", name.c_str()); - // } - if (old_status == ArrowStatus::Active) { - m_topology_state.active_or_draining_arrow_count--; - for (size_t downstream: m_topology_state.arrow_states[index].downstream_arrow_indices) { - m_topology_state.arrow_states[downstream].active_or_draining_upstream_arrow_count--; - } - } - if (old_status != ArrowStatus::Finalized) { - LOG_TRACE(logger) << "JArrow '" << name << "': Finalizing (this must only happen once)" << LOG_END; - as.arrow->finalize(); - } - m_topology_state.arrow_states[index].status = ArrowStatus::Finalized; -} - - -std::ostream& operator<<(std::ostream& os, JScheduler::TopologyStatus status) { - switch(status) { - case JScheduler::TopologyStatus::Uninitialized: os << "Uninitialized"; break; - case JScheduler::TopologyStatus::Running: os << "Running"; break; - case JScheduler::TopologyStatus::Pausing: os << "Pausing"; break; - case JScheduler::TopologyStatus::Draining: os << "Draining"; break; - case JScheduler::TopologyStatus::Paused: os << "Paused"; break; - case JScheduler::TopologyStatus::Finalized: os << "Finalized"; break; - } - return os; -} - - -std::ostream& operator<<(std::ostream& os, JScheduler::ArrowStatus status) { - switch (status) { - case JScheduler::ArrowStatus::Uninitialized: os << "Uninitialized"; break; - case JScheduler::ArrowStatus::Active: os << "Active"; break; - case JScheduler::ArrowStatus::Draining: os << "Draining"; break; - case JScheduler::ArrowStatus::Inactive: os << "Inactive"; break; - case JScheduler::ArrowStatus::Finalized: os << "Finalized"; break; - } - return os; -} - - -void JScheduler::summarize_arrows(std::vector& summaries) { - - // Make sure we have exactly one ArrowSummary for each arrow - if (summaries.size() != m_topology_state.arrow_states.size()) { - summaries = std::vector(m_topology_state.arrow_states.size()); - } - - std::lock_guard lock(m_mutex); - - // Copy over arrow status variables - for (size_t i=0; iget_name(); - summary.is_parallel = as.arrow->is_parallel(); - summary.is_source = as.arrow->is_source(); - summary.is_sink = as.arrow->is_sink(); - summary.chunksize = as.arrow->get_chunksize(); - summary.messages_pending = as.arrow->get_pending(); - summary.threshold = as.arrow->get_threshold(); - - summary.thread_count = as.thread_count; - summary.running_upstreams = as.active_or_draining_upstream_arrow_count; - - JArrowMetrics::Status last_status; - size_t total_message_count; - size_t last_message_count; - size_t total_queue_visits; - size_t last_queue_visits; - JArrowMetrics::duration_t total_latency; - JArrowMetrics::duration_t last_latency; - JArrowMetrics::duration_t total_queue_latency; - JArrowMetrics::duration_t last_queue_latency; - - as.arrow->get_metrics().get( - last_status, - total_message_count, - last_message_count, - total_queue_visits, - last_queue_visits, - total_latency, - last_latency, - total_queue_latency, - last_queue_latency - ); - - using millisecs = std::chrono::duration; - auto total_latency_ms = millisecs(total_latency).count(); - auto total_queue_latency_ms = millisecs(total_queue_latency).count(); - - summary.total_messages_completed = total_message_count; - summary.last_messages_completed = last_message_count; - summary.queue_visit_count = total_queue_visits; - - summary.avg_queue_latency_ms = (total_queue_visits == 0) - ? std::numeric_limits::infinity() - : total_queue_latency_ms / total_queue_visits; - - summary.avg_queue_overhead_frac = total_queue_latency_ms / (total_queue_latency_ms + total_latency_ms); - - summary.avg_latency_ms = (total_message_count == 0) - ? std::numeric_limits::infinity() - : total_latency_ms/total_message_count; - - summary.last_latency_ms = (last_message_count == 0) - ? std::numeric_limits::infinity() - : millisecs(last_latency).count()/last_message_count; - - } - - -} - diff --git a/src/libraries/JANA/Engine/JScheduler.h b/src/libraries/JANA/Engine/JScheduler.h deleted file mode 100644 index 1a2ed22e0..000000000 --- a/src/libraries/JANA/Engine/JScheduler.h +++ /dev/null @@ -1,113 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include - -#include -#include -#include - - -struct JArrowTopology; - -/// Scheduler assigns Arrows to Workers in a first-come-first-serve manner, -/// not unlike OpenMP's `schedule dynamic`. -class JScheduler { -public: - enum class TopologyStatus { - Uninitialized, // Arrows have not been initialized() yet - Running, // At least one arrow is active - Pausing, // At least one arrow is active, but a pause has been requested - Draining, // At least one arrow is active, but a drain has been requested - Paused, // No arrows are active, but may be activated or re-activated - Finalized // No arrows are active, and all arrows have been finalized(), so they many not be re-activated - }; - - enum class ArrowStatus { Uninitialized, // Arrow has not been initialized() yet - Active, // Arrow may be scheduled - Draining, // All upstreams are inactive and queues are emptied, but arrow still has at least one worker out. Should not be scheduled. - Inactive, // Arrow should not be scheduled, but may be re-activated - Finalized // Arrow should not be scheduled, and has been finalized(), so it may not be re-activated - }; - - struct ArrowState { - JArrow* arrow = nullptr; - ArrowStatus status = ArrowStatus::Uninitialized; - int64_t thread_count = 0; // Current number of threads assigned to this arrow - int64_t active_or_draining_upstream_arrow_count = 0; // Current number of active or draining arrows immediately upstream - std::vector downstream_arrow_indices; - }; - - struct TopologyState { - std::vector arrow_states; - TopologyStatus current_topology_status = TopologyStatus::Uninitialized; - int64_t active_or_draining_arrow_count = 0; // Detects when the topology has paused - size_t next_arrow_index; - }; - -private: - // This mutex controls ALL scheduler state - std::mutex m_mutex; - - std::shared_ptr m_topology; - - // Protected state - TopologyState m_topology_state; - - -public: - - /// Constructor. Note that a Scheduler operates on a vector of Arrow*s. - JScheduler(std::shared_ptr topology); - - - // Worker-facing operations - - /// Lets a Worker ask the Scheduler for another assignment. If no assignments make sense, - /// Scheduler returns nullptr, which tells that Worker to idle until his next checkin. - /// If next_assignment() makes any changes to internal Scheduler state or to any of its arrows, - /// it must be synchronized. - JArrow* next_assignment(uint32_t worker_id, JArrow* assignment, JArrowMetrics::Status result); - - /// Lets a Worker tell the scheduler that he is shutting down and won't be working on his assignment - /// any more. The scheduler is thus free to reassign the arrow to one of the remaining workers. - void last_assignment(uint32_t worker_id, JArrow* assignment, JArrowMetrics::Status result); - - /// Lets a Worker, test case, or user request a specific arrow. Returns nullptr if arrow can not be - /// checked up because it's no longer active or because it's already at its max parallelism. - JArrow* checkout(size_t arrow_index); - - /// Logger is public so that somebody else can configure it - JLogger logger; - - - void initialize_topology(); - void drain_topology(); - void run_topology(int nthreads); - void request_topology_pause(); - void achieve_topology_pause(); - void finish_topology(); - - TopologyStatus get_topology_status(); - TopologyState get_topology_state(); - void summarize_arrows(std::vector& summaries); - - -private: - void achieve_topology_pause_unprotected(); - void run_arrow_unprotected(size_t index); - void pause_arrow_unprotected(size_t index); - void finish_arrow_unprotected(size_t index); - void checkin_unprotected(JArrow* arrow, JArrowMetrics::Status last_result); - JArrow* checkout_unprotected(); - -}; - -std::ostream& operator<<(std::ostream& os, JScheduler::TopologyStatus status); -std::ostream& operator<<(std::ostream& os, JScheduler::ArrowStatus status); - - - diff --git a/src/libraries/JANA/Engine/JWorker.cc b/src/libraries/JANA/Engine/JWorker.cc deleted file mode 100644 index c24afd2b6..000000000 --- a/src/libraries/JANA/Engine/JWorker.cc +++ /dev/null @@ -1,285 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#include -#include -#include - -/// This allows someone (aka JArrowProcessingController) to declare that this -/// thread has timed out. This ensures that the underlying thread will be detached -/// rather than joined when it is time to wait_for_stop(). -/// Note: Another option was to non-cooperatively kill the thread, at the risk of -/// resource leaks and data corruption. This is NOT supported by std::thread, because -/// smarter people than me have concluded it is a Bad Idea, so we'd have to use the -/// underlying pthread interface if we decide we REALLY want this feature. It doesn't -/// make a difference in the common case because the program will terminate upon timeout -/// anyhow; it only matters in case we are running JANA from a REPL. My instinct is to -/// leave the offending thread alone so that the user has a chance to attach a debugger -/// to the process. -void JWorker::declare_timeout() { - m_run_state = RunState::TimedOut; -} - -void JWorker::measure_perf(WorkerSummary& summary) { - // Read (do not clear) worker metrics - // Read and clear arrow metrics - // Push arrow metrics upstream - - JArrowMetrics latest_arrow_metrics; - latest_arrow_metrics.clear(); - latest_arrow_metrics.take(m_arrow_metrics); // move local arrow metrics onto stack - std::string arrow_name = "idle"; - { - std::lock_guard lock(m_assignment_mutex); - if (m_assignment != nullptr) { - m_assignment->get_metrics().update(latest_arrow_metrics); // propagate to global arrow context - arrow_name = m_assignment->get_name(); - } - } - // Unpack latest_arrow_metrics, add to WorkerSummary - - // If we wanted to average over our measurement interval only, we would also do: - // From inside JProcessingController, do _after_ JWorker::measure_perf() for all workers: - // for all arrows: - // arrow->overall_metrics.take(arrow->interval_metrics); - - // From right here, at the very end: - // worker->overall_metrics.take(worker->interval_metrics); - - JWorkerMetrics latest_worker_metrics; - latest_worker_metrics.clear(); - latest_worker_metrics.update(m_worker_metrics); // nondestructive - // Unpack latest_worker_metrics, add to WorkerSummary - - using millis = std::chrono::duration; - - long scheduler_visit_count; - JWorkerMetrics::duration_t total_useful_time, total_retry_time, total_scheduler_time, total_idle_time; - JWorkerMetrics::duration_t last_useful_time, last_retry_time, last_scheduler_time, last_idle_time; - JWorkerMetrics::time_point_t last_heartbeat; - - latest_worker_metrics.get(last_heartbeat, scheduler_visit_count, total_useful_time, total_retry_time, total_scheduler_time, - total_idle_time, last_useful_time, last_retry_time, last_scheduler_time, last_idle_time); - - summary.total_useful_time_ms = millis(total_useful_time).count(); - summary.total_retry_time_ms = millis(total_retry_time).count(); - summary.total_scheduler_time_ms = millis(total_scheduler_time).count(); - summary.total_idle_time_ms = millis(total_idle_time).count(); - summary.last_useful_time_ms = millis(last_useful_time).count(); - summary.last_retry_time_ms = millis(last_retry_time).count(); - summary.last_scheduler_time_ms = millis(last_scheduler_time).count(); - summary.last_idle_time_ms = millis(last_idle_time).count(); - summary.last_heartbeat_ms = millis(JWorkerMetrics::clock_t::now() - last_heartbeat).count(); - - summary.worker_id = m_worker_id; - summary.cpu_id = m_cpu_id; - summary.is_pinned = m_pin_to_cpu; - summary.scheduler_visit_count = scheduler_visit_count; - - summary.last_arrow_name = arrow_name; - - JArrowMetrics::Status last_status; - size_t total_message_count, last_message_count, total_queue_visits, last_queue_visits; - JArrowMetrics::duration_t total_latency, last_latency, total_queue_latency, last_queue_latency; - - latest_arrow_metrics.get(last_status, total_message_count, last_message_count, total_queue_visits, - last_queue_visits, total_latency, last_latency, total_queue_latency, last_queue_latency); - - - summary.last_arrow_last_latency_ms = millis(last_latency).count(); - summary.last_arrow_queue_visit_count = total_queue_visits; // TODO: Why do we have last_queue_visits? - - if (total_message_count == 0) { - summary.last_arrow_avg_latency_ms = std::numeric_limits::infinity(); - summary.last_arrow_avg_queue_latency_ms = std::numeric_limits::infinity(); - } - else { - summary.last_arrow_avg_queue_latency_ms = millis(total_queue_latency).count() / total_message_count; - summary.last_arrow_avg_latency_ms = millis(total_latency).count() / total_message_count; - } - summary.last_arrow_last_queue_latency_ms = millis(last_queue_latency).count(); - - // TODO: Do we even want last_message_count? This is in [0, chunksize], nothing to do with - // measurement interval. total gives us the measurement interval here. - // What about Arrow::metrics::last_message_count? This would have to be the same... - -} - - -JWorker::JWorker(JArrowProcessingController* japc, JScheduler* scheduler, unsigned worker_id, unsigned cpu_id, unsigned location_id, bool pin_to_cpu) : - m_scheduler(scheduler), - m_japc(japc), - m_worker_id(worker_id), - m_cpu_id(cpu_id), - m_location_id(location_id), - m_pin_to_cpu(pin_to_cpu), - m_run_state(RunState::Stopped), - m_assignment(nullptr), - m_thread(nullptr) { - - m_arrow_metrics.clear(); - m_worker_metrics.clear(); -} - -JWorker::~JWorker() { - wait_for_stop(); -} - -void JWorker::start() { - // start() requires a wait_until_stopped() first. Otherwise everything gets very confused. - if (m_run_state == RunState::Stopped) { - - m_run_state = RunState::Running; - m_thread = new std::thread(&JWorker::loop, this); - - if (m_pin_to_cpu) { - JCpuInfo::PinThreadToCpu(m_thread, m_cpu_id); - } - } -} - -void JWorker::request_stop() { - if (m_run_state == RunState::Running) { - m_run_state = RunState::Stopping; - } -} - -void JWorker::wait_for_stop() { - if (m_thread != nullptr) { - if (m_run_state == RunState::TimedOut) { - m_thread->detach(); - // Thread has timed out. Rather than non-cooperatively killing it, - // we relinquish ownership of it but remember that it was ours once and - // is still out there, somewhere, biding its time - } - else { - m_thread->join(); - } - delete m_thread; - m_thread = nullptr; - if (m_run_state == RunState::Stopping) { - m_run_state = RunState::Stopped; - // By this point, the JWorker must either be Stopped or TimedOut - } - } -} - -const JException& JWorker::get_exception() const { - return m_exception; -} - -void JWorker::loop() { - using jclock_t = JWorkerMetrics::clock_t; - try { - LOG_DEBUG(logger) << "Worker " << m_worker_id << " has entered loop()." << LOG_END; - JArrowMetrics::Status last_result = JArrowMetrics::Status::NotRunYet; - - while (m_run_state == RunState::Running) { - - auto start_time = jclock_t::now(); - - { - std::lock_guard lock(m_assignment_mutex); - m_assignment = m_scheduler->next_assignment(m_worker_id, m_assignment, last_result); - } - last_result = JArrowMetrics::Status::NotRunYet; - - auto scheduler_time = jclock_t::now(); - - auto scheduler_duration = scheduler_time - start_time; - auto idle_duration = jclock_t::duration::zero(); - auto retry_duration = jclock_t::duration::zero(); - auto useful_duration = jclock_t::duration::zero(); - - if (m_assignment == nullptr) { - LOG_DEBUG(logger) << "Worker " << m_worker_id << " shutdown driven by topology pause" << LOG_END; - m_run_state = RunState::Stopped; - return; - - // LOG_DEBUG(logger) << "Worker " << m_worker_id << " idling due to lack of assignments" << LOG_END; - // std::this_thread::sleep_for(std::chrono::microseconds(1)); - // idle_duration = jclock_t::now() - scheduler_time; - } - else { - - uint32_t current_tries = 0; - auto backoff_duration = m_initial_backoff_time; - - while (current_tries <= m_backoff_tries && - (last_result == JArrowMetrics::Status::KeepGoing || last_result == JArrowMetrics::Status::ComeBackLater || last_result == JArrowMetrics::Status::NotRunYet) && - (m_run_state == RunState::Running) && - (jclock_t::now() - start_time) < m_checkin_time) { - - LOG_TRACE(logger) << "Worker " << m_worker_id << " is executing " - << m_assignment->get_name() << LOG_END; - auto before_execute_time = jclock_t::now(); - m_assignment->execute(m_arrow_metrics, m_location_id); - last_result = m_arrow_metrics.get_last_status(); - useful_duration += (jclock_t::now() - before_execute_time); - - - if (last_result == JArrowMetrics::Status::KeepGoing) { - LOG_DEBUG(logger) << "Worker " << m_worker_id << " succeeded at " - << m_assignment->get_name() << LOG_END; - current_tries = 0; - backoff_duration = m_initial_backoff_time; - } - else { - current_tries++; - if (m_backoff_tries > 0) { - if (m_backoff_strategy == BackoffStrategy::Linear) { - backoff_duration += m_initial_backoff_time; - } - else if (m_backoff_strategy == BackoffStrategy::Exponential) { - backoff_duration *= 2; - } - LOG_TRACE(logger) << "Worker " << m_worker_id << " backing off with " - << m_assignment->get_name() << ", tries = " << current_tries - << LOG_END; - - std::this_thread::sleep_for(backoff_duration); - retry_duration += backoff_duration; - } - } - } - } - m_worker_metrics.update(start_time, 1, useful_duration, retry_duration, scheduler_duration, idle_duration); - if (m_assignment != nullptr) { - JArrowMetrics latest_arrow_metrics; - latest_arrow_metrics.clear(); - latest_arrow_metrics.take(m_arrow_metrics); // move local arrow metrics onto stack - m_assignment->get_metrics().update(latest_arrow_metrics); // propagate to global arrow context - } - } - - m_scheduler->last_assignment(m_worker_id, m_assignment, last_result); - m_assignment = nullptr; // Worker has 'handed in' the assignment - LOG_DEBUG(logger) << "Worker " << m_worker_id << " shutdown due to worker->request_stop()." << LOG_END; - } - catch (const JException& e) { - // For now the excepting Worker prints the error, and then terminates the whole program. - // Eventually we want to unify error handling across JApplication::Run, and maybe even across the entire JApplication. - // This means that Workers must pass JExceptions back to the master thread. - LOG_INFO(logger) << "Worker " << m_worker_id << " shutdown due to JException: " << e.what() << LOG_END; - LOG_DEBUG(logger) << e << LOG_END; - m_run_state = RunState::Excepted; - m_exception = e; - m_japc->request_pause(); // We aren't going to even try to drain queues. - } - catch (std::runtime_error& e){ - // same as above - LOG_INFO(logger) << "Worker " << m_worker_id << " shutdown due to std::runtime_error:" << e.what() << LOG_END; - LOG_DEBUG(logger) << e.what() << LOG_END; - m_run_state = RunState::Excepted; - m_exception = JException(e.what()); - m_exception.nested_exception = std::current_exception(); - m_japc->request_pause(); // We aren't going to even try to drain queues. - } -} - - - - - - diff --git a/src/libraries/JANA/Engine/JWorker.h b/src/libraries/JANA/Engine/JWorker.h deleted file mode 100644 index fae64290c..000000000 --- a/src/libraries/JANA/Engine/JWorker.h +++ /dev/null @@ -1,97 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include -#include -#include -#include -#include - - -class JArrowProcessingController; - -class JWorker { - /// Designed so that the Worker checks in with the Scheduler on his own terms; - /// i.e. nobody will update the worker's assignment externally. This eliminates - /// a whole lot of synchronization since we can assume - /// that the Worker's internal state won't be updated by another thread. - -public: - enum class RunState { Running, Stopping, Stopped, TimedOut, Excepted }; - - enum class BackoffStrategy { Constant, Linear, Exponential }; - - using duration_t = std::chrono::steady_clock::duration; - - /// The logger is made public so that somebody else may set it - JLogger logger; - -private: - /// Machinery that nobody else should modify. These should be protected eventually. - /// Probably simply make them private and expose via get_status() -> Worker::Status - JScheduler* m_scheduler; - JArrowProcessingController* m_japc; // This is used to turn off the other workers if an exception happens - unsigned m_worker_id; - unsigned m_cpu_id; - unsigned m_location_id; - bool m_pin_to_cpu; - std::atomic m_run_state; - JArrow* m_assignment; - std::thread* m_thread; // JWorker encapsulates a thread of some kind. Nothing else should care how. - JWorkerMetrics m_worker_metrics; - JArrowMetrics m_arrow_metrics; - std::mutex m_assignment_mutex; - JException m_exception; - - BackoffStrategy m_backoff_strategy = BackoffStrategy::Exponential; - duration_t m_initial_backoff_time = std::chrono::microseconds(1); - duration_t m_checkin_time = std::chrono::milliseconds(500); - unsigned m_backoff_tries = 4; - -public: - JWorker(JArrowProcessingController* japc, JScheduler* scheduler, unsigned worker_id, unsigned cpu_id, unsigned domain_id, bool pin_to_cpu); - ~JWorker(); - - /// If we copy or move the Worker, the underlying std::thread will be left with a - /// dangling pointer back to `this`. So we forbid copying, assigning, and moving. - - JWorker(const JWorker &other) = delete; - JWorker(JWorker &&other) = delete; - JWorker &operator=(const JWorker &other) = delete; - - RunState get_runstate() { return m_run_state; }; - - void start(); - void request_stop(); - void wait_for_stop(); - void declare_timeout(); - const JException& get_exception() const; - - /// This is what the encapsulated thread is supposed to be doing - void loop(); - - /// Summarize what/how this Worker is doing. This is meant to be called from - /// JProcessingController::measure_perf() - void measure_perf(WorkerSummary& result); - - inline void set_backoff_tries(unsigned backoff_tries) { m_backoff_tries = backoff_tries; } - - inline unsigned get_backoff_tries() const { return m_backoff_tries; } - - inline BackoffStrategy get_backoff_strategy() const { return m_backoff_strategy; } - - inline void set_backoff_strategy(BackoffStrategy backoff_strategy) { m_backoff_strategy = backoff_strategy; } - - inline duration_t get_initial_backoff_time() const { return m_initial_backoff_time; } - - inline void set_initial_backoff_time(duration_t initial_backoff_time) { m_initial_backoff_time = initial_backoff_time; } - - inline duration_t get_checkin_time() const { return m_checkin_time; } - - inline void set_checkin_time(duration_t checkin_time) { m_checkin_time = checkin_time; } - -}; - diff --git a/src/libraries/JANA/Engine/JWorkerMetrics.h b/src/libraries/JANA/Engine/JWorkerMetrics.h deleted file mode 100644 index eaba34775..000000000 --- a/src/libraries/JANA/Engine/JWorkerMetrics.h +++ /dev/null @@ -1,143 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include - -class JWorkerMetrics { - /// Workers need to maintain metrics. Similar to Arrow::Metrics, these form a monoid - /// where the identity element is (0,0,0) and the combine operation accumulates totals - /// in a thread-safe way. Alas, the combine operation mutates state for performance reasons. - /// We've separated Metrics from the Worker itself because it is not always obvious - /// who should be performing the accumulation or when, and this gives us the freedom to - /// try different possibilities. - -public: - using clock_t = std::chrono::steady_clock; - using duration_t = clock_t::duration; - using time_point_t = clock_t::time_point; - -private: - mutable std::mutex m_mutex; - // mutex is mutable so that we can lock before reading from a const ref - - time_point_t m_last_heartbeat; - long m_scheduler_visit_count; - - duration_t m_total_useful_time; - duration_t m_total_retry_time; - duration_t m_total_scheduler_time; - duration_t m_total_idle_time; - duration_t m_last_useful_time; - duration_t m_last_retry_time; - duration_t m_last_scheduler_time; - duration_t m_last_idle_time; - - -public: - JWorkerMetrics() { - m_mutex.lock(); - m_last_heartbeat = clock_t::now(); - m_scheduler_visit_count = 0; - auto zero = duration_t::zero(); - m_total_useful_time = zero; - m_total_retry_time = zero; - m_total_scheduler_time = zero; - m_total_idle_time = zero; - m_last_useful_time = zero; - m_last_retry_time = zero; - m_last_scheduler_time = zero; - m_last_idle_time = zero; - m_mutex.unlock(); - } - - void clear() { - m_mutex.lock(); - m_last_heartbeat = clock_t::now(); - m_scheduler_visit_count = 0; - auto zero = duration_t::zero(); - m_total_useful_time = zero; - m_total_retry_time = zero; - m_total_scheduler_time = zero; - m_total_idle_time = zero; - m_last_useful_time = zero; - m_last_retry_time = zero; - m_last_scheduler_time = zero; - m_last_idle_time = zero; - m_mutex.unlock(); - } - - - void update(const JWorkerMetrics &other) { - - m_mutex.lock(); - other.m_mutex.lock(); - m_last_heartbeat = other.m_last_heartbeat; - m_scheduler_visit_count += other.m_scheduler_visit_count; - m_total_useful_time += other.m_total_useful_time; - m_total_retry_time += other.m_total_retry_time; - m_total_scheduler_time += other.m_total_scheduler_time; - m_total_idle_time += other.m_total_idle_time; - m_last_useful_time = other.m_last_useful_time; - m_last_retry_time = other.m_last_retry_time; - m_last_scheduler_time = other.m_last_scheduler_time; - m_last_idle_time = other.m_last_idle_time; - other.m_mutex.unlock(); - m_mutex.unlock(); - } - - - void update( - const time_point_t& heartbeat, - const long& scheduler_visit_count, - const duration_t& useful_time, - const duration_t& retry_time, - const duration_t& scheduler_time, - const duration_t& idle_time) { - - m_mutex.lock(); - m_scheduler_visit_count += scheduler_visit_count; - m_total_useful_time += useful_time; - m_total_retry_time += retry_time; - m_total_scheduler_time += scheduler_time; - m_total_idle_time += idle_time; - m_last_useful_time = useful_time; - m_last_retry_time = retry_time; - m_last_scheduler_time = scheduler_time; - m_last_idle_time = idle_time; - m_last_heartbeat = heartbeat; - m_mutex.unlock(); - } - - - void get( - time_point_t& last_heartbeat, - long& scheduler_visit_count, - duration_t& total_useful_time, - duration_t& total_retry_time, - duration_t& total_scheduler_time, - duration_t& total_idle_time, - duration_t& last_useful_time, - duration_t& last_retry_time, - duration_t& last_scheduler_time, - duration_t& last_idle_time) { - - m_mutex.lock(); - scheduler_visit_count = m_scheduler_visit_count; - total_useful_time = m_total_useful_time; - total_retry_time = m_total_retry_time; - total_scheduler_time = m_total_scheduler_time; - total_idle_time = m_total_idle_time; - last_useful_time = m_last_useful_time; - last_retry_time = m_last_retry_time; - last_scheduler_time = m_last_scheduler_time; - last_idle_time = m_last_idle_time; - last_heartbeat = m_last_heartbeat; - m_mutex.unlock(); - } - -}; - - diff --git a/src/libraries/JANA/JApplication.cc b/src/libraries/JANA/JApplication.cc index dbcd2b5bb..7fb9ae402 100644 --- a/src/libraries/JANA/JApplication.cc +++ b/src/libraries/JANA/JApplication.cc @@ -3,21 +3,18 @@ // Subject to the terms in the LICENSE file found in the top-level directory. #include - #include - -#include -#include +#include +#include #include +#include #include -#include -#include #include -#include -#include -#include +#include +#include #include +#include #include #include @@ -43,14 +40,14 @@ JApplication::JApplication(JParameterManager* params) { m_component_manager = std::make_shared(); m_plugin_loader = std::make_shared(); m_service_locator = std::make_unique(); + m_execution_engine = std::make_unique(); ProvideService(m_params); ProvideService(m_component_manager); ProvideService(m_plugin_loader); - ProvideService(std::make_shared()); + ProvideService(m_execution_engine); ProvideService(std::make_shared()); ProvideService(std::make_shared()); - } @@ -110,25 +107,32 @@ void JApplication::Initialize() { // Only run this once if (m_initialized) return; - std::ostringstream oss; - oss << "Initializing..." << std::endl; - JVersion::PrintSplash(oss); - JVersion::PrintVersionDescription(oss); - LOG_INFO(m_logger) << oss.str() << LOG_END; - // Now that all parameters, components, plugin names, etc have been set, // we can expose our builtin services to the user via GetService() m_services_available = true; - + // We trigger initialization - auto logging_service = m_service_locator->get(); + m_service_locator->get(); auto component_manager = m_service_locator->get(); auto plugin_loader = m_service_locator->get(); auto topology_builder = m_service_locator->get(); // Set logger on JApplication itself - m_logger = logging_service->get_logger("JApplication"); - m_logger.show_classname = false; + m_logger = m_params->GetLogger("jana"); + + if (m_logger.level > JLogger::Level::INFO) { + std::ostringstream oss; + oss << "Initializing..." << std::endl << std::endl; + JVersion::PrintVersionDescription(oss); + LOG_WARN(m_logger) << oss.str() << LOG_END; + } + else { + std::ostringstream oss; + oss << "Initializing..." << std::endl; + JVersion::PrintSplash(oss); + JVersion::PrintVersionDescription(oss); + LOG_WARN(m_logger) << oss.str() << LOG_END; + } // Set up wiring ProvideService(std::make_shared()); @@ -146,14 +150,8 @@ void JApplication::Initialize() { m_desired_nthreads = JCpuInfo::GetNumCpus(); } - m_params->SetDefaultParameter("jana:ticker_interval", m_ticker_interval_ms, "Controls the ticker interval (in ms)"); - m_params->SetDefaultParameter("jana:extended_report", m_extended_report, "Controls whether the ticker shows simple vs detailed performance metrics"); - topology_builder->create_topology(); - ProvideService(std::make_shared()); - m_processing_controller = m_service_locator->get(); // Get deps from SL - m_processing_controller->initialize(); - + auto execution_engine = m_service_locator->get(); m_initialized = true; // This needs to be at the end so that m_initialized==false while InitPlugin() is being called } @@ -176,7 +174,7 @@ void JApplication::Initialize() { /// See JProcessingController::run() for more details. /// /// @param [in] wait_until_finished If true (default) do not return until the work has completed. -void JApplication::Run(bool wait_until_finished) { +void JApplication::Run(bool wait_until_stopped, bool finish) { Initialize(); if(m_quitting) return; @@ -193,90 +191,30 @@ void JApplication::Run(bool wait_until_finished) { // Print summary of all config parameters m_params->PrintParameters(); - LOG_INFO(m_logger) << GetComponentSummary() << LOG_END; - LOG_INFO(m_logger) << "Starting processing with " << m_desired_nthreads << " threads requested..." << LOG_END; - m_processing_controller->run(m_desired_nthreads); + LOG_WARN(m_logger) << "Starting processing with " << m_desired_nthreads << " threads requested..." << LOG_END; + m_execution_engine->ScaleWorkers(m_desired_nthreads); + m_execution_engine->RunTopology(); - if (!wait_until_finished) { + if (!wait_until_stopped) { return; } - // Monitor status of all threads - while (!m_quitting) { - - // If we are finishing up (all input sources are closed, and are waiting for all events to finish processing) - // This flag is used by the integrated rate calculator - if (!m_draining_queues) { - bool draining = true; - for (auto evt_src : m_component_manager->get_evt_srces()) { - draining &= (evt_src->GetStatus() == JEventSource::Status::Finalized); - } - m_draining_queues = draining; - } - - // Run until topology is deactivated, either because it finished or because another thread called stop() - if (m_processing_controller->is_stopped()) { - LOG_INFO(m_logger) << "All workers have stopped." << LOG_END; - break; - } - if (m_processing_controller->is_finished()) { - LOG_INFO(m_logger) << "All workers have finished." << LOG_END; - break; - } - - // Sleep a few cycles - std::this_thread::sleep_for(std::chrono::milliseconds(m_ticker_interval_ms)); - - // Print status - if( m_ticker_on ) PrintStatus(); - - // Test for timeout - if(m_timeout_on && m_processing_controller->is_timed_out()) { - LOG_FATAL(m_logger) << "Detected timeout in worker! Stopping." << LOG_END; - SetExitCode((int) ExitCode::Timeout); - m_processing_controller->request_stop(); - // TODO: Timeout error is not obvious enough from UI. Maybe throw an exception instead? - // TODO: Send USR2 signal to obtain a backtrace for timed out threads? - break; - } - - // Test for exception - if (m_processing_controller->is_excepted()) { - LOG_FATAL(m_logger) << "Detected exception in worker! Re-throwing on main thread." << LOG_END; - SetExitCode((int) ExitCode::UnhandledException); - m_processing_controller->request_stop(); - - // We are going to throw the first exception and ignore the others. - throw m_processing_controller->get_exceptions()[0]; - } - - if (m_inspecting) { - Inspect(); - } + m_execution_engine->RunSupervisor(); + if (finish) { + m_execution_engine->FinishTopology(); } // Join all threads if (!m_skip_join) { - LOG_INFO(m_logger) << "Merging threads ..." << LOG_END; - m_processing_controller->wait_until_stopped(); - } - - LOG_INFO(m_logger) << "Event processing ended." << LOG_END; - PrintFinalReport(); - - // Test for exception one more time, in case it shut down the topology before the supervisor could detect it - if (m_processing_controller->is_excepted()) { - LOG_FATAL(m_logger) << "Exception in worker!" << LOG_END; - SetExitCode((int) ExitCode::UnhandledException); - // We are going to throw the first exception and ignore the others. - throw m_processing_controller->get_exceptions()[0]; + m_execution_engine->ScaleWorkers(0); } } void JApplication::Scale(int nthreads) { - LOG_INFO(m_logger) << "Scaling to " << nthreads << " threads" << LOG_END; - m_processing_controller->scale(nthreads); + LOG_WARN(m_logger) << "Scaling to " << nthreads << " threads" << LOG_END; + m_execution_engine->ScaleWorkers(nthreads); + m_execution_engine->RunTopology(); } void JApplication::Inspect() { @@ -287,7 +225,7 @@ void JApplication::Inspect() { m_inspecting = false; } -void JApplication::Stop(bool wait_until_idle) { +void JApplication::Stop(bool wait_until_stopped, bool finish) { if (!m_initialized) { // People might call Stop() during Initialize() rather than Run(). // For instance, during JEventProcessor::Init, or via Ctrl-C. @@ -299,11 +237,13 @@ void JApplication::Stop(bool wait_until_idle) { else { // Once we've called Initialize(), we can Finish() all of our components // whenever we like - m_processing_controller->request_stop(); - if (wait_until_idle) { - m_processing_controller->wait_until_stopped(); + m_execution_engine->DrainTopology(); + if (wait_until_stopped) { + m_execution_engine->RunSupervisor(); + if (finish) { + m_execution_engine->FinishTopology(); + } } - } } @@ -312,7 +252,7 @@ void JApplication::Quit(bool skip_join) { if (m_initialized) { m_skip_join = skip_join; m_quitting = true; - if (!skip_join && m_processing_controller != nullptr) { + if (!skip_join && m_execution_engine != nullptr) { Stop(true); } } @@ -344,29 +284,6 @@ int JApplication::GetExitCode() { return m_exit_code; } -void JApplication::HandleSigint() { - m_sigint_count++; - switch (m_sigint_count) { - case 1: - LOG_WARN(m_logger) << "Entering Inspector..." << LOG_END; - m_inspecting = true; - m_processing_controller->request_pause(); - break; - case 2: - LOG_FATAL(m_logger) << "Exiting gracefully..." << LOG_END; - japp->Quit(false); - break; - case 3: - LOG_FATAL(m_logger) << "Exiting without waiting for threads to join..." << LOG_END; - japp->Quit(true); - break; - default: - LOG_FATAL(m_logger) << "Exiting immediately." << LOG_END; - exit(-2); - } - -} - const JComponentSummary& JApplication::GetComponentSummary() { /// Returns a data object describing all components currently running return m_component_manager->get_component_summary(); @@ -374,82 +291,64 @@ const JComponentSummary& JApplication::GetComponentSummary() { // Performance/status monitoring void JApplication::SetTicker(bool ticker_on) { - m_ticker_on = ticker_on; + m_execution_engine->SetTickerEnabled(ticker_on); } bool JApplication::IsTickerEnabled() { - return m_ticker_on; + return m_execution_engine->IsTickerEnabled(); } void JApplication::SetTimeoutEnabled(bool enabled) { - m_timeout_on = enabled; + m_execution_engine->SetTimeoutEnabled(enabled); } bool JApplication::IsTimeoutEnabled() { - return m_timeout_on; -} - -void JApplication::PrintStatus() { - if (m_extended_report) { - m_processing_controller->print_report(); - } - else { - std::lock_guard lock(m_status_mutex); - update_status(); - LOG_INFO(m_logger) << "Status: " << m_perf_summary->total_events_completed << " events processed " - << JTypeInfo::to_string_with_si_prefix(m_perf_summary->latest_throughput_hz) << "Hz (" - << JTypeInfo::to_string_with_si_prefix(m_perf_summary->avg_throughput_hz) << "Hz avg)" << LOG_END; - } -} - -void JApplication::PrintFinalReport() { - m_processing_controller->print_final_report(); + return m_execution_engine->IsTimeoutEnabled(); } -/// Performs a new measurement if the time elapsed since the previous measurement exceeds some threshold -void JApplication::update_status() { - auto now = std::chrono::high_resolution_clock::now(); - if ((now - m_last_measurement) >= std::chrono::milliseconds(m_ticker_interval_ms) || m_perf_summary == nullptr) { - m_perf_summary = m_processing_controller->measure_performance(); - m_last_measurement = now; - } +bool JApplication::IsDrainingQueues() { + return (m_execution_engine->GetRunStatus() == JExecutionEngine::RunStatus::Draining); } /// Returns the number of threads currently being used. -/// Note: This data gets stale. If you need event counts and rates -/// which are more consistent with one another, call GetStatus() instead. uint64_t JApplication::GetNThreads() { - std::lock_guard lock(m_status_mutex); - update_status(); - return m_perf_summary->thread_count; + return m_execution_engine->GetPerf().thread_count; } /// Returns the number of events processed since Run() was called. -/// Note: This data gets stale. If you need event counts and rates -/// which are more consistent with one another, call GetStatus() instead. uint64_t JApplication::GetNEventsProcessed() { - std::lock_guard lock(m_status_mutex); - update_status(); - return m_perf_summary->total_events_completed; + return m_execution_engine->GetPerf().event_count; } /// Returns the total integrated throughput so far in Hz since Run() was called. -/// Note: This data gets stale. If you need event counts and rates -/// which are more consistent with one another, call GetStatus() instead. float JApplication::GetIntegratedRate() { - std::lock_guard lock(m_status_mutex); - update_status(); - return m_perf_summary->avg_throughput_hz; + return m_execution_engine->GetPerf().throughput_hz; } -/// Returns the 'instantaneous' throughput in Hz since the last perf measurement was made. -/// Note: This data gets stale. If you need event counts and rates -/// which are more consistent with one another, call GetStatus() instead. +/// Returns the 'instantaneous' throughput in Hz since the last such call was made. float JApplication::GetInstantaneousRate() { - std::lock_guard lock(m_status_mutex); - update_status(); - return m_perf_summary->latest_throughput_hz; + std::lock_guard lock(m_inst_rate_mutex); + auto latest_event_count = m_execution_engine->GetPerf().event_count; + auto latest_time = JExecutionEngine::clock_t::now(); + + auto duration_ms = std::chrono::duration_cast(latest_time - m_last_measurement_time).count(); + auto instantaneous_throughput = (duration_ms == 0) ? 0 : (latest_event_count - m_last_event_count) * 1000.0 / duration_ms; + + m_last_event_count = latest_event_count; + m_last_measurement_time = latest_time; + + return instantaneous_throughput; } +void JApplication::PrintStatus() { + auto perf = m_execution_engine->GetPerf(); + LOG_INFO(m_logger) << "Topology status: " << ToString(perf.runstatus) << LOG_END; + LOG_INFO(m_logger) << "Worker thread count: " << perf.thread_count << LOG_END; + LOG_INFO(m_logger) << "Events processed: " << perf.event_count << LOG_END; + LOG_INFO(m_logger) << "Uptime [s]: " << perf.uptime_ms*1000 << LOG_END; + LOG_INFO(m_logger) << "Throughput [Hz]: " << perf.throughput_hz << LOG_END; +} + + diff --git a/src/libraries/JANA/JApplicationFwd.h b/src/libraries/JANA/JApplicationFwd.h index 3adbcfe52..4770caf9f 100644 --- a/src/libraries/JANA/JApplicationFwd.h +++ b/src/libraries/JANA/JApplicationFwd.h @@ -17,7 +17,7 @@ class JFactoryGenerator; class JFactorySet; class JComponentManager; class JPluginLoader; -class JArrowProcessingController; +class JExecutionEngine; class JEventUnfolder; class JServiceLocator; class JParameter; @@ -26,7 +26,6 @@ class JApplication; extern JApplication* japp; #include -#include #include @@ -72,29 +71,24 @@ class JApplication { // Controlling processing void Initialize(void); - void Run(bool wait_until_finished = true); + void Run(bool wait_until_stopped=true, bool finish=true); void Scale(int nthreads); - void Stop(bool wait_until_idle = false); - void Resume() {}; // TODO: Do we need this? + void Stop(bool wait_until_stopped=false, bool finish=true); void Inspect(); void Quit(bool skip_join = false); void SetExitCode(int exitCode); int GetExitCode(); - void HandleSigint(); - // Performance/status monitoring - + void PrintStatus(); bool IsInitialized(void){return m_initialized;} bool IsQuitting(void) { return m_quitting; } - bool IsDrainingQueues(void) { return m_draining_queues; } + bool IsDrainingQueues(); void SetTicker(bool ticker_on = true); bool IsTickerEnabled(); void SetTimeoutEnabled(bool enabled = true); bool IsTimeoutEnabled(); - void PrintStatus(); - void PrintFinalReport(); uint64_t GetNThreads(); uint64_t GetNEventsProcessed(); float GetIntegratedRate(); @@ -143,27 +137,21 @@ class JApplication { std::shared_ptr m_params; std::shared_ptr m_plugin_loader; std::shared_ptr m_component_manager; - std::shared_ptr m_processing_controller; + std::shared_ptr m_execution_engine; bool m_inspecting = false; bool m_quitting = false; - bool m_draining_queues = false; bool m_skip_join = false; std::atomic_bool m_initialized {false}; std::atomic_bool m_services_available {false}; - bool m_ticker_on = true; - bool m_timeout_on = true; - bool m_extended_report = false; int m_exit_code = (int) ExitCode::Success; int m_desired_nthreads; std::atomic_int m_sigint_count {0}; - std::mutex m_status_mutex; - int m_ticker_interval_ms = 1000; - std::chrono::time_point m_last_measurement; - std::unique_ptr m_perf_summary; - - void update_status(); + // For instantaneous rate calculations + std::mutex m_inst_rate_mutex; + std::chrono::steady_clock::time_point m_last_measurement_time = std::chrono::steady_clock::now(); + size_t m_last_event_count = 0; }; diff --git a/src/libraries/JANA/JEvent.cc b/src/libraries/JANA/JEvent.cc new file mode 100644 index 000000000..6692828aa --- /dev/null +++ b/src/libraries/JANA/JEvent.cc @@ -0,0 +1,126 @@ + +#include +#include + + +JEvent::JEvent() : mInspector(this){ + mFactorySet = new JFactorySet(); +} + +JEvent::JEvent(JApplication* app) : mInspector(this) { + // Furnish the JEvent with the parameter values and factory generators provided to the JApplication + app->Initialize(); + app->GetService()->configure_event(*this); +} + +JEvent::~JEvent() { + if (mFactorySet != nullptr) { + // Prevent memory leaks of factory contents + mFactorySet->Clear(); + // We mustn't call EndRun() or Finish() here because that would give us an excepting destructor + } + delete mFactorySet; +} + +void JEvent::SetFactorySet(JFactorySet* factorySet) { + delete mFactorySet; + mFactorySet = factorySet; +#if JANA2_HAVE_PODIO + // Maintain the index of PODIO factories + for (JFactory* factory : mFactorySet->GetAllFactories()) { + if (dynamic_cast(factory) != nullptr) { + auto tag = factory->GetTag(); + auto it = mPodioFactories.find(tag); + if (it != mPodioFactories.end()) { + throw JException("SetFactorySet failed because PODIO factory tag '%s' is not unique", tag.c_str()); + } + mPodioFactories[tag] = factory; + } + } +#endif +} + + +/// GetFactory() should be used with extreme care because it subverts the JEvent abstraction. +/// Most historical uses of GetFactory are far better served by JMultifactory +JFactory* JEvent::GetFactory(const std::string& object_name, const std::string& tag) const { + return mFactorySet->GetFactory(object_name, tag); +} + +/// GetAllFactories() should be used with extreme care because it subverts the JEvent abstraction. +/// Most historical uses of GetFactory are far better served by JMultifactory +std::vector JEvent::GetAllFactories() const { + return mFactorySet->GetAllFactories(); +} + +bool JEvent::HasParent(JEventLevel level) const { + for (const auto& pair : mParents) { + if (pair.first == level) return true; + } + return false; +} + +const JEvent& JEvent::GetParent(JEventLevel level) const { + for (const auto& pair : mParents) { + if (pair.first == level) return *pair.second; + } + throw JException("Unable to find parent at level %s", + toString(level).c_str()); +} + +void JEvent::SetParent(JEvent* parent) { + JEventLevel level = parent->GetLevel(); + for (const auto& pair : mParents) { + if (pair.first == level) throw JException("Event already has a parent at level %s", + toString(parent->GetLevel()).c_str()); + } + mParents.push_back({level, parent}); + parent->mReferenceCount.fetch_add(1); +} + +JEvent* JEvent::ReleaseParent(JEventLevel level) { + if (mParents.size() == 0) { + throw JException("ReleaseParent failed: child has no parents!"); + } + auto pair = mParents.back(); + if (pair.first != level) { + throw JException("JEvent::ReleaseParent called out of level order: Caller expected %s, but parent was actually %s", + toString(level).c_str(), toString(pair.first).c_str()); + } + mParents.pop_back(); + auto remaining_refs = pair.second->mReferenceCount.fetch_sub(1); + if (remaining_refs < 1) { // Remember, this was fetched _before_ the last subtraction + throw JException("Parent refcount has gone negative!"); + } + if (remaining_refs == 1) { + return pair.second; + // Parent is no longer shared. Transfer back to arrow + } + else { + return nullptr; // Parent is still shared by other children + } +} + +void JEvent::Release() { + auto remaining_refs = mReferenceCount.fetch_sub(1); + if (remaining_refs < 0) { + throw JException("JEvent's own refcount has gone negative!"); + } +} + +void JEvent::Clear() { + if (mEventSource != nullptr) { + mEventSource->DoFinishEvent(*this); + } + mFactorySet->Clear(); + mInspector.Reset(); + mCallGraph.Reset(); + mReferenceCount = 1; + mIsWarmedUp = true; +} + +void JEvent::Finish() { + mFactorySet->Finish(); +} + + diff --git a/src/libraries/JANA/JEvent.h b/src/libraries/JANA/JEvent.h index e60e7cbf7..c570e239d 100644 --- a/src/libraries/JANA/JEvent.h +++ b/src/libraries/JANA/JEvent.h @@ -9,7 +9,6 @@ #include #include #include - #include #include @@ -22,9 +21,7 @@ #include #include #include -#include #include -#include #if JANA2_HAVE_PODIO #include @@ -37,245 +34,108 @@ class JApplication; class JEventSource; -class JEvent : public std::enable_shared_from_this -{ - public: - - explicit JEvent(JApplication* aApplication=nullptr) : mInspector(&(*this)) { - mApplication = aApplication; - mFactorySet = new JFactorySet(); - } - virtual ~JEvent() { - if (mFactorySet != nullptr) mFactorySet->Release(); - delete mFactorySet; - } - - void SetFactorySet(JFactorySet* aFactorySet) { - delete mFactorySet; - mFactorySet = aFactorySet; -#if JANA2_HAVE_PODIO - // Maintain the index of PODIO factories - for (JFactory* factory : mFactorySet->GetAllFactories()) { - if (dynamic_cast(factory) != nullptr) { - auto tag = factory->GetTag(); - auto it = mPodioFactories.find(tag); - if (it != mPodioFactories.end()) { - throw JException("SetFactorySet failed because PODIO factory tag '%s' is not unique", tag.c_str()); - } - mPodioFactories[tag] = factory; - } - } -#endif - } - - JFactorySet* GetFactorySet() const { return mFactorySet; } - - JFactory* GetFactory(const std::string& object_name, const std::string& tag) const; - std::vector GetAllFactories() const; - template JFactoryT* GetFactory(const std::string& tag = "", bool throw_on_missing=false) const; - template std::vector*> GetFactoryAll(bool throw_on_missing = false) const; - - template JMetadata GetMetadata(const std::string& tag = "") const; +class JEvent : public std::enable_shared_from_this { - //OBJECTS - // C style getters - template JFactoryT* Get(const T** item, const std::string& tag="") const; - template JFactoryT* Get(std::vector &vec, const std::string& tag = "", bool strict=true) const; - template void GetAll(std::vector &vec) const; +private: + JApplication* mApplication = nullptr; + int32_t mRunNumber = 0; + uint64_t mEventNumber = 0; + mutable JFactorySet* mFactorySet = nullptr; + mutable JCallGraphRecorder mCallGraph; + mutable JInspector mInspector; + bool mUseDefaultTags = false; + std::map mDefaultTags; + JEventSource* mEventSource = nullptr; + bool mIsBarrierEvent = false; + bool mIsWarmedUp = false; - // C++ style getters - template const T* GetSingle(const std::string& tag = "") const; - template const T* GetSingleStrict(const std::string& tag = "") const; - template std::vector Get(const std::string& tag = "", bool strict=true) const; - template typename JFactoryT::PairType GetIterators(const std::string& aTag = "") const; - template std::vector GetAll() const; - template std::map,std::vector> GetAllChildren() const; + // Hierarchical event memory management + std::vector> mParents; + std::atomic_int mReferenceCount {1}; + int64_t mEventIndex = -1; - // JANA1 compatibility getters - template JFactoryT* GetSingle(const T* &t, const char *tag="", bool exception_if_not_one=true) const; - - // Insert - template JFactoryT* Insert(T* item, const std::string& aTag = "") const; - template JFactoryT* Insert(const std::vector& items, const std::string& tag = "") const; - - // PODIO #if JANA2_HAVE_PODIO - std::vector GetAllCollectionNames() const; - const podio::CollectionBase* GetCollectionBase(std::string name, bool throw_on_missing=true) const; - template const typename JFactoryPodioT::CollectionT* GetCollection(std::string name, bool throw_on_missing=true) const; - template JFactoryPodioT* InsertCollection(typename JFactoryPodioT::CollectionT&& collection, std::string name); - template JFactoryPodioT* InsertCollectionAlreadyInFrame(const podio::CollectionBase* collection, std::string name); + std::map mPodioFactories; #endif - //SETTERS - void SetRunNumber(int32_t aRunNumber){mRunNumber = aRunNumber;} - void SetEventNumber(uint64_t aEventNumber){mEventNumber = aEventNumber;} - void SetJApplication(JApplication* app){mApplication = app;} - void SetJEventSource(JEventSource* aSource){mEventSource = aSource;} - void SetDefaultTags(std::map aDefaultTags){mDefaultTags=aDefaultTags; mUseDefaultTags = !mDefaultTags.empty();} - void SetSequential(bool isSequential) {mIsBarrierEvent = isSequential;} - - //GETTERS - int32_t GetRunNumber() const {return mRunNumber;} - uint64_t GetEventNumber() const {return mEventNumber;} - JApplication* GetJApplication() const {return mApplication;} - JEventSource* GetJEventSource() const {return mEventSource; } - JCallGraphRecorder* GetJCallGraphRecorder() const {return &mCallGraph;} - JInspector* GetJInspector() const {return &mInspector;} - void Inspect() const { mInspector.Loop();} // TODO: Force this not to be inlined AND used so it is defined in libJANA.a - bool GetSequential() const {return mIsBarrierEvent;} - friend class JEventPool; - - - // Hierarchical - JEventLevel GetLevel() const { return mFactorySet->GetLevel(); } - void SetLevel(JEventLevel level) { mFactorySet->SetLevel(level); } - void SetEventIndex(int event_index) { mEventIndex = event_index; } - int64_t GetEventIndex() const { return mEventIndex; } - - bool HasParent(JEventLevel level) const { - for (const auto& pair : mParents) { - if (pair.first == level) return true; - } - return false; - } - - const JEvent& GetParent(JEventLevel level) const { - for (const auto& pair : mParents) { - if (pair.first == level) return *(*(pair.second)); - } - throw JException("Unable to find parent at level %s", - toString(level).c_str()); - } - - void SetParent(std::shared_ptr* parent) { - JEventLevel level = parent->get()->GetLevel(); - for (const auto& pair : mParents) { - if (pair.first == level) throw JException("Event already has a parent at level %s", - toString(parent->get()->GetLevel()).c_str()); - } - mParents.push_back({level, parent}); - parent->get()->mReferenceCount.fetch_add(1); - } - - std::shared_ptr* ReleaseParent(JEventLevel level) { - if (mParents.size() == 0) { - throw JException("ReleaseParent failed: child has no parents!"); - } - auto pair = mParents.back(); - if (pair.first != level) { - throw JException("JEvent::ReleaseParent called out of level order: Caller expected %s, but parent was actually %s", - toString(level).c_str(), toString(pair.first).c_str()); - } - mParents.pop_back(); - auto remaining_refs = pair.second->get()->mReferenceCount.fetch_sub(1); - if (remaining_refs < 1) { // Remember, this was fetched _before_ the last subtraction - throw JException("Parent refcount has gone negative!"); - } - if (remaining_refs == 1) { - return pair.second; - // Parent is no longer shared. Transfer back to arrow - } - else { - return nullptr; // Parent is still shared by other children - } - } - - void Release() { - auto remaining_refs = mReferenceCount.fetch_sub(1); - if (remaining_refs < 0) { - throw JException("JEvent's own refcount has gone negative!"); - } - } - - void Reset() { - mReferenceCount = 1; - } - - - private: - JApplication* mApplication = nullptr; - int32_t mRunNumber = 0; - uint64_t mEventNumber = 0; - mutable JFactorySet* mFactorySet = nullptr; - mutable JCallGraphRecorder mCallGraph; - mutable JInspector mInspector; - bool mUseDefaultTags = false; - std::map mDefaultTags; - JEventSource* mEventSource = nullptr; - bool mIsBarrierEvent = false; - - // Hierarchical stuff - std::vector*>> mParents; - std::atomic_int mReferenceCount {1}; - int64_t mEventIndex = -1; - - +public: + JEvent(); + explicit JEvent(JApplication* app); + virtual ~JEvent(); + + void SetFactorySet(JFactorySet* aFactorySet); + void SetRunNumber(int32_t aRunNumber){mRunNumber = aRunNumber;} + void SetEventNumber(uint64_t aEventNumber){mEventNumber = aEventNumber;} + void SetJApplication(JApplication* app){mApplication = app;} + void SetJEventSource(JEventSource* aSource){mEventSource = aSource;} + void SetDefaultTags(std::map aDefaultTags){mDefaultTags=aDefaultTags; mUseDefaultTags = !mDefaultTags.empty();} + void SetSequential(bool isSequential) {mIsBarrierEvent = isSequential;} + + JFactorySet* GetFactorySet() const { return mFactorySet; } + int32_t GetRunNumber() const {return mRunNumber;} + uint64_t GetEventNumber() const {return mEventNumber;} + JApplication* GetJApplication() const {return mApplication;} + JEventSource* GetJEventSource() const {return mEventSource; } + JCallGraphRecorder* GetJCallGraphRecorder() const {return &mCallGraph;} + JInspector* GetJInspector() const {return &mInspector;} + void Inspect() const { mInspector.Loop();} + bool GetSequential() const {return mIsBarrierEvent;} + bool IsWarmedUp() { return mIsWarmedUp; } + + // Hierarchical + JEventLevel GetLevel() const { return mFactorySet->GetLevel(); } + void SetLevel(JEventLevel level) { mFactorySet->SetLevel(level); } + void SetEventIndex(int event_index) { mEventIndex = event_index; } + int64_t GetEventIndex() const { return mEventIndex; } + + bool HasParent(JEventLevel level) const; + const JEvent& GetParent(JEventLevel level) const; + void SetParent(JEvent* parent); + JEvent* ReleaseParent(JEventLevel level); + void Release(); + + // Lifecycle + void Clear(); + void Finish(); + + JFactory* GetFactory(const std::string& object_name, const std::string& tag) const; + std::vector GetAllFactories() const; + + + template JFactoryT* GetFactory(const std::string& tag = "", bool throw_on_missing=false) const; + template std::vector*> GetFactoryAll(bool throw_on_missing = false) const; + + // C style getters + template JFactoryT* GetSingle(const T* &t, const char *tag="", bool exception_if_not_one=true) const; + template JFactoryT* Get(const T** item, const std::string& tag="") const; + template JFactoryT* Get(std::vector &vec, const std::string& tag = "", bool strict=true) const; + template void GetAll(std::vector &vec) const; + + // C++ style getters + template const T* GetSingle(const std::string& tag = "") const; + template const T* GetSingleStrict(const std::string& tag = "") const; + template std::vector Get(const std::string& tag = "", bool strict=true) const; + template typename JFactoryT::PairType GetIterators(const std::string& aTag = "") const; + template std::vector GetAll() const; + template std::map,std::vector> GetAllChildren() const; + + // Insert + template JFactoryT* Insert(T* item, const std::string& aTag = "") const; + template JFactoryT* Insert(const std::vector& items, const std::string& tag = "") const; + + // PODIO #if JANA2_HAVE_PODIO - std::map mPodioFactories; + std::vector GetAllCollectionNames() const; + const podio::CollectionBase* GetCollectionBase(std::string name, bool throw_on_missing=true) const; + template const typename JFactoryPodioT::CollectionT* GetCollection(std::string name, bool throw_on_missing=true) const; + template JFactoryPodioT* InsertCollection(typename JFactoryPodioT::CollectionT&& collection, std::string name); + template JFactoryPodioT* InsertCollectionAlreadyInFrame(const podio::CollectionBase* collection, std::string name); #endif -}; -/// Insert() allows an EventSource to insert items directly into the JEvent, -/// removing the need for user-extended JEvents and/or JEventSource::GetObjects(...) -/// Repeated calls to Insert() will append to the previous data rather than overwrite it, -/// which saves the user from having to allocate a throwaway vector and requires less error handling. -template -inline JFactoryT* JEvent::Insert(T* item, const std::string& tag) const { - - std::string resolved_tag = tag; - if (mUseDefaultTags && tag.empty()) { - auto defaultTag = mDefaultTags.find(JTypeInfo::demangle()); - if (defaultTag != mDefaultTags.end()) resolved_tag = defaultTag->second; - } - auto factory = mFactorySet->GetFactory(resolved_tag); - if (factory == nullptr) { - factory = new JFactoryT; - factory->SetTag(tag); - factory->SetLevel(mFactorySet->GetLevel()); - mFactorySet->Add(factory); - } - factory->Insert(item); - factory->SetInsertOrigin( mCallGraph.GetInsertDataOrigin() ); // (see note at top of JCallGraphRecorder.h) - return factory; -} - -template -inline JFactoryT* JEvent::Insert(const std::vector& items, const std::string& tag) const { - - std::string resolved_tag = tag; - if (mUseDefaultTags && tag.empty()) { - auto defaultTag = mDefaultTags.find(JTypeInfo::demangle()); - if (defaultTag != mDefaultTags.end()) resolved_tag = defaultTag->second; - } - auto factory = mFactorySet->GetFactory(resolved_tag); - if (factory == nullptr) { - factory = new JFactoryT; - factory->SetTag(tag); - factory->SetLevel(mFactorySet->GetLevel()); - mFactorySet->Add(factory); - } - for (T* item : items) { - factory->Insert(item); - } - factory->SetStatus(JFactory::Status::Inserted); // for when items is empty - factory->SetCreationStatus(JFactory::CreationStatus::Inserted); // for when items is empty - factory->SetInsertOrigin( mCallGraph.GetInsertDataOrigin() ); // (see note at top of JCallGraphRecorder.h) - return factory; -} -/// GetFactory() should be used with extreme care because it subverts the JEvent abstraction. -/// Most historical uses of GetFactory are far better served by JMultifactory -inline JFactory* JEvent::GetFactory(const std::string& object_name, const std::string& tag) const { - return mFactorySet->GetFactory(object_name, tag); -} +}; -/// GetAllFactories() should be used with extreme care because it subverts the JEvent abstraction. -/// Most historical uses of GetFactory are far better served by JMultifactory -inline std::vector JEvent::GetAllFactories() const { - return mFactorySet->GetAllFactories(); -} /// GetFactory() should be used with extreme care because it subverts the JEvent abstraction. /// Most historical uses of GetFactory are far better served by JMultifactory. @@ -299,20 +159,52 @@ inline JFactoryT* JEvent::GetFactory(const std::string& tag, bool throw_on_mi } -/// GetMetadata() provides access to any metadata generated by the underlying JFactory during Process() +/// GetFactoryAll returns all JFactoryT's for type T (each corresponds to a different tag). +/// This is useful when there are many different tags, or the tags are unknown, and the user +/// wishes to examine them all together. template -[[deprecated("Use JMultifactory instead")]] -inline JMetadata JEvent::GetMetadata(const std::string& tag) const { - - auto factory = GetFactory(tag, true); - // Make sure that JFactoryT::Process has already been called before returning the metadata - factory->CreateAndGetData(this->shared_from_this()); - return factory->GetMetadata(); +inline std::vector*> JEvent::GetFactoryAll(bool throw_on_missing) const { + auto factories = mFactorySet->GetAllFactories(); + if (factories.size() == 0) { + if (throw_on_missing) { + JException ex("Could not find any JFactoryT<" + JTypeInfo::demangle() + "> (from any tag)"); + ex.show_stacktrace = false; + throw ex; + } + }; + return factories; } /// C-style getters +template +JFactoryT* JEvent::GetSingle(const T* &t, const char *tag, bool exception_if_not_one) const +{ + /// This is a convenience method that can be used to get a pointer to the single + /// object of type T from the specified factory. It simply calls the Get(vector<...>) method + /// and copies the first pointer into "t" (or NULL if something other than 1 object is returned). + /// + /// This is intended to address the common situation in which there is an interest + /// in the event if and only if there is exactly 1 object of type T. If the event + /// has no objects of that type or more than 1 object of that type (for the specified + /// factory) then an exception of type "unsigned long" is thrown with the value + /// being the number of objects of type T. You can supress the exception by setting + /// exception_if_not_one to false. In that case, you will have to check if t==NULL to + /// know if the call succeeded. + + std::vector v; + auto fac = GetFactory(tag, true); // throw exception if factory not found + JCallGraphEntryMaker cg_entry(mCallGraph, fac); // times execution until this goes out of scope + Get(v, tag); + if(v.size()!=1){ + t = NULL; + if(exception_if_not_one) throw v.size(); + } + t = v[0]; + return fac; +} + /// Get conveniently returns one item from inside the JFactory. This should be used when the data in question /// is optional and the caller wants to examine the result and decide how to proceed. The caller should embed this /// inside an if-block. Get updates the `destination` out parameter and returns a pointer to the enclosing JFactory. @@ -350,6 +242,20 @@ JFactoryT* JEvent::Get(std::vector& destination, const std::string& } +/// GetAll returns all JObjects of (child) type T, regardless of tag. +template +void JEvent::GetAll(std::vector& destination) const { + auto factories = GetFactoryAll(true); + for (auto factory : factories) { + auto iterators = factory->CreateAndGetData(this->shared_from_this()); + for (auto it = iterators.first; it != iterators.second; it++) { + destination.push_back(*it); + } + } +} + + + /// C++ style getters /// GetSingle conveniently returns one item from inside the JFactory. This should be used when the data in question @@ -369,6 +275,8 @@ template const T* JEvent::GetSingle(const std::string& tag) const { return *iterators.first; } + + /// GetSingleStrict conveniently returns one item from inside the JFactory. This should be used when the data in /// question is mandatory, and its absence indicates an error which should stop execution. The caller does not need /// to embed this in an if- or try-catch block; it can be a one-liner. @@ -407,32 +315,12 @@ std::vector JEvent::Get(const std::string& tag, bool strict) const { return vec; // Assumes RVO } -/// GetFactoryAll returns all JFactoryT's for type T (each corresponds to a different tag). -/// This is useful when there are many different tags, or the tags are unknown, and the user -/// wishes to examine them all together. -template -inline std::vector*> JEvent::GetFactoryAll(bool throw_on_missing) const { - auto factories = mFactorySet->GetAllFactories(); - if (factories.size() == 0) { - if (throw_on_missing) { - JException ex("Could not find any JFactoryT<" + JTypeInfo::demangle() + "> (from any tag)"); - ex.show_stacktrace = false; - throw ex; - } - }; - return factories; -} - -/// GetAll returns all JObjects of (child) type T, regardless of tag. template -void JEvent::GetAll(std::vector& destination) const { - auto factories = GetFactoryAll(true); - for (auto factory : factories) { - auto iterators = factory->CreateAndGetData(this->shared_from_this()); - for (auto it = iterators.first; it != iterators.second; it++) { - destination.push_back(*it); - } - } +typename JFactoryT::PairType JEvent::GetIterators(const std::string& tag) const { + auto factory = GetFactory(tag, true); + JCallGraphEntryMaker cg_entry(mCallGraph, factory); // times execution until this goes out of scope + auto iters = factory->CreateAndGetData(this->shared_from_this()); + return iters; } /// GetAll returns all JObjects of (child) type T, regardless of tag. @@ -451,7 +339,6 @@ std::vector JEvent::GetAll() const { return vec; // Assumes RVO } - // GetAllChildren will furnish a map { (type_name,tag_name) : [BaseClass*] } containing all JFactoryT data where // T inherits from BaseClass. Note that this _won't_ compute any results (unlike GetAll) because this is meant for // things like visualizing and persisting DSTs. @@ -471,42 +358,57 @@ std::map, std::vector> JEvent::GetAllChi } -template -typename JFactoryT::PairType JEvent::GetIterators(const std::string& tag) const { - auto factory = GetFactory(tag, true); - JCallGraphEntryMaker cg_entry(mCallGraph, factory); // times execution until this goes out of scope - auto iters = factory->CreateAndGetData(this->shared_from_this()); - return iters; -} +/// Insert() allows an EventSource to insert items directly into the JEvent, +/// removing the need for user-extended JEvents and/or JEventSource::GetObjects(...) +/// Repeated calls to Insert() will append to the previous data rather than overwrite it, +/// which saves the user from having to allocate a throwaway vector and requires less error handling. +template +inline JFactoryT* JEvent::Insert(T* item, const std::string& tag) const { -template -JFactoryT* JEvent::GetSingle(const T* &t, const char *tag, bool exception_if_not_one) const -{ - /// This is a convenience method that can be used to get a pointer to the single - /// object of type T from the specified factory. It simply calls the Get(vector<...>) method - /// and copies the first pointer into "t" (or NULL if something other than 1 object is returned). - /// - /// This is intended to address the common situation in which there is an interest - /// in the event if and only if there is exactly 1 object of type T. If the event - /// has no objects of that type or more than 1 object of that type (for the specified - /// factory) then an exception of type "unsigned long" is thrown with the value - /// being the number of objects of type T. You can supress the exception by setting - /// exception_if_not_one to false. In that case, you will have to check if t==NULL to - /// know if the call succeeded. + std::string resolved_tag = tag; + if (mUseDefaultTags && tag.empty()) { + auto defaultTag = mDefaultTags.find(JTypeInfo::demangle()); + if (defaultTag != mDefaultTags.end()) resolved_tag = defaultTag->second; + } + auto factory = mFactorySet->GetFactory(resolved_tag); + if (factory == nullptr) { + factory = new JFactoryT; + factory->SetTag(tag); + factory->SetLevel(mFactorySet->GetLevel()); + mFactorySet->Add(factory); + } + factory->Insert(item); + factory->SetInsertOrigin( mCallGraph.GetInsertDataOrigin() ); // (see note at top of JCallGraphRecorder.h) + return factory; +} - std::vector v; - auto fac = GetFactory(tag, true); // throw exception if factory not found - JCallGraphEntryMaker cg_entry(mCallGraph, fac); // times execution until this goes out of scope - Get(v, tag); - if(v.size()!=1){ - t = NULL; - if(exception_if_not_one) throw v.size(); +template +inline JFactoryT* JEvent::Insert(const std::vector& items, const std::string& tag) const { + + std::string resolved_tag = tag; + if (mUseDefaultTags && tag.empty()) { + auto defaultTag = mDefaultTags.find(JTypeInfo::demangle()); + if (defaultTag != mDefaultTags.end()) resolved_tag = defaultTag->second; } - t = v[0]; - return fac; + auto factory = mFactorySet->GetFactory(resolved_tag); + if (factory == nullptr) { + factory = new JFactoryT; + factory->SetTag(tag); + factory->SetLevel(mFactorySet->GetLevel()); + mFactorySet->Add(factory); + } + for (T* item : items) { + factory->Insert(item); + } + factory->SetStatus(JFactory::Status::Inserted); // for when items is empty + factory->SetCreationStatus(JFactory::CreationStatus::Inserted); // for when items is empty + factory->SetInsertOrigin( mCallGraph.GetInsertDataOrigin() ); // (see note at top of JCallGraphRecorder.h) + return factory; } + + #if JANA2_HAVE_PODIO inline std::vector JEvent::GetAllCollectionNames() const { @@ -535,9 +437,9 @@ inline const podio::CollectionBase* JEvent::GetCollectionBase(std::string name, JCallGraphEntryMaker cg_entry(mCallGraph, it->second); // times execution until this goes out of scope it->second->Create(this->shared_from_this()); return factory->GetCollection(); - // TODO: Might be cheaper/simpler to obtain factory from mPodioFactories instead of mFactorySet } + template const typename JFactoryPodioT::CollectionT* JEvent::GetCollection(std::string name, bool throw_on_missing) const { JFactoryT* factory = GetFactory(name, throw_on_missing); diff --git a/src/libraries/JANA/JEventFolder.h b/src/libraries/JANA/JEventFolder.h new file mode 100644 index 000000000..d33dbc692 --- /dev/null +++ b/src/libraries/JANA/JEventFolder.h @@ -0,0 +1,164 @@ +// Copyright 2024, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. + +#pragma once + +#include +#include +#include +#include +#include + +class JApplication; +class JEventFolder : public jana::components::JComponent, + public jana::components::JHasRunCallbacks, + public jana::components::JHasInputs, + public jana::components::JHasOutputs { + +private: + int32_t m_last_run_number = -1; + JEventLevel m_child_level; + bool m_call_preprocess_upstream = true; + + +public: + + JEventFolder() = default; + virtual ~JEventFolder() {}; + + virtual void Init() {}; + + virtual void Preprocess(const JEvent& /*parent*/) const {}; + + virtual void Fold(const JEvent& /*child*/, JEvent& /*parent*/, int /*item_nr*/) { + throw JException("Not implemented yet!"); + }; + + virtual void Finish() {}; + + + // Configuration + + void SetParentLevel(JEventLevel level) { m_level = level; } + + void SetChildLevel(JEventLevel level) { m_child_level = level; } + + void SetCallPreprocessUpstream(bool call_upstream) { m_call_preprocess_upstream = call_upstream; } + + JEventLevel GetChildLevel() { return m_child_level; } + + + public: + // Backend + + void DoInit() { + std::lock_guard lock(m_mutex); + if (m_status != Status::Uninitialized) { + throw JException("JEventFolder: Attempting to initialize twice or from an invalid state"); + } + // TODO: Obtain overrides of collection names from param manager + for (auto* parameter : m_parameters) { + parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); + } + for (auto* service : m_services) { + service->Fetch(m_app); + } + CallWithJExceptionWrapper("JEventFolder::Init", [&](){Init();}); + m_status = Status::Initialized; + } + + void DoPreprocess(const JEvent& child) { + { + std::lock_guard lock(m_mutex); + if (m_status != Status::Initialized) { + throw JException("JEventFolder: Component needs to be initialized and not finalized before Fold can be called"); + // TODO: Consider calling Initialize(with_lock=false) like we do elsewhere + } + } + for (auto* input : m_inputs) { + input->PrefetchCollection(child); + } + if (m_callback_style != CallbackStyle::DeclarativeMode) { + CallWithJExceptionWrapper("JEventFolder::Preprocess", [&](){ + Preprocess(child); + }); + } + } + + void DoFold(const JEvent& child, JEvent& parent) { + std::lock_guard lock(m_mutex); + if (m_status != Status::Initialized) { + throw JException("Component needs to be initialized and not finalized before Fold() can be called"); + } + if (!m_call_preprocess_upstream) { + CallWithJExceptionWrapper("JEventFolder::Preprocess", [&](){ + Preprocess(parent); + }); + } + if (m_last_run_number != parent.GetRunNumber()) { + for (auto* resource : m_resources) { + resource->ChangeRun(parent.GetRunNumber(), m_app); + } + if (m_callback_style == CallbackStyle::DeclarativeMode) { + CallWithJExceptionWrapper("JEventFolder::ChangeRun", [&](){ + ChangeRun(parent.GetRunNumber()); + }); + } + else { + CallWithJExceptionWrapper("JEventFolder::ChangeRun", [&](){ + ChangeRun(parent); + }); + } + m_last_run_number = parent.GetRunNumber(); + } + for (auto* input : m_inputs) { + input->GetCollection(parent); + // TODO: This requires that all inputs come from the parent. + // However, eventually we will want to support inputs + // that come from the child. + } + for (auto* output : m_outputs) { + output->Reset(); + } + auto child_number = child.GetEventIndex(); + CallWithJExceptionWrapper("JEventFolder::Fold", [&](){ + Fold(child, parent, child_number); + }); + + for (auto* output : m_outputs) { + output->InsertCollection(parent); + } + } + + void DoFinish() { + std::lock_guard lock(m_mutex); + if (m_status != Status::Finalized) { + CallWithJExceptionWrapper("JEventFolder::Finish", [&](){ + Finish(); + }); + m_status = Status::Finalized; + } + } + + void Summarize(JComponentSummary& summary) const override { + auto* us = new JComponentSummary::Component( + "Folder", GetPrefix(), GetTypeName(), GetLevel(), GetPluginName()); + + for (const auto* input : m_inputs) { + size_t subinput_count = input->names.size(); + for (size_t i=0; iAddInput(new JComponentSummary::Collection("", input->names[i], input->type_name, input->levels[i])); + } + } + for (const auto* output : m_outputs) { + size_t suboutput_count = output->collection_names.size(); + for (size_t i=0; iAddOutput(new JComponentSummary::Collection("", output->collection_names[i], output->type_name, GetLevel())); + } + } + summary.Add(us); + } + +}; + + diff --git a/src/libraries/JANA/JEventProcessor.h b/src/libraries/JANA/JEventProcessor.h index b170e9267..33775a142 100644 --- a/src/libraries/JANA/JEventProcessor.h +++ b/src/libraries/JANA/JEventProcessor.h @@ -8,6 +8,7 @@ #include #include #include +#include class JApplication; @@ -30,39 +31,38 @@ class JEventProcessor : public jana::components::JComponent, uint64_t GetEventCount() const { return m_event_count; }; - bool AreEventsOrdered() const { return m_receive_events_in_order; } - virtual void DoInitialize() { + std::lock_guard lock(m_mutex); for (auto* parameter : m_parameters) { parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); } for (auto* service : m_services) { - service->Init(m_app); + service->Fetch(m_app); } CallWithJExceptionWrapper("JEventProcessor::Init", [&](){ Init(); }); m_status = Status::Initialized; } - virtual void DoMap(const std::shared_ptr& e) { + virtual void DoMap(const JEvent& event) { if (m_callback_style == CallbackStyle::LegacyMode) { throw JException("Called DoMap() on a legacy-mode JEventProcessor"); } for (auto* input : m_inputs) { - input->PrefetchCollection(*e); + input->PrefetchCollection(event); } if (m_callback_style == CallbackStyle::ExpertMode) { - ProcessParallel(*e); + ProcessParallel(event); } else { - ProcessParallel(e->GetRunNumber(), e->GetEventNumber(), e->GetEventIndex()); + ProcessParallel(event.GetRunNumber(), event.GetEventNumber(), event.GetEventIndex()); } } - virtual void DoTap(const std::shared_ptr& e) { + virtual void DoTap(const JEvent& event) { if (m_callback_style == CallbackStyle::LegacyMode) { throw JException("Called DoReduce() on a legacy-mode JEventProcessor"); @@ -73,7 +73,7 @@ class JEventProcessor : public jana::components::JComponent, // any contention. if (m_status == Status::Uninitialized) { - DoInitialize(); + throw JException("JEventProcessor: Attempted to call DoTap() before Initialize()"); } else if (m_status == Status::Finalized) { throw JException("JEventProcessor: Attempted to call DoMap() after Finalize()"); @@ -82,26 +82,23 @@ class JEventProcessor : public jana::components::JComponent, // This collection should have already been computed during DoMap() // We do this before ChangeRun() just in case we will need to pull data out of // a begin-of-run event. - input->GetCollection(*e); + input->GetCollection(event); } - auto run_number = e->GetRunNumber(); + auto run_number = event.GetRunNumber(); if (m_last_run_number != run_number) { - if (m_last_run_number != -1) { - CallWithJExceptionWrapper("JEventProcessor::EndRun", [&](){ EndRun(); }); - } for (auto* resource : m_resources) { - resource->ChangeRun(e->GetRunNumber(), m_app); + resource->ChangeRun(event.GetRunNumber(), m_app); } m_last_run_number = run_number; - CallWithJExceptionWrapper("JEventProcessor::BeginRun", [&](){ BeginRun(e); }); + CallWithJExceptionWrapper("JEventProcessor::ChangeRun", [&](){ ChangeRun(event); }); } if (m_callback_style == CallbackStyle::DeclarativeMode) { CallWithJExceptionWrapper("JEventProcessor::Process", [&](){ - Process(e->GetRunNumber(), e->GetEventNumber(), e->GetEventIndex()); + Process(event.GetRunNumber(), event.GetEventNumber(), event.GetEventIndex()); }); } else if (m_callback_style == CallbackStyle::ExpertMode) { - CallWithJExceptionWrapper("JEventProcessor::Process", [&](){ Process(*e); }); + CallWithJExceptionWrapper("JEventProcessor::Process", [&](){ Process(event); }); } m_event_count += 1; } @@ -109,29 +106,38 @@ class JEventProcessor : public jana::components::JComponent, virtual void DoLegacyProcess(const std::shared_ptr& event) { - // DoLegacyProcess doesn't hold any locks, as it requires the user to hold a lock for it. - // Because of this, + // DoLegacyProcess holds a lock to make sure that {Begin,Change,End}Run() are always called before Process(). + // Note that in LegacyMode, Process() requires the user to manage a _separate_ lock for its critical section. + // This arrangement means that {Begin,Change,End}Run() will definitely be called at least once before `Process`, but there + // may be races when there are multiple run numbers present in the stream. This isn't a problem in practice for now, + // but future work should use ExpertMode or DeclarativeMode for this reason (but also for the usability improvements!) + if (m_callback_style != CallbackStyle::LegacyMode) { throw JException("Called DoLegacyProcess() on a non-legacy-mode JEventProcessor"); } auto run_number = event->GetRunNumber(); - if (m_status == Status::Uninitialized) { - DoInitialize(); - } - else if (m_status == Status::Finalized) { - throw JException("JEventProcessor: Attempted to call DoMap() after Finalize()"); - } - if (m_last_run_number != run_number) { - if (m_last_run_number != -1) { - CallWithJExceptionWrapper("JEventProcessor::EndRun", [&](){ EndRun(); }); + { + // Protect the call to BeginRun(), etc, to prevent some threads from running Process() before BeginRun(). + std::lock_guard lock(m_mutex); + + if (m_status == Status::Uninitialized) { + throw JException("JEventProcessor: Attempted to call DoLegacyProcess() before Initialize()"); } - for (auto* resource : m_resources) { - resource->ChangeRun(event->GetRunNumber(), m_app); + else if (m_status == Status::Finalized) { + throw JException("JEventProcessor: Attempted to call DoLegacyProcess() after Finalize()"); + } + if (m_last_run_number != run_number) { + if (m_last_run_number != -1) { + CallWithJExceptionWrapper("JEventProcessor::EndRun", [&](){ EndRun(); }); + } + for (auto* resource : m_resources) { + resource->ChangeRun(event->GetRunNumber(), m_app); + } + m_last_run_number = run_number; + CallWithJExceptionWrapper("JEventProcessor::BeginRun", [&](){ BeginRun(event); }); } - m_last_run_number = run_number; - CallWithJExceptionWrapper("JEventProcessor::BeginRun", [&](){ BeginRun(event); }); } CallWithJExceptionWrapper("JEventProcessor::Process", [&](){ Process(event); }); m_event_count += 1; @@ -170,33 +176,29 @@ class JEventProcessor : public jana::components::JComponent, // LegacyMode-specific callbacks virtual void Process(const std::shared_ptr& /*event*/) { - throw JException("Not implemented yet!"); } - + // ExpertMode-specific callbacks virtual void ProcessParallel(const JEvent& /*event*/) { } virtual void Process(const JEvent& /*event*/) { - throw JException("Not implemented yet!"); } // DeclarativeMode-specific callbacks virtual void ProcessParallel(int64_t /*run_nr*/, uint64_t /*event_nr*/, uint64_t /*event_idx*/) { - throw JException("Not implemented yet!"); } virtual void Process(int64_t /*run_nr*/, uint64_t /*event_nr*/, uint64_t /*event_idx*/) { - throw JException("Not implemented yet!"); } virtual void Finish() {} - // TODO: Deprecate + [[deprecated]] virtual std::string GetType() const { return m_type_name; } @@ -217,18 +219,10 @@ class JEventProcessor : public jana::components::JComponent, // void SetResourceName(std::string resource_name) { m_resource_name = std::move(resource_name); } - /// SetEventsOrdered allows the user to tell the parallelization engine that it needs to see - /// the event stream ordered by increasing event IDs. (Note that this requires all EventSources - /// emit event IDs which are consecutive.) Ordering by event ID makes for cleaner output, but comes - /// with a performance penalty, so it is best if this is enabled during debugging, and disabled otherwise. - - // void SetEventsOrdered(bool receive_events_in_order) { m_receive_events_in_order = receive_events_in_order; } - private: std::string m_resource_name; std::atomic_ullong m_event_count {0}; - bool m_receive_events_in_order = false; }; diff --git a/src/libraries/JANA/JEventSource.cc b/src/libraries/JANA/JEventSource.cc new file mode 100644 index 000000000..af99d9fc4 --- /dev/null +++ b/src/libraries/JANA/JEventSource.cc @@ -0,0 +1,276 @@ +#include + +void JEventSource::DoInit() { + std::lock_guard lock(m_mutex); + if (m_status != Status::Uninitialized) { + throw JException("Attempted to initialize a JEventSource that is already initialized!"); + } + for (auto* parameter : m_parameters) { + parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); + } + for (auto* service : m_services) { + service->Fetch(m_app); + } + CallWithJExceptionWrapper("JEventSource::Init", [&](){ Init(); }); + m_status = Status::Initialized; + LOG_INFO(GetLogger()) << "Initialized JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; +} + +void JEventSource::DoInitialize() { + DoOpen(); +} + +void JEventSource::DoOpen(bool with_lock) { + if (with_lock) { + std::lock_guard lock(m_mutex); + if (m_status != Status::Initialized) { + throw JException("Attempted to open a JEventSource that hasn't been initialized!"); + } + CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); + if (GetResourceName().empty()) { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + m_status = Status::Opened; + } + else { + if (m_status != Status::Initialized) { + throw JException("Attempted to open a JEventSource that hasn't been initialized!"); + } + CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); + if (GetResourceName().empty()) { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + m_status = Status::Opened; + } +} + +void JEventSource::DoClose(bool with_lock) { + if (with_lock) { + std::lock_guard lock(m_mutex); + + if (m_status != JEventSource::Status::Opened) return; + + CallWithJExceptionWrapper("JEventSource::Close", [&](){ Close();}); + if (GetResourceName().empty()) { + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + m_status = Status::Closed; + } + else { + if (m_status != JEventSource::Status::Opened) return; + + CallWithJExceptionWrapper("JEventSource::Close", [&](){ Close();}); + if (GetResourceName().empty()) { + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; + } + m_status = Status::Closed; + } +} + +JEventSource::Result JEventSource::DoNext(std::shared_ptr event) { + + std::lock_guard lock(m_mutex); // In general, DoNext must be synchronized. + + if (m_status == Status::Uninitialized) { + throw JException("JEventSource has not been initialized!"); + } + + if (m_callback_style == CallbackStyle::LegacyMode) { + return DoNextCompatibility(event); + } + + auto first_evt_nr = m_nskip; + auto last_evt_nr = m_nevents + m_nskip; + + if (m_status == Status::Initialized) { + DoOpen(false); + } + if (m_status == Status::Opened) { + if (m_nevents != 0 && (m_events_emitted == last_evt_nr)) { + // We exit early (and recycle) because we hit our jana:nevents limit + DoClose(false); + return Result::FailureFinished; + } + // If we reach this point, we will need to actually read an event + + // We configure the event + event->SetEventNumber(m_events_emitted); // Default event number to event count + event->SetJEventSource(this); + event->SetSequential(false); + event->GetJCallGraphRecorder()->Reset(); + + // Now we call the new-style interface + auto previous_origin = event->GetJCallGraphRecorder()->SetInsertDataOrigin( JCallGraphRecorder::ORIGIN_FROM_SOURCE); // (see note at top of JCallGraphRecorder.h) + JEventSource::Result result; + CallWithJExceptionWrapper("JEventSource::Emit", [&](){ + result = Emit(*event); + }); + event->GetJCallGraphRecorder()->SetInsertDataOrigin( previous_origin ); + + if (result == Result::Success) { + m_events_emitted += 1; + // We end up here if we read an entry in our file or retrieved a message from our socket, + // and believe we could obtain another one immediately if we wanted to + for (auto* output : m_outputs) { + output->InsertCollection(*event); + } + if (m_events_emitted <= first_evt_nr) { + // We immediately throw away this whole event because of nskip + // (although really we should be handling this with Seek()) + return Result::FailureTryAgain; + } + return Result::Success; + } + else if (result == Result::FailureFinished) { + // We end up here if we tried to read an entry in a file, but found EOF + // or if we received a message from a socket that contained no data and indicated no more data will be coming + DoClose(false); + return Result::FailureFinished; + } + else if (result == Result::FailureTryAgain) { + // We end up here if we tried to read an entry in a file but it is on a tape drive and isn't ready yet + // or if we polled the socket, found no new messages, but still expect messages later + return Result::FailureTryAgain; + } + else { + throw JException("Invalid JEventSource::Result value!"); + } + } + else { // status == Closed + return Result::FailureFinished; + } +} + +JEventSource::Result JEventSource::DoNextCompatibility(std::shared_ptr event) { + + auto first_evt_nr = m_nskip; + auto last_evt_nr = m_nevents + m_nskip; + + try { + if (m_status == Status::Initialized) { + DoOpen(false); + } + if (m_status == Status::Opened) { + if (m_events_emitted < first_evt_nr) { + // Skip these events due to nskip + event->SetEventNumber(m_events_emitted); // Default event number to event count + auto previous_origin = event->GetJCallGraphRecorder()->SetInsertDataOrigin( JCallGraphRecorder::ORIGIN_FROM_SOURCE); // (see note at top of JCallGraphRecorder.h) + GetEvent(event); + event->GetJCallGraphRecorder()->SetInsertDataOrigin( previous_origin ); + m_events_emitted += 1; + return Result::FailureTryAgain; // Reject this event and recycle it + } else if (m_nevents != 0 && (m_events_emitted == last_evt_nr)) { + // Declare ourselves finished due to nevents + DoClose(false); // Close out the event source as soon as it declares itself finished + return Result::FailureFinished; + } else { + // Actually emit an event. + // GetEvent() expects the following things from its incoming JEvent + event->SetEventNumber(m_events_emitted); + event->SetJApplication(m_app); + event->SetJEventSource(this); + event->SetSequential(false); + event->GetJCallGraphRecorder()->Reset(); + auto previous_origin = event->GetJCallGraphRecorder()->SetInsertDataOrigin( JCallGraphRecorder::ORIGIN_FROM_SOURCE); // (see note at top of JCallGraphRecorder.h) + GetEvent(event); + for (auto* output : m_outputs) { + output->InsertCollection(*event); + } + event->GetJCallGraphRecorder()->SetInsertDataOrigin( previous_origin ); + m_events_emitted += 1; + return Result::Success; // Don't reject this event! + } + } else if (m_status == Status::Closed) { + return Result::FailureFinished; + } else { + throw JException("Invalid m_status"); + } + } + catch (RETURN_STATUS rs) { + + if (rs == RETURN_STATUS::kNO_MORE_EVENTS) { + DoClose(false); + return Result::FailureFinished; + } + else if (rs == RETURN_STATUS::kTRY_AGAIN || rs == RETURN_STATUS::kBUSY) { + return Result::FailureTryAgain; + } + else if (rs == RETURN_STATUS::kERROR || rs == RETURN_STATUS::kUNKNOWN) { + JException ex ("JEventSource threw RETURN_STATUS::kERROR or kUNKNOWN"); + ex.plugin_name = m_plugin_name; + ex.type_name = m_type_name; + ex.function_name = "JEventSource::GetEvent"; + ex.instance_name = m_resource_name; + throw ex; + } + else { + return Result::Success; + } + } + catch (JException& ex) { + if (ex.function_name.empty()) ex.function_name = "JEventSource::GetEvent"; + if (ex.type_name.empty()) ex.type_name = m_type_name; + if (ex.instance_name.empty()) ex.instance_name = m_prefix; + if (ex.plugin_name.empty()) ex.plugin_name = m_plugin_name; + throw ex; + } + catch (std::exception& e){ + auto ex = JException(e.what()); + ex.exception_type = JTypeInfo::demangle_current_exception_type(); + ex.nested_exception = std::current_exception(); + ex.function_name = "JEventSource::GetEvent"; + ex.type_name = m_type_name; + ex.instance_name = m_prefix; + ex.plugin_name = m_plugin_name; + throw ex; + } + catch (...) { + auto ex = JException("Unknown exception"); + ex.exception_type = JTypeInfo::demangle_current_exception_type(); + ex.nested_exception = std::current_exception(); + ex.function_name = "JEventSource::GetEvent"; + ex.type_name = m_type_name; + ex.instance_name = m_prefix; + ex.plugin_name = m_plugin_name; + throw ex; + } +} + + +void JEventSource::DoFinishEvent(JEvent& event) { + + m_events_finished.fetch_add(1); + if (m_enable_finish_event) { + std::lock_guard lock(m_mutex); + CallWithJExceptionWrapper("JEventSource::FinishEvent", [&](){ + FinishEvent(event); + }); + } +} + +void JEventSource::Summarize(JComponentSummary& summary) const { + + auto* result = new JComponentSummary::Component( + "Source", GetPrefix(), GetTypeName(), GetLevel(), GetPluginName()); + + for (const auto* output : m_outputs) { + size_t suboutput_count = output->collection_names.size(); + for (size_t i=0; iAddOutput(new JComponentSummary::Collection("", output->collection_names[i], output->type_name, GetLevel())); + } + } + + summary.Add(result); +} diff --git a/src/libraries/JANA/JEventSource.h b/src/libraries/JANA/JEventSource.h index c0fcc7764..c3cc96048 100644 --- a/src/libraries/JANA/JEventSource.h +++ b/src/libraries/JANA/JEventSource.h @@ -16,26 +16,33 @@ class JApplication; class JFactory; -class JEventSource : public jana::components::JComponent, +class JEventSource : public jana::components::JComponent, public jana::components::JHasOutputs { public: - /// Result describes what happened the last time a GetEvent() was attempted. /// If Emit() or GetEvent() reaches an error state, it should throw a JException instead. enum class Result { Success, FailureTryAgain, FailureFinished }; - // TODO: Deprecate me! /// The user is supposed to _throw_ RETURN_STATUS::kNO_MORE_EVENTS or kBUSY from GetEvent() enum class RETURN_STATUS { kSUCCESS, kNO_MORE_EVENTS, kBUSY, kTRY_AGAIN, kERROR, kUNKNOWN }; - // Constructor - // TODO: Deprecate me! +private: + std::string m_resource_name; + std::atomic_ullong m_events_emitted {0}; + std::atomic_ullong m_events_finished {0}; + uint64_t m_nskip = 0; + uint64_t m_nevents = 0; + bool m_enable_finish_event = false; + bool m_enable_get_objects = false; + bool m_enable_preprocess = false; + + +public: + [[deprecated]] explicit JEventSource(std::string resource_name, JApplication* app = nullptr) - : m_resource_name(std::move(resource_name)) - , m_event_count{0} - { + : m_resource_name(std::move(resource_name)) { m_app = app; } @@ -48,7 +55,7 @@ class JEventSource : public jana::components::JComponent, virtual void Init() {} - // To be implemented by the user + /// `Open` is called by JANA when it is ready to accept events from this event source. The implementor should open /// file pointers or sockets here, instead of in the constructor. This is because the implementor won't know how many /// or which event sources the user will decide to activate within one job. Thus the implementor can avoid problems @@ -68,6 +75,22 @@ class JEventSource : public jana::components::JComponent, virtual Result Emit(JEvent&) { return Result::Success; }; + /// For work that should be done in parallel on a JEvent, but is tightly coupled to the JEventSource for some reason. + /// Called after Emit() by JEventMapArrow + virtual void Preprocess(const JEvent&) const {}; + + + /// `FinishEvent` is used to notify the `JEventSource` that an event has been completely processed. This is the final + /// chance to interact with the `JEvent` before it is either cleared and recycled, or deleted. Although it is + /// possible to use this for freeing JObjects stored in the JEvent , this is strongly discouraged in favor of putting + /// that logic on the destructor, RAII-style. Instead, this callback should be used for updating and freeing state + /// owned by the JEventSource, e.g. raw data which is keyed off of run number and therefore shared among multiple + /// JEvents. `FinishEvent` is also well-suited for use with `EventGroup`s, e.g. to notify someone that a batch of + /// events has finished, or to implement "barrier events". + + virtual void FinishEvent(JEvent&) {}; + + /// `Close` is called by JANA when it is finished accepting events from this event source. Here is where you should /// cleanly close files, sockets, etc. Although GetEvent() knows when (for instance) there are no more events in a /// file, the logic for closing needs to live here because there are other ways a computation may end besides @@ -96,343 +119,79 @@ class JEventSource : public jana::components::JComponent, virtual void GetEvent(std::shared_ptr) {}; - virtual void Preprocess(const JEvent&) {}; - - - /// `FinishEvent` is used to notify the `JEventSource` that an event has been completely processed. This is the final - /// chance to interact with the `JEvent` before it is either cleared and recycled, or deleted. Although it is - /// possible to use this for freeing JObjects stored in the JEvent , this is strongly discouraged in favor of putting - /// that logic on the destructor, RAII-style. Instead, this callback should be used for updating and freeing state - /// owned by the JEventSource, e.g. raw data which is keyed off of run number and therefore shared among multiple - /// JEvents. `FinishEvent` is also well-suited for use with `EventGroup`s, e.g. to notify someone that a batch of - /// events has finished, or to implement "barrier events". - virtual void FinishEvent(JEvent&) {}; - /// `GetObjects` was historically used for lazily unpacking data from a JEvent and putting it into a "dummy" JFactory. /// This mechanism has been replaced by `JEvent::Insert`. All lazy evaluation should happen in a (non-dummy) /// JFactory, whereas eager evaluation should happen in `JEventSource::GetEvent` via `JEvent::Insert`. + virtual bool GetObjects(const std::shared_ptr&, JFactory*) { return false; } - virtual void DoInit() { - if (m_status == Status::Uninitialized) { - CallWithJExceptionWrapper("JEventSource::Init", [&](){ Init();}); - m_status = Status::Initialized; - LOG_INFO(GetLogger()) << "Initialized JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; - } - else { - throw JException("Attempted to initialize a JEventSource that is not uninitialized!"); - } - } - - [[deprecated("Replaced by JEventSource::DoOpen()")]] - virtual void DoInitialize() { - DoOpen(); - } - - virtual void DoOpen(bool with_lock=true) { - if (with_lock) { - std::lock_guard lock(m_mutex); - if (m_status != Status::Initialized) { - throw JException("Attempted to open a JEventSource that hasn't been initialized!"); - } - CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); - if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "'" << LOG_END; - } - else { - LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; - } - m_status = Status::Opened; - } - else { - if (m_status != Status::Initialized) { - throw JException("Attempted to open a JEventSource that hasn't been initialized!"); - } - CallWithJExceptionWrapper("JEventSource::Open", [&](){ Open();}); - if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "'" << LOG_END; - } - else { - LOG_INFO(GetLogger()) << "Opened JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; - } - m_status = Status::Opened; - } - } - - virtual void DoClose(bool with_lock=true) { - if (with_lock) { - std::lock_guard lock(m_mutex); - - if (m_status != JEventSource::Status::Opened) return; - - CallWithJExceptionWrapper("JEventSource::Close", [&](){ Close();}); - if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "'" << LOG_END; - } - else { - LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; - } - m_status = Status::Closed; - } - else { - if (m_status != JEventSource::Status::Opened) return; - - CallWithJExceptionWrapper("JEventSource::Close", [&](){ Close();}); - if (GetResourceName().empty()) { - LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "'" << LOG_END; - } - else { - LOG_INFO(GetLogger()) << "Closed JEventSource '" << GetTypeName() << "' ('" << GetResourceName() << "')" << LOG_END; - } - m_status = Status::Closed; - } - } + // Getters - Result DoNext(std::shared_ptr event) { - - std::lock_guard lock(m_mutex); // In general, DoNext must be synchronized. - - if (m_status == Status::Uninitialized) { - throw JException("JEventSource has not been initialized!"); - } - - if (m_callback_style == CallbackStyle::LegacyMode) { - return DoNextCompatibility(event); - } - - auto first_evt_nr = m_nskip; - auto last_evt_nr = m_nevents + m_nskip; - - if (m_status == Status::Initialized) { - DoOpen(false); - } - if (m_status == Status::Opened) { - if (m_nevents != 0 && (m_event_count == last_evt_nr)) { - // We exit early (and recycle) because we hit our jana:nevents limit - DoClose(false); - return Result::FailureFinished; - } - // If we reach this point, we will need to actually read an event - - // We configure the event - event->SetEventNumber(m_event_count); // Default event number to event count - event->SetJEventSource(this); - event->SetSequential(false); - event->GetJCallGraphRecorder()->Reset(); - - // Now we call the new-style interface - auto previous_origin = event->GetJCallGraphRecorder()->SetInsertDataOrigin( JCallGraphRecorder::ORIGIN_FROM_SOURCE); // (see note at top of JCallGraphRecorder.h) - JEventSource::Result result; - CallWithJExceptionWrapper("JEventSource::Emit", [&](){ - result = Emit(*event); - }); - event->GetJCallGraphRecorder()->SetInsertDataOrigin( previous_origin ); - - if (result == Result::Success) { - m_event_count += 1; - // We end up here if we read an entry in our file or retrieved a message from our socket, - // and believe we could obtain another one immediately if we wanted to - for (auto* output : m_outputs) { - output->InsertCollection(*event); - } - if (m_event_count <= first_evt_nr) { - // We immediately throw away this whole event because of nskip - // (although really we should be handling this with Seek()) - return Result::FailureTryAgain; - } - return Result::Success; - } - else if (result == Result::FailureFinished) { - // We end up here if we tried to read an entry in a file, but found EOF - // or if we received a message from a socket that contained no data and indicated no more data will be coming - DoClose(false); - return Result::FailureFinished; - } - else if (result == Result::FailureTryAgain) { - // We end up here if we tried to read an entry in a file but it is on a tape drive and isn't ready yet - // or if we polled the socket, found no new messages, but still expect messages later - return Result::FailureTryAgain; - } - else { - throw JException("Invalid JEventSource::Result value!"); - } - } - else { // status == Closed - return Result::FailureFinished; - } - } - - Result DoNextCompatibility(std::shared_ptr event) { - - auto first_evt_nr = m_nskip; - auto last_evt_nr = m_nevents + m_nskip; - - try { - if (m_status == Status::Initialized) { - DoOpen(false); - } - if (m_status == Status::Opened) { - if (m_event_count < first_evt_nr) { - // Skip these events due to nskip - event->SetEventNumber(m_event_count); // Default event number to event count - auto previous_origin = event->GetJCallGraphRecorder()->SetInsertDataOrigin( JCallGraphRecorder::ORIGIN_FROM_SOURCE); // (see note at top of JCallGraphRecorder.h) - GetEvent(event); - event->GetJCallGraphRecorder()->SetInsertDataOrigin( previous_origin ); - m_event_count += 1; - return Result::FailureTryAgain; // Reject this event and recycle it - } else if (m_nevents != 0 && (m_event_count == last_evt_nr)) { - // Declare ourselves finished due to nevents - DoClose(false); // Close out the event source as soon as it declares itself finished - return Result::FailureFinished; - } else { - // Actually emit an event. - // GetEvent() expects the following things from its incoming JEvent - event->SetEventNumber(m_event_count); - event->SetJApplication(m_app); - event->SetJEventSource(this); - event->SetSequential(false); - event->GetJCallGraphRecorder()->Reset(); - auto previous_origin = event->GetJCallGraphRecorder()->SetInsertDataOrigin( JCallGraphRecorder::ORIGIN_FROM_SOURCE); // (see note at top of JCallGraphRecorder.h) - GetEvent(event); - for (auto* output : m_outputs) { - output->InsertCollection(*event); - } - event->GetJCallGraphRecorder()->SetInsertDataOrigin( previous_origin ); - m_event_count += 1; - return Result::Success; // Don't reject this event! - } - } else if (m_status == Status::Closed) { - return Result::FailureFinished; - } else { - throw JException("Invalid m_status"); - } - } - catch (RETURN_STATUS rs) { - - if (rs == RETURN_STATUS::kNO_MORE_EVENTS) { - DoClose(false); - return Result::FailureFinished; - } - else if (rs == RETURN_STATUS::kTRY_AGAIN || rs == RETURN_STATUS::kBUSY) { - return Result::FailureTryAgain; - } - else if (rs == RETURN_STATUS::kERROR || rs == RETURN_STATUS::kUNKNOWN) { - JException ex ("JEventSource threw RETURN_STATUS::kERROR or kUNKNOWN"); - ex.plugin_name = m_plugin_name; - ex.type_name = m_type_name; - ex.function_name = "JEventSource::GetEvent"; - ex.instance_name = m_resource_name; - throw ex; - } - else { - return Result::Success; - } - } - catch (JException& ex) { - if (ex.function_name.empty()) ex.function_name = "JEventSource::GetEvent"; - if (ex.type_name.empty()) ex.type_name = m_type_name; - if (ex.instance_name.empty()) ex.instance_name = m_prefix; - if (ex.plugin_name.empty()) ex.plugin_name = m_plugin_name; - throw ex; - } - catch (std::exception& e){ - auto ex = JException(e.what()); - ex.exception_type = JTypeInfo::demangle_current_exception_type(); - ex.nested_exception = std::current_exception(); - ex.function_name = "JEventSource::GetEvent"; - ex.type_name = m_type_name; - ex.instance_name = m_prefix; - ex.plugin_name = m_plugin_name; - throw ex; - } - catch (...) { - auto ex = JException("Unknown exception"); - ex.exception_type = JTypeInfo::demangle_current_exception_type(); - ex.nested_exception = std::current_exception(); - ex.function_name = "JEventSource::GetEvent"; - ex.type_name = m_type_name; - ex.instance_name = m_prefix; - ex.plugin_name = m_plugin_name; - throw ex; - } - } - - /// Calls the optional-and-discouraged user-provided FinishEvent virtual method, enforcing - /// 1. Thread safety - /// 2. The m_enable_free_event flag - - void DoFinish(JEvent& event) { - if (m_enable_free_event) { - std::lock_guard lock(m_mutex); - CallWithJExceptionWrapper("JEventSource::FinishEvent", [&](){ - FinishEvent(event); - }); - } - } - - void Summarize(JComponentSummary& summary) const override { - - auto* result = new JComponentSummary::Component( - "Source", GetPrefix(), GetTypeName(), GetLevel(), GetPluginName()); - - for (const auto* output : m_outputs) { - size_t suboutput_count = output->collection_names.size(); - for (size_t i=0; iAddOutput(new JComponentSummary::Collection("", output->collection_names[i], output->type_name, GetLevel())); - } - } - - summary.Add(result); - } - - // Getters and setters - - void SetResourceName(std::string resource_name) { m_resource_name = resource_name; } - std::string GetResourceName() const { return m_resource_name; } - uint64_t GetEventCount() const { return m_event_count; }; + [[deprecated]] + uint64_t GetEventCount() const { return m_events_emitted; }; + uint64_t GetEmittedEventCount() const { return m_events_emitted; }; + uint64_t GetFinishedEventCount() const { return m_events_finished; }; - // TODO: Deprecate me + [[deprecated]] virtual std::string GetType() const { return m_type_name; } - // TODO: Deprecate me + [[deprecated]] std::string GetName() const { return m_resource_name; } - // TODO: Deprecate me + bool IsGetObjectsEnabled() const { return m_enable_get_objects; } + bool IsFinishEventEnabled() const { return m_enable_finish_event; } + bool IsPreprocessEnabled() const { return m_enable_preprocess; } + + uint64_t GetNSkip() { return m_nskip; } + uint64_t GetNEvents() { return m_nevents; } + virtual std::string GetVDescription() const { return ""; } ///< Optional for getting description via source rather than JEventSourceGenerator - uint64_t GetNSkip() { return m_nskip; } - uint64_t GetNEvents() { return m_nevents; } + // Setters + + void SetResourceName(std::string resource_name) { m_resource_name = resource_name; } - // Meant to be called by user /// EnableFinishEvent() is intended to be called by the user in the constructor in order to /// tell JANA to call the provided FinishEvent method after all JEventProcessors /// have finished with a given event. This should only be enabled when absolutely necessary /// (e.g. for backwards compatibility) because it introduces contention for the JEventSource mutex, /// which will hurt performance. Conceptually, FinishEvent isn't great, and so should be avoided when possible. - void EnableFinishEvent() { m_enable_free_event = true; } + void EnableFinishEvent(bool enable=true) { m_enable_finish_event = enable; } + void EnableGetObjects(bool enable=true) { m_enable_get_objects = enable; } + void EnablePreprocess(bool enable=true) { m_enable_preprocess = enable; } - // Meant to be called by JANA void SetNEvents(uint64_t nevents) { m_nevents = nevents; }; - - // Meant to be called by JANA void SetNSkip(uint64_t nskip) { m_nskip = nskip; }; -private: - std::string m_resource_name; - std::atomic_ullong m_event_count {0}; - uint64_t m_nskip = 0; - uint64_t m_nevents = 0; - bool m_enable_free_event = false; + // Internal + + [[deprecated("Replaced by JEventSource::DoOpen()")]] + void DoInitialize(); + + virtual void DoInit(); + + void DoOpen(bool with_lock=true); + + void DoClose(bool with_lock=true); + + Result DoNext(std::shared_ptr event); + + Result DoNextCompatibility(std::shared_ptr event); + + void DoFinishEvent(JEvent& event); + + void Summarize(JComponentSummary& summary) const override; + }; diff --git a/src/libraries/JANA/JEventUnfolder.h b/src/libraries/JANA/JEventUnfolder.h index f6e8834eb..bd67a69f6 100644 --- a/src/libraries/JANA/JEventUnfolder.h +++ b/src/libraries/JANA/JEventUnfolder.h @@ -61,23 +61,18 @@ class JEventUnfolder : public jana::components::JComponent, void DoInit() { std::lock_guard lock(m_mutex); + if (m_status != Status::Uninitialized) { + throw JException("JEventUnfolder: Attempting to initialize twice or from an invalid state"); + } // TODO: Obtain overrides of collection names from param manager - for (auto* parameter : m_parameters) { parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); } for (auto* service : m_services) { - service->Init(m_app); - } - if (m_status == Status::Uninitialized) { - CallWithJExceptionWrapper("JEventUnfolder::Init", [&](){ - Init(); - }); - m_status = Status::Initialized; - } - else { - throw JException("JEventUnfolder: Attempting to initialize twice or from an invalid state"); + service->Fetch(m_app); } + CallWithJExceptionWrapper("JEventUnfolder::Init", [&](){Init();}); + m_status = Status::Initialized; } void DoPreprocess(const JEvent& parent) { diff --git a/src/libraries/JANA/JException.h b/src/libraries/JANA/JException.h index 745dbc3d1..92d81663a 100644 --- a/src/libraries/JANA/JException.h +++ b/src/libraries/JANA/JException.h @@ -5,8 +5,8 @@ #pragma once #include +#include // This is only here in order to not break halld_recon #include -#include /// JException is a data object which attaches JANA-specific context information to a generic exception. /// As it unwinds the call stack, different exception handlers may add or change information as they see fit. @@ -17,9 +17,9 @@ struct JException : public std::exception { /// Basic constructor explicit JException(std::string message = "Unknown exception") : message(std::move(message)) { - std::ostringstream ss; - make_backtrace(ss); - stacktrace = ss.str(); + JBacktrace backtrace; + backtrace.Capture(2); + stacktrace = backtrace.ToString(); } virtual ~JException() = default; @@ -89,6 +89,10 @@ JException::JException(std::string format_str, Args... args) { char cmess[1024]; snprintf(cmess, 1024, format_str.c_str(), args...); message = cmess; + + JBacktrace backtrace; + backtrace.Capture(2); + stacktrace = backtrace.ToString(); } diff --git a/src/libraries/JANA/JFactory.cc b/src/libraries/JANA/JFactory.cc index 998c35b31..aa229a2c4 100644 --- a/src/libraries/JANA/JFactory.cc +++ b/src/libraries/JANA/JFactory.cc @@ -4,32 +4,85 @@ #include #include +#include #include void JFactory::Create(const std::shared_ptr& event) { + if (m_app == nullptr && event->GetJApplication() != nullptr) { + // These are usually set by JFactoryGeneratorT, but some user code has custom JFactoryGenerators which don't! + // The design of JFactoryGenerator doesn't give us a better place to inject things + m_app = event->GetJApplication(); + m_logger = m_app->GetJParameterManager()->GetLogger(GetLoggerName()); + } + if (mStatus == Status::Uninitialized) { - CallWithJExceptionWrapper("JFactory::Init", [&](){ Init(); }); - mStatus = Status::Unprocessed; + DoInit(); + } + + // How do we obtain our data? The priority is as follows: + // 1. JFactory::Process() if REGENERATE flag is set + // 2. JEvent::Insert() + // 3. JEventSource::GetObjects() if source has GetObjects() enabled + // 4. JFactory::Process() + + // --------------------------------------------------------------------- + // 1. JFactory::Process() if REGENERATE flag is set + // --------------------------------------------------------------------- + + if (TestFactoryFlag(REGENERATE)) { + if (mStatus == Status::Inserted) { + // Status::Inserted indicates that the data came from either src->GetObjects() or evt->Insert() + ClearData(); + // ClearData() resets mStatus to Unprocessed so that the data will be regenerated exactly once. + } + // After this point, control flow falls through to "4. JFactory::Process" } + else { - if (TestFactoryFlag(REGENERATE) && (mStatus == Status::Inserted)) { - ClearData(); - // ClearData will reset mStatus to Status::Unprocessed + // --------------------------------------------------------------------- + // 2. JEvent::Insert() + // --------------------------------------------------------------------- + + if (mStatus == Status::Inserted) { + // This may include data cached from eventsource->GetObjects(). + // Either way, short-circuit here, because the data is present. + return; + } + + // --------------------------------------------------------------------- + // 3. JEventSource::GetObjects() if source has GetObjects() enabled + // --------------------------------------------------------------------- + + auto src = event->GetJEventSource(); + if (src != nullptr && src->IsGetObjectsEnabled()) { + bool found_data = false; + + CallWithJExceptionWrapper("JEventSource::GetObjects", [&](){ + found_data = src->GetObjects(event, this); }); + + if (found_data) { + mStatus = Status::Inserted; + mCreationStatus = CreationStatus::InsertedViaGetObjects; + return; + } + } + // If neither "2. JEvent::Insert()" nor "3. JEventSource::GetObjects()" succeeded, fall through to "4. JFactory::Process()" } + // --------------------------------------------------------------------- + // 4. JFactory::Process() + // --------------------------------------------------------------------- + + // If the data was Processed (instead of Inserted), it will be in cache, and we can just exit. + // Otherwise we call Process() to create the data in the first place. if (mStatus == Status::Unprocessed) { auto run_number = event->GetRunNumber(); - if (mPreviousRunNumber == -1) { - // This is the very first run - CallWithJExceptionWrapper("JFactory::ChangeRun", [&](){ ChangeRun(event); }); - CallWithJExceptionWrapper("JFactory::BeginRun", [&](){ BeginRun(event); }); - mPreviousRunNumber = run_number; - } - else if (mPreviousRunNumber != run_number) { - // This is a later run, and it has changed - CallWithJExceptionWrapper("JFactory::EndRun", [&](){ EndRun(); }); + if (mPreviousRunNumber != run_number) { + if (mPreviousRunNumber != -1) { + CallWithJExceptionWrapper("JFactory::EndRun", [&](){ EndRun(); }); + } CallWithJExceptionWrapper("JFactory::ChangeRun", [&](){ ChangeRun(event); }); CallWithJExceptionWrapper("JFactory::BeginRun", [&](){ BeginRun(event); }); mPreviousRunNumber = run_number; @@ -41,9 +94,26 @@ void JFactory::Create(const std::shared_ptr& event) { } void JFactory::DoInit() { - if (mStatus == Status::Uninitialized) { - CallWithJExceptionWrapper("JFactory::Init", [&](){ Init(); }); - mStatus = Status::Unprocessed; + if (mStatus != Status::Uninitialized) { + return; + } + for (auto* parameter : m_parameters) { + parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); + } + for (auto* service : m_services) { + service->Fetch(m_app); + } + CallWithJExceptionWrapper("JFactory::Init", [&](){ Init(); }); + mStatus = Status::Unprocessed; +} + +void JFactory::DoFinish() { + if (mStatus == Status::Unprocessed || mStatus == Status::Processed) { + if (mPreviousRunNumber != -1) { + CallWithJExceptionWrapper("JFactory::EndRun", [&](){ EndRun(); }); + } + CallWithJExceptionWrapper("JFactory::Finish", [&](){ Finish(); }); + mStatus = Status::Finished; } } diff --git a/src/libraries/JANA/JFactory.h b/src/libraries/JANA/JFactory.h index 823c324e7..da3066fea 100644 --- a/src/libraries/JANA/JFactory.h +++ b/src/libraries/JANA/JFactory.h @@ -25,7 +25,7 @@ class JApplication; class JFactory : public jana::components::JComponent { public: - enum class Status {Uninitialized, Unprocessed, Processed, Inserted}; + enum class Status {Uninitialized, Unprocessed, Processed, Inserted, Finished}; enum class CreationStatus { NotCreatedYet, Created, Inserted, InsertedViaGetObjects, NeverCreated }; enum JFactory_Flags_t { @@ -171,6 +171,7 @@ class JFactory : public jana::components::JComponent { /// type of object contained. In order to access these objects when all you have is a JFactory*, use JFactory::GetAs(). virtual void Create(const std::shared_ptr& event); void DoInit(); + void DoFinish(); void Summarize(JComponentSummary& summary) const override; diff --git a/src/libraries/JANA/JFactoryGenerator.h b/src/libraries/JANA/JFactoryGenerator.h index 8ac3f0ab6..e774801ee 100644 --- a/src/libraries/JANA/JFactoryGenerator.h +++ b/src/libraries/JANA/JFactoryGenerator.h @@ -60,7 +60,7 @@ class JFactoryGeneratorT : public JFactoryGenerator { factory->SetTypeName(JTypeInfo::demangle()); factory->SetPluginName(GetPluginName()); factory->SetApplication(GetApplication()); - factory->SetLogger(GetApplication()->template GetService()->get_logger(factory->GetPrefix())); + factory->SetLogger(GetApplication()->GetJParameterManager()->GetLogger(factory->GetPrefix())); factory_set->Add(factory); } }; diff --git a/src/libraries/JANA/JFactorySet.cc b/src/libraries/JANA/JFactorySet.cc index 3b29258dc..05d83a4df 100644 --- a/src/libraries/JANA/JFactorySet.cc +++ b/src/libraries/JANA/JFactorySet.cc @@ -165,12 +165,27 @@ void JFactorySet::Print() const } } -/// Release() loops over all contained factories, clearing their data -void JFactorySet::Release() { +//--------------------------------- +// Clear +//--------------------------------- +void JFactorySet::Clear() { for (const auto& sFactoryPair : mFactories) { auto sFactory = sFactoryPair.second; sFactory->ClearData(); + // This automatically clears multifactories because their data is stored in helper factories! + } +} + +//--------------------------------- +// Finish +//--------------------------------- +void JFactorySet::Finish() { + for (auto& p : mFactories) { + p.second->DoFinish(); + } + for (auto& multifac : mMultifactories) { + multifac->DoFinish(); } } diff --git a/src/libraries/JANA/JFactorySet.h b/src/libraries/JANA/JFactorySet.h index 25ebc50c0..706883169 100644 --- a/src/libraries/JANA/JFactorySet.h +++ b/src/libraries/JANA/JFactorySet.h @@ -26,8 +26,9 @@ class JFactorySet { bool Add(JFactory* aFactory); bool Add(JMultifactory* multifactory); - void Print(void) const; - void Release(void); + void Print() const; + void Clear(); + void Finish(); JFactory* GetFactory(const std::string& object_name, const std::string& tag="") const; template JFactoryT* GetFactory(const std::string& tag = "") const; diff --git a/src/libraries/JANA/JFactoryT.h b/src/libraries/JANA/JFactoryT.h index 46fbccb33..2e6eddf86 100644 --- a/src/libraries/JANA/JFactoryT.h +++ b/src/libraries/JANA/JFactoryT.h @@ -18,12 +18,6 @@ #endif -/// Class template for metadata. This constrains JFactoryT to use the same (user-defined) -/// metadata structure, JMetadata for that T. This is essential for retrieving metadata from -/// JFactoryT's without breaking the Liskov substitution property. -template -struct JMetadata {}; - template class JFactoryT : public JFactory { public: @@ -31,24 +25,6 @@ class JFactoryT : public JFactory { using IteratorType = typename std::vector::const_iterator; using PairType = std::pair; - /// JFactoryT constructor requires a name and a tag. - /// Name should always be JTypeInfo::demangle(), tag is usually "". - JFactoryT(const std::string& aName, const std::string& aTag) __attribute__ ((deprecated)) : JFactory(aName, aTag) { - EnableGetAs(); - EnableGetAs( std::is_convertible() ); // Automatically add JObject if this can be converted to it -#if JANA2_HAVE_ROOT - EnableGetAs( std::is_convertible() ); // Automatically add TObject if this can be converted to it -#endif - } - - JFactoryT(const std::string& aName) __attribute__ ((deprecated)) : JFactory(aName, "") { - EnableGetAs(); - EnableGetAs( std::is_convertible() ); // Automatically add JObject if this can be converted to it -#if JANA2_HAVE_ROOT - EnableGetAs( std::is_convertible() ); // Automatically add TObject if this can be converted to it -#endif - } - JFactoryT() : JFactory(JTypeInfo::demangle(), ""){ EnableGetAs(); EnableGetAs( std::is_convertible() ); // Automatically add JObject if this can be converted to it @@ -83,6 +59,10 @@ class JFactoryT : public JFactory { return std::make_pair(mData.cbegin(), mData.cend()); } + // Retrieve a const reference to the data directly (no copying!) + const std::vector& GetData() { + return mData; + } /// Please use the typed setters instead whenever possible // TODO: Deprecate this! @@ -167,20 +147,9 @@ class JFactoryT : public JFactory { mCreationStatus = CreationStatus::NotCreatedYet; } - /// Set the JFactory's metadata. This is meant to be called by user during their JFactoryT::Process - /// Metadata will *not* be cleared on ClearData(), but will be destroyed when the JFactoryT is. - [[deprecated("Use JMultifactory instead")]] - void SetMetadata(JMetadata metadata) { mMetadata = metadata; } - - /// Get the JFactory's metadata. This is meant to be called by user during their JFactoryT::Process - /// and also used by JEvent under the hood. - /// Metadata will *not* be cleared on ClearData(), but will be destroyed when the JFactoryT is. - JMetadata GetMetadata() { return mMetadata; } - protected: std::vector mData; - JMetadata mMetadata; }; template diff --git a/src/libraries/JANA/JLogger.h b/src/libraries/JANA/JLogger.h index 189f5941e..df02124a5 100644 --- a/src/libraries/JANA/JLogger.h +++ b/src/libraries/JANA/JLogger.h @@ -4,33 +4,40 @@ #pragma once +#include + #include #include -#include +#include +#include +#include +#include + + struct JLogger { enum class Level { TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF }; Level level; std::ostream *destination; - std::string className; + std::string group; bool show_level = true; - bool show_classname = false; - bool show_timestamp = false; + bool show_group = false; + bool show_timestamp = true; bool show_threadstamp = false; explicit JLogger(JLogger::Level level = JLogger::Level::INFO, std::ostream* destination = &std::cout, - std::string className = "") - : level(level), destination(destination), className(std::move(className)) {}; + std::string group = "") + : level(level), destination(destination), group(std::move(group)) {}; JLogger(const JLogger&) = default; JLogger& operator=(const JLogger&) = default; - void SetTag(std::string tag) {className = tag; } - void SetTimestampFlag() {show_timestamp = true; } - void UnsetTimestampFlag() {show_timestamp = false; } - void SetThreadstampFlag() {show_threadstamp = true; } - void UnsetThreadstampFlag() {show_threadstamp = false; } + void SetGroup(std::string group) {this->group = group; } + void ShowGroup(bool show) {show_group = show; } + void ShowLevel(bool show) {show_level = show; } + void ShowTimestamp(bool show) {show_timestamp = show; } + void ShowThreadstamp(bool show) {show_threadstamp = show; } }; static JLogger default_cout_logger = JLogger(JLogger::Level::TRACE, &std::cout, "JANA"); @@ -38,13 +45,13 @@ static JLogger default_cerr_logger = JLogger(JLogger::Level::TRACE, &std::cerr, inline std::ostream& operator<<(std::ostream& s, JLogger::Level l) { switch (l) { - case JLogger::Level::TRACE: return s << "TRACE"; - case JLogger::Level::DEBUG: return s << "DEBUG"; - case JLogger::Level::INFO: return s << "INFO"; - case JLogger::Level::WARN: return s << "WARN"; - case JLogger::Level::ERROR: return s << "ERROR"; - case JLogger::Level::FATAL: return s << "FATAL"; - default: return s << "OFF"; + case JLogger::Level::TRACE: return s << "trace"; + case JLogger::Level::DEBUG: return s << "debug"; + case JLogger::Level::INFO: return s << "info"; + case JLogger::Level::WARN: return s << "warn "; + case JLogger::Level::ERROR: return s << "error"; + case JLogger::Level::FATAL: return s << "fatal"; + default: return s << "off"; } } @@ -63,13 +70,35 @@ struct JLogMessage { JLogger::Level level = JLogger::Level::INFO) : logger(logger), level(level) { + if (logger.show_timestamp) { + auto now = std::chrono::system_clock::now(); + std::time_t current_time = std::chrono::system_clock::to_time_t(now); + + tm tm_buf; + localtime_r(¤t_time, &tm_buf); + + // Extract milliseconds by calculating the duration since the last whole second + auto milliseconds = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + builder << std::put_time(&tm_buf, "%H:%M:%S."); + builder << std::setfill('0') << std::setw(3) << milliseconds.count() << std::setfill(' ') << " "; + } if (logger.show_level) { - builder << "[" << level << "] "; + switch (level) { + case JLogger::Level::TRACE: builder << "[trace] "; break; + case JLogger::Level::DEBUG: builder << "[debug] "; break; + case JLogger::Level::INFO: builder << "[info] "; break; + case JLogger::Level::WARN: builder << "[warn] "; break; + case JLogger::Level::ERROR: builder << "[error] "; break; + case JLogger::Level::FATAL: builder << "[fatal] "; break; + default: builder << "[?????] "; + } + } + if (logger.show_threadstamp) { + builder << std::this_thread::get_id() << " "; } - if (logger.show_classname) { - builder << logger.className << ": "; + if (logger.show_group) { + builder << "[" << logger.group << "] "; } - // TODO: Re-add thread and timestamp info? } // Helper function for truncating long strings to keep our log readable diff --git a/src/libraries/JANA/JMultifactory.cc b/src/libraries/JANA/JMultifactory.cc index e6da04fc1..f53c50fcb 100644 --- a/src/libraries/JANA/JMultifactory.cc +++ b/src/libraries/JANA/JMultifactory.cc @@ -46,14 +46,14 @@ void JMultifactory::Execute(const std::shared_ptr& event) { }); } -void JMultifactory::Release() { +void JMultifactory::DoFinish() { std::lock_guard lock(m_mutex); // Only call Finish() if we actually initialized // Only call Finish() once + if (m_status == Status::Initialized) { - CallWithJExceptionWrapper("JMultifactory::Finish", [&](){ - Finish(); - }); + CallWithJExceptionWrapper("JMultifactory::EndRun", [&](){ EndRun(); }); + CallWithJExceptionWrapper("JMultifactory::Finish", [&](){ Finish(); }); m_status = Status::Finalized; } } @@ -66,12 +66,17 @@ JFactorySet* JMultifactory::GetHelpers() { void JMultifactory::DoInit() { std::lock_guard lock(m_mutex); - if (m_status == Status::Uninitialized) { - CallWithJExceptionWrapper("JMultifactory::Init", [&](){ - Init(); - }); - m_status = Status::Initialized; + if (m_status != Status::Uninitialized) { + throw JException("Attempted to initialzie a JMultifactory that has already been initialized!"); + } + CallWithJExceptionWrapper("JMultifactory::Init", [&](){ Init(); }); + for (auto* parameter : m_parameters) { + parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); + } + for (auto* service : m_services) { + service->Fetch(m_app); } + m_status = Status::Initialized; } void JMultifactory::Summarize(JComponentSummary& summary) const { diff --git a/src/libraries/JANA/JMultifactory.h b/src/libraries/JANA/JMultifactory.h index d41dc1442..77f9832b6 100644 --- a/src/libraries/JANA/JMultifactory.h +++ b/src/libraries/JANA/JMultifactory.h @@ -118,12 +118,9 @@ class JMultifactory : public jana::components::JComponent, void DoInit(); - void Execute(const std::shared_ptr&); - // Should this be execute or create? Who is tracking that this is called at most once per event? - // Do we need something like JFactory::Status? Also, how do we ensure that CreationStatus is correct as well? + void DoFinish(); - void Release(); - // Release makes sure Finish() is called exactly once + void Execute(const std::shared_ptr&); JFactorySet* GetHelpers(); // This exposes the mHelpers JFactorySet, which contains a JFactoryT for each declared output of the multifactory. diff --git a/src/libraries/JANA/JService.cc b/src/libraries/JANA/JService.cc index 0a66f258c..60635b262 100644 --- a/src/libraries/JANA/JService.cc +++ b/src/libraries/JANA/JService.cc @@ -9,12 +9,18 @@ void JService::DoInit(JServiceLocator* sl) { std::lock_guard lock(m_mutex); if (this->m_status != Status::Uninitialized) return; - for (auto* parameter : m_parameters) { - parameter->Configure(*(m_app->GetJParameterManager()), m_prefix); - } + if (m_app != nullptr) { + + auto params = m_app->GetJParameterManager(); + m_logger = params->GetLogger(GetLoggerName()); + + for (auto* parameter : m_parameters) { + parameter->Configure(*params, m_prefix); + } - for (auto* service : m_services) { - service->Init(GetApplication()); // TODO: This badly needs to be renamed. Maybe Fetch? + for (auto* service : m_services) { + service->Fetch(m_app); + } } CallWithJExceptionWrapper("JService::acquire_services", [&](){ this->acquire_services(sl); }); diff --git a/src/libraries/JANA/JVersion.cc b/src/libraries/JANA/JVersion.cc index c5482ef7e..97ec511af 100644 --- a/src/libraries/JANA/JVersion.cc +++ b/src/libraries/JANA/JVersion.cc @@ -31,7 +31,7 @@ void JVersion::PrintSplash(std::ostream& os) { void JVersion::PrintVersionDescription(std::ostream& os) { - os << "JANA2 version: " << JVersion::GetVersion() << " "; + os << " JANA2 version: " << JVersion::GetVersion() << " "; if (is_unknown) { os << " (unknown git status)"; } @@ -46,12 +46,12 @@ void JVersion::PrintVersionDescription(std::ostream& os) { } os << std::endl; if (!JVersion::is_unknown) { - os << "Commit hash: " << JVersion::GetCommitHash() << std::endl; - os << "Commit date: " << JVersion::GetCommitDate() << std::endl; + os << " Commit hash: " << JVersion::GetCommitHash() << std::endl; + os << " Commit date: " << JVersion::GetCommitDate() << std::endl; } - os << "Install prefix: " << JVersion::GetInstallDir() << std::endl; + os << " Install prefix: " << JVersion::GetInstallDir() << std::endl; if (JVersion::HasPodio() || JVersion::HasROOT() || JVersion::HasXerces()) { - os << "Optional deps: "; + os << " Optional deps: "; if (JVersion::HasPodio()) os << "Podio "; if (JVersion::HasROOT()) os << "ROOT "; if (JVersion::HasXerces()) os << "Xerces "; diff --git a/src/libraries/JANA/Services/JComponentManager.cc b/src/libraries/JANA/Services/JComponentManager.cc index fb050eec7..e7fe17fd4 100644 --- a/src/libraries/JANA/Services/JComponentManager.cc +++ b/src/libraries/JANA/Services/JComponentManager.cc @@ -10,7 +10,9 @@ #include #include -JComponentManager::JComponentManager() {} +JComponentManager::JComponentManager() { + SetPrefix("jana"); +} JComponentManager::~JComponentManager() { @@ -69,11 +71,11 @@ void JComponentManager::configure_components() { void JComponentManager::preinitialize_components() { for (auto* src : m_evt_srces) { src->SetApplication(GetApplication()); - src->SetLogger(m_logging->get_logger(src->GetLoggerName())); + src->SetLogger(m_params->GetLogger(src->GetLoggerName())); } for (auto* proc : m_evt_procs) { proc->SetApplication(GetApplication()); - proc->SetLogger(m_logging->get_logger(proc->GetLoggerName())); + proc->SetLogger(m_params->GetLogger(proc->GetLoggerName())); } for (auto* fac_gen : m_fac_gens) { fac_gen->SetApplication(GetApplication()); @@ -85,7 +87,7 @@ void JComponentManager::preinitialize_components() { } for (auto* unfolder : m_unfolders) { unfolder->SetApplication(GetApplication()); - unfolder->SetLogger(m_logging->get_logger(unfolder->GetLoggerName())); + unfolder->SetLogger(m_params->GetLogger(unfolder->GetLoggerName())); } } @@ -118,6 +120,7 @@ void JComponentManager::initialize_components() { try { // Run Init() on each factory in order to capture any parameters // (and eventually services) that are retrieved via GetApplication(). + fac->SetApplication(GetApplication()); fac->DoInit(); } catch (...) { @@ -229,11 +232,7 @@ JEventSourceGenerator *JComponentManager::resolve_event_source(std::string sourc } // Otherwise, report the problem and throw - auto ex = JException("Unable to open event source \"%s\": No suitable generator found!", source_name.c_str()); - std::ostringstream os; - make_backtrace(os); - ex.stacktrace = os.str(); - throw ex; + throw JException("Unable to open event source \"%s\": No suitable generator found!", source_name.c_str()); } diff --git a/src/libraries/JANA/Services/JComponentManager.h b/src/libraries/JANA/Services/JComponentManager.h index 691976999..3f6a501e7 100644 --- a/src/libraries/JANA/Services/JComponentManager.h +++ b/src/libraries/JANA/Services/JComponentManager.h @@ -56,7 +56,6 @@ class JComponentManager : public JService { private: Service m_params {this}; - Service m_logging {this}; std::string m_current_plugin_name; std::vector m_src_names; diff --git a/src/libraries/JANA/Services/JLoggingService.cc b/src/libraries/JANA/Services/JLoggingService.cc deleted file mode 100644 index 69645e792..000000000 --- a/src/libraries/JANA/Services/JLoggingService.cc +++ /dev/null @@ -1,118 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#include - -#include - - -template <> -inline void JParameterManager::Parse(const std::string& in, JLogger::Level& out) { - std::string token(in); - std::transform(in.begin(), in.end(), token.begin(), ::tolower); - if (std::strcmp(token.c_str(), "trace") == 0) { - out = JLogger::Level::TRACE; - } - else if (std::strcmp(token.c_str(), "debug") == 0) { - out = JLogger::Level::DEBUG; - } - else if (std::strcmp(token.c_str(), "info") == 0) { - out = JLogger::Level::INFO; - } - else if (std::strcmp(token.c_str(), "warn") == 0) { - out = JLogger::Level::WARN; - } - else if (std::strcmp(token.c_str(), "error") == 0) { - out = JLogger::Level::ERROR; - } - else if (std::strcmp(token.c_str(), "fatal") == 0) { - out = JLogger::Level::FATAL; - } - else if (std::strcmp(token.c_str(), "off") == 0) { - out = JLogger::Level::OFF; - } - else { - throw JException("Unable to parse log level: '%s'. Options are: TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF", in.c_str()); - } -} - -void JLoggingService::acquire_services(JServiceLocator* serviceLocator) { - - auto params = serviceLocator->get(); - params->SetDefaultParameter("log:global", m_global_log_level, "Default log level"); - - std::vector groups; - params->SetDefaultParameter("log:off", groups, "Comma-separated list of loggers that should be turned off completely"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::OFF; - } - groups.clear(); - params->SetDefaultParameter("log:fatal", groups, "Comma-separated list of loggers that should only print FATAL"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::FATAL; - } - groups.clear(); - params->SetDefaultParameter("log:error", groups, "Comma-separated list of loggers that should only print ERROR or higher"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::ERROR; - } - groups.clear(); - params->SetDefaultParameter("log:warn", groups, "Comma-separated list of loggers that should only print WARN or higher"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::WARN; - } - groups.clear(); - params->SetDefaultParameter("log:info", groups, "Comma-separated list of loggers that should only print INFO or higher"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::INFO; - } - groups.clear(); - params->SetDefaultParameter("log:debug", groups, "Comma-separated list of loggers that should only print DEBUG or higher"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::DEBUG; - } - groups.clear(); - params->SetDefaultParameter("log:trace", groups, "Comma-separated list of loggers that should print everything"); - for (auto& s : groups) { - m_local_log_levels[s] = JLogger::Level::TRACE; - } - // Set the log level on the parameter manager, resolving the chicken-and-egg problem. - params->SetLogger(get_logger("JParameterManager")); -} - - -void JLoggingService::set_level(JLogger::Level level) { - m_global_log_level = level; -} - - -void JLoggingService::set_level(std::string className, JLogger::Level level) { - m_local_log_levels[className] = level; -} - - -JLogger JLoggingService::get_logger() { - return JLogger(m_global_log_level); -} - - -JLogger JLoggingService::get_logger(std::string className) { - - JLogger logger; - logger.show_classname = true; - logger.className = className; - - auto search = m_local_log_levels.find(className); - if (search != m_local_log_levels.end()) { - logger.level = search->second; - } else { - logger.level = m_global_log_level; - } - return logger; -} - - - - diff --git a/src/libraries/JANA/Services/JLoggingService.h b/src/libraries/JANA/Services/JLoggingService.h deleted file mode 100644 index 692d5b890..000000000 --- a/src/libraries/JANA/Services/JLoggingService.h +++ /dev/null @@ -1,48 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once - -#include -#include -#include - - -// Convenience macros for temporary debugging statements. -#ifndef _DBG__ -#define _DBG__ std::cerr<<__FILE__<<":"<<__LINE__< m_local_log_levels; - -public: - - void acquire_services(JServiceLocator* serviceLocator) override; - - void set_level(JLogger::Level level); - - void set_level(std::string className, JLogger::Level level); - - JLogger get_logger(); - - JLogger get_logger(std::string className); - -}; - - - diff --git a/src/libraries/JANA/Services/JParameterManager.cc b/src/libraries/JANA/Services/JParameterManager.cc index f61759693..7b8014af9 100644 --- a/src/libraries/JANA/Services/JParameterManager.cc +++ b/src/libraries/JANA/Services/JParameterManager.cc @@ -3,7 +3,9 @@ // Subject to the terms in the LICENSE file found in the top-level directory. #include "JParameterManager.h" +#include "JANA/JLogger.h" +#include #include #include #include @@ -15,10 +17,7 @@ using namespace std; /// @brief Default constructor JParameterManager::JParameterManager() { - // Set the logger temporarily, until the JLoggingService figures out the correct log level - m_logger.show_classname = true; - m_logger.className = "JParameterManager"; - m_logger.level = JLogger::Level::INFO; + SetLoggerName("jana"); } /// @brief Copy constructor @@ -94,22 +93,6 @@ void JParameterManager::PrintParameters() { PrintParameters(m_verbosity, m_strictness); } -void JParameterManager::PrintParameters(bool show_defaulted, bool show_advanced, bool warn_on_unused) { - int verbosity = 1; - int strictness = 0; - - if (show_advanced) { - verbosity = 3; - } - else if (show_defaulted) { - verbosity = 2; - } - if (warn_on_unused) { - strictness = 1; - } - PrintParameters(verbosity, strictness); -} - /// @brief Prints parameters to stdout /// @@ -128,15 +111,9 @@ void JParameterManager::PrintParameters(int verbosity, int strictness) { return; } - bool warnings_present = false; bool strictness_violation = false; - // We don't need to show "Advanced" as a warning unless we are using full verbosity - if (verbosity == 3) { - warnings_present = true; - } - - // Check for warnings and unused parameters first + // Unused parameters first // The former might change the table columns and the latter might change the filter verbosity for (auto& pair : m_parameters) { const auto& key = pair.first; @@ -144,17 +121,10 @@ void JParameterManager::PrintParameters(int verbosity, int strictness) { if ((strictness > 0) && (!param->IsDefault()) && (!param->IsUsed())) { strictness_violation = true; - warnings_present = true; - if (strictness < 2) { - LOG_WARN(m_logger) << "Parameter '" << key << "' appears to be unused. Possible typo?" << LOG_END; - } - else { - LOG_FATAL(m_logger) << "Parameter '" << key << "' appears to be unused. Possible typo?" << LOG_END; - } + LOG_ERROR(m_logger) << "Parameter '" << key << "' appears to be unused. Possible typo?" << LOG_END; } if ((!param->IsDefault()) && (param->IsDeprecated())) { - LOG_WARN(m_logger) << "Parameter '" << key << "' has been deprecated and will no longer be supported in the next release." << LOG_END; - warnings_present = true; + LOG_ERROR(m_logger) << "Parameter '" << key << "' has been deprecated and may no longer be supported in future releases." << LOG_END; } } @@ -169,10 +139,10 @@ void JParameterManager::PrintParameters(int verbosity, int strictness) { for (auto& pair : m_parameters) { auto param = pair.second; - if (param->IsDeprecated() && (param->IsDefault())) continue; + if (param->IsDeprecated() && (param->IsDefault())) continue; // Always hide deprecated parameters that are NOT in use - if ((verbosity == 1) && (param->IsDefault())) continue; + if ((verbosity == 1) && (param->IsDefault())) continue; // At verbosity level 1, hide all default-valued parameters if ((verbosity == 2) && (param->IsDefault()) && (param->IsAdvanced())) continue; @@ -187,48 +157,48 @@ void JParameterManager::PrintParameters(int verbosity, int strictness) { return; } - // Print table - JTablePrinter table; - table.AddColumn("Name", JTablePrinter::Justify::Left, 20); - if (warnings_present) { - table.AddColumn("Warnings"); // IsDeprecated column - } - table.AddColumn("Value", JTablePrinter::Justify::Left, 25); - table.AddColumn("Default", JTablePrinter::Justify::Left, 25); - table.AddColumn("Description", JTablePrinter::Justify::Left, 50); - + LOG_WARN(m_logger) << "Configuration Parameters" << LOG_END; for (JParameter* p: params_to_print) { - if (warnings_present) { - std::string warning; - if (p->IsDeprecated()) { - // If deprecated, it no longer matters whether it is advanced or not. If unused, won't show up here anyway. - warning = "Deprecated"; - } - else if (!p->IsUsed()) { - // Can't be both deprecated and unused, since JANA only finds out that it is deprecated by trying to use it - // Can't be both advanced and unused, since JANA only finds out that it is advanced by trying to use it - warning = "Unused"; - } - else if (p->IsAdvanced()) { - warning = "Advanced"; - } - table | p->GetKey() - | warning - | p->GetValue() - | p->GetDefault() - | p->GetDescription(); + LOG_WARN(m_logger) << LOG_END; + LOG_WARN(m_logger) << " - key: " << p->GetKey() << LOG_END; + if (!p->IsDefault()) { + LOG_WARN(m_logger) << " value: " << p->GetValue() << LOG_END; } - else { - table | p->GetKey() - | p->GetValue() - | p->GetDefault() - | p->GetDescription(); + if (p->HasDefault()) { + LOG_WARN(m_logger) << " default: " << p->GetDefault() << LOG_END; + } + if (!p->GetDescription().empty()) { + std::istringstream iss(p->GetDescription()); + std::string line; + bool is_first_line = true; + while (std::getline(iss, line)) { + if (is_first_line) { + LOG_INFO(m_logger) << " description: " << line << LOG_END; + } + else { + LOG_INFO(m_logger) << " " << line << LOG_END; + } + is_first_line = false; + } + } + if (p->IsConflicted()) { + LOG_WARN(m_logger) << " warning: Conflicting defaults" << LOG_END; + } + if (p->IsDeprecated()) { + LOG_WARN(m_logger) << " warning: Deprecated" << LOG_END; + // If deprecated, it no longer matters whether it is advanced or not. If unused, won't show up here anyway. + } + if (!p->IsUsed()) { + // Can't be both deprecated and unused, since JANA only finds out that it is deprecated by trying to use it + // Can't be both advanced and unused, since JANA only finds out that it is advanced by trying to use it + LOG_WARN(m_logger) << " warning: Unused" << LOG_END; + } + if (p->IsAdvanced()) { + LOG_WARN(m_logger) << " warning: Advanced" << LOG_END; } } - std::ostringstream ss; - table.Render(ss); - LOG_INFO(m_logger) << "Configuration Parameters\n" << ss.str() << LOG_END; + LOG_WARN(m_logger) << LOG_END; // Now that we've printed the table, we can throw an exception if we are being super strict if (strictness_violation && strictness > 1) { @@ -409,3 +379,33 @@ void JParameterManager::FilterParameters(std::map &par parms[key] = value; } } + +JLogger JParameterManager::GetLogger(const std::string& component_prefix) { + + JLogger logger; + logger.group = component_prefix; + + auto global_log_level = RegisterParameter("jana:global_loglevel", JLogger::Level::INFO, "Global log level"); + + bool enable_timestamp = RegisterParameter("jana:log:show_timestamp", true, "Show timestamp in log output"); + auto enable_threadstamp = RegisterParameter("jana:log:show_threadstamp", false, "Show threadstamp in log output"); + auto enable_group = RegisterParameter("jana:log:show_group", false, "Show threadstamp in log output"); + auto enable_level = RegisterParameter("jana:log:show_level", true, "Show threadstamp in log output"); + + if (component_prefix.empty()) { + logger.level = global_log_level; + } + else { + std::ostringstream os; + os << component_prefix << ":loglevel"; + logger.level = RegisterParameter(os.str(), global_log_level, "Component log level"); + } + logger.ShowLevel(enable_level); + logger.ShowTimestamp(enable_timestamp); + logger.ShowThreadstamp(enable_threadstamp); + logger.ShowGroup(enable_group); + logger.show_threadstamp = enable_threadstamp; + return logger; +} + + diff --git a/src/libraries/JANA/Services/JParameterManager.h b/src/libraries/JANA/Services/JParameterManager.h index d4ea981be..5fe67c2ee 100644 --- a/src/libraries/JANA/Services/JParameterManager.h +++ b/src/libraries/JANA/Services/JParameterManager.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,7 @@ class JParameter { // We want to differentiate these from the parameters that users are meant to control and understand. bool m_is_used = false; // If a parameter hasn't been used, it probably contains a typo, and we should warn the user. + bool m_is_conflicted = false; // Whether or not this parameter has been registered with inconsistent default values public: @@ -55,6 +57,7 @@ class JParameter { inline bool IsAdvanced() const { return m_is_advanced; } inline bool IsUsed() const { return m_is_used; } inline bool IsDeprecated() const { return m_is_deprecated; } + inline bool IsConflicted() const { return m_is_conflicted; } inline void SetKey(std::string key) { m_name = std::move(key); } inline void SetValue(std::string val) { m_value = std::move(val); } @@ -65,6 +68,7 @@ class JParameter { inline void SetIsAdvanced(bool isHidden) { m_is_advanced = isHidden; } inline void SetIsUsed(bool isUsed) { m_is_used = isUsed; } inline void SetIsDeprecated(bool isDeprecated) { m_is_deprecated = isDeprecated; } + inline void SetIsConflicted(bool isConflicted) { m_is_conflicted = isConflicted; } }; @@ -87,9 +91,6 @@ class JParameterManager : public JService { void PrintParameters(int verbosity, int strictness); - [[deprecated]] - void PrintParameters(bool show_defaulted, bool show_advanced=true, bool warn_on_unused=false); - std::map GetAllParameters(); template @@ -141,6 +142,8 @@ class JParameterManager : public JService { static std::string ToLower(const std::string& name); + JLogger GetLogger(const std::string& prefix); + private: std::map m_parameters; @@ -148,8 +151,6 @@ class JParameterManager : public JService { int m_strictness = 1; int m_verbosity = 1; - JLogger m_logger; - std::mutex m_mutex; }; @@ -264,15 +265,20 @@ JParameter* JParameterManager::SetDefaultParameter(std::string name, T& val, std // However, we still want to warn the user if the same parameter was declared with different values. Parse(param->GetDefault(),t); if (!Equals(val, t)) { - LOG_WARN(m_logger) << "Parameter '" << name << "' has conflicting defaults: '" - << Stringify(val) << "' vs '" << param->GetDefault() << "'" - << LOG_END; - if (param->IsDefault()) { - // If we tried to set the same default parameter twice with different values, and there is no - // existing non-default value, we remember the _latest_ default value. This way, we return the - // default provided by the caller, instead of the default provided by the mysterious interloper. - param->SetValue(Stringify(val)); + if (!param->IsConflicted()) { + // Only show this warning once per parameter + LOG_WARN(m_logger) << "Parameter '" << name << "' has conflicting defaults: '" + << Stringify(val) << "' vs '" << param->GetDefault() << "'" + << LOG_END; + param->SetIsConflicted(true); } + // If we tried to set the same default parameter twice with different values, and there is no + // existing non-default value, we remember the _latest_ default value. This way, we return the + // default provided by the caller, instead of the default provided by the mysterious interloper. + param->SetDefault(Stringify(val)); + } + if (param->IsDefault()) { + param->SetValue(Stringify(val)); // Use _latest_ default value } } } @@ -384,6 +390,37 @@ inline void JParameterManager::Parse(const std::string& value, std::vector &v } } +/// @brief Specialization for JLogger::Level enum +template <> +inline void JParameterManager::Parse(const std::string& in, JLogger::Level& out) { + std::string token(in); + std::transform(in.begin(), in.end(), token.begin(), ::tolower); + if (std::strcmp(token.c_str(), "trace") == 0) { + out = JLogger::Level::TRACE; + } + else if (std::strcmp(token.c_str(), "debug") == 0) { + out = JLogger::Level::DEBUG; + } + else if (std::strcmp(token.c_str(), "info") == 0) { + out = JLogger::Level::INFO; + } + else if (std::strcmp(token.c_str(), "warn") == 0) { + out = JLogger::Level::WARN; + } + else if (std::strcmp(token.c_str(), "error") == 0) { + out = JLogger::Level::ERROR; + } + else if (std::strcmp(token.c_str(), "fatal") == 0) { + out = JLogger::Level::FATAL; + } + else if (std::strcmp(token.c_str(), "off") == 0) { + out = JLogger::Level::OFF; + } + else { + throw JException("Unable to parse log level: '%s'. Options are: TRACE, DEBUG, INFO, WARN, ERROR, FATAL, OFF", in.c_str()); + } +} + #if __cplusplus >= 201703L /// @brief Basic implementation of Stringify for C++17 and newer. Provides a helpful error message when attempting to stringify a type that doesn't come with a stream operator. template diff --git a/src/libraries/JANA/Services/JPluginLoader.cc b/src/libraries/JANA/Services/JPluginLoader.cc index 5ab20ae2c..f3f7bd918 100644 --- a/src/libraries/JANA/Services/JPluginLoader.cc +++ b/src/libraries/JANA/Services/JPluginLoader.cc @@ -189,7 +189,7 @@ void JPluginLoader::attach_plugin(std::string name, std::string path) { } // Run InitPlugin() and wrap exceptions as needed - LOG_INFO(m_logger) << "Initializing plugin \"" << name << "\" at path \"" << path << "\"" << LOG_END; + LOG_WARN(m_logger) << "Loading plugin '" << name << "' from '" << path << "'" << LOG_END; try { (*initialize_proc)(GetApplication()); diff --git a/src/libraries/JANA/Services/JPluginLoader.h b/src/libraries/JANA/Services/JPluginLoader.h index 3707292dd..6a090c872 100644 --- a/src/libraries/JANA/Services/JPluginLoader.h +++ b/src/libraries/JANA/Services/JPluginLoader.h @@ -36,7 +36,9 @@ class JPluginLoader : public JService { public: - JPluginLoader() = default; + JPluginLoader() { + SetPrefix("jana"); + } ~JPluginLoader() override = default; void Init() override; diff --git a/src/libraries/JANA/Services/JWiringService.cc b/src/libraries/JANA/Services/JWiringService.cc index ca237066d..0056e371b 100644 --- a/src/libraries/JANA/Services/JWiringService.cc +++ b/src/libraries/JANA/Services/JWiringService.cc @@ -1,8 +1,9 @@ #include "JWiringService.h" -#include "JANA/Utils/JEventLevel.h" +#include "toml.hpp" +#include #include -#include +#include namespace jana::services { @@ -11,40 +12,54 @@ void JWiringService::Init() { // User is _only_ allowed to specify wiring file via parameter // This way, we can restrict calling JWiringService::Init until inside JApplication::Init // Then we can load the wiring file exactly once. All WiredFactoryGenerators - // (Recursively) load files + // (recursively) load files + if (!m_wirings_input_file().empty()) { + AddWirings(*m_wirings_input_file); + } } -std::unique_ptr JWiringService::overlay(std::unique_ptr&& above, std::unique_ptr&& below) { +void JWiringService::AddWirings(std::vector>& wirings_bundle, const std::string& bundle_source) { - // In theory this should be handled by the caller, but let's check just in case - if (above->plugin_name != below->plugin_name) throw JException("Plugin name mismatch!"); - if (above->type_name != below->type_name) throw JException("Type name mismatch!"); + std::set prefixes_in_bundle; + for (auto& wiring: wirings_bundle) { - if (above->input_names.empty() && !below->input_names.empty()) { - above->input_names = std::move(below->input_names); - } - if (above->input_levels.empty() && !below->input_levels.empty()) { - above->input_levels = std::move(below->input_levels); - } - if (above->output_names.empty() && !below->output_names.empty()) { - above->output_names = std::move(below->output_names); - } - for (const auto& [key, val] : below->configs) { - if (above->configs.find(key) == above->configs.end()) { - above->configs[key] = val; + // Assert that this wiring's prefix is unique _within_ this bundle. + auto bundle_it = prefixes_in_bundle.find(wiring->prefix); + if (bundle_it != prefixes_in_bundle.end()) { + throw JException("Duplicated prefix '%s' in wiring bundle '%s'", wiring->prefix.c_str(), bundle_source.c_str()); + } + prefixes_in_bundle.insert(wiring->prefix); + + // Check whether we have seen this prefix before + auto it = m_wirings_from_prefix.find(wiring->prefix); + if (it == m_wirings_from_prefix.end()) { + // This is a new wiring + m_wirings_from_prefix[wiring->prefix] = wiring.get(); + m_wirings_from_type_and_plugin_names[{wiring->type_name, wiring->plugin_name}].push_back(wiring.get()); + m_wirings.push_back(std::move(wiring)); + } + else { + // Wiring is already defined; overlay this wiring _below_ the existing wiring + // First we do some sanity checks + if (wiring->type_name != it->second->type_name) { + throw JException("Wiring mismatch: type name '%s' vs '%s'", wiring->type_name.c_str(), it->second->type_name.c_str()); + } + if (wiring->plugin_name != it->second->plugin_name) { + throw JException("Wiring mismatch: plugin name '%s' vs '%s'", wiring->plugin_name.c_str(), it->second->plugin_name.c_str()); + } + Overlay(*(it->second), *wiring); + // Useful information from `wiring` has been copied into `it->second`. + // `wiring` will now be automatically destroyed } } - // below gets automatically deleted - return std::move(above); + // At this point all wirings have been moved out of wirings_bundle, so we clear it to avoid confusing callers + wirings_bundle.clear(); } - -std::vector> JWiringService::parse_table(const toml::table& table) { +void JWiringService::AddWirings(const toml::table& table, const std::string& source) { std::vector> wirings; - std::map prefix_lookup; - auto facs = table["factory"].as_array(); if (facs == nullptr) { throw JException("No factories found!"); @@ -60,16 +75,6 @@ std::vector> JWiringService::parse_table wiring->plugin_name = f["plugin_name"].value().value(); wiring->type_name = f["type_name"].value().value(); wiring->prefix = f["prefix"].value().value(); - auto it = prefix_lookup.find(wiring->prefix); - if (it != prefix_lookup.end()) { - std::ostringstream oss; - oss << "Duplicated factory prefix in wiring file: " << std::endl; - oss << " Prefix: " << wiring->prefix << std::endl; - oss << " Type name: " << wiring->type_name << " vs " << it->second->type_name << std::endl; - oss << " Plugin name: " << wiring->plugin_name << " vs " << it->second->plugin_name << std::endl; - throw JException(oss.str()); - } - prefix_lookup[wiring->prefix] = wiring.get(); wiring->level = parseEventLevel(f["level"].value_or("None")); @@ -106,9 +111,61 @@ std::vector> JWiringService::parse_table wirings.push_back(std::move(wiring)); } - return wirings; + AddWirings(wirings, source); +} + +void JWiringService::AddWirings(const std::string& filename) { + try { + auto tbl = toml::parse_file(filename); + AddWirings(tbl, filename); + } + catch (const toml::parse_error& err) { + auto e = JException("Error parsing TOML file: '%s'", filename.c_str()); + e.nested_exception = std::current_exception(); + throw e; + } } +const JWiringService::Wiring* JWiringService::GetWiring(const std::string& prefix) const { + auto it = m_wirings_from_prefix.find(prefix); + if (it == m_wirings_from_prefix.end()) { + return nullptr; + } + return it->second; +} + +const std::vector& +JWiringService::GetWirings(const std::string& plugin_name, const std::string& type_name) const { + + auto it = m_wirings_from_type_and_plugin_names.find({type_name, plugin_name}); + if (it == m_wirings_from_type_and_plugin_names.end()) { + return m_no_wirings; + } + return it->second; +} + + +void JWiringService::Overlay(Wiring& above, const Wiring& below) { + + // In theory this should be handled by the caller, but let's check just in case + if (above.plugin_name != below.plugin_name) throw JException("Plugin name mismatch!"); + if (above.type_name != below.type_name) throw JException("Type name mismatch!"); + + if (above.input_names.empty() && !below.input_names.empty()) { + above.input_names = std::move(below.input_names); + } + if (above.input_levels.empty() && !below.input_levels.empty()) { + above.input_levels = std::move(below.input_levels); + } + if (above.output_names.empty() && !below.output_names.empty()) { + above.output_names = std::move(below.output_names); + } + for (const auto& [key, val] : below.configs) { + if (above.configs.find(key) == above.configs.end()) { + above.configs[key] = val; + } + } +} diff --git a/src/libraries/JANA/Services/JWiringService.h b/src/libraries/JANA/Services/JWiringService.h index 49f313353..75e79cd72 100644 --- a/src/libraries/JANA/Services/JWiringService.h +++ b/src/libraries/JANA/Services/JWiringService.h @@ -10,8 +10,7 @@ #include #include -namespace jana { -namespace services { +namespace jana::services { class JWiringService : public JService { @@ -25,12 +24,10 @@ class JWiringService : public JService { std::vector input_names; std::vector input_levels; std::vector output_names; + std::vector output_levels; std::map configs; }; - using WiringIndex = std::map, std::map>; - // { (plugin_name,typename) : {prefix : const Wiring*}} - private: Parameter m_wirings_input_file {this, "jana:wiring_file", "wiring.toml", "Path to TOML file containing wiring definitions"}; @@ -39,18 +36,28 @@ class JWiringService : public JService { "Allow multiple definitions inside wiring files"}; std::vector> m_wirings; - WiringIndex m_wirings_index; + std::map m_wirings_from_prefix; + std::map, std::vector> m_wirings_from_type_and_plugin_names; + std::vector m_no_wirings; public: void Init() override; - void parse_file(std::string filename); - void add_wiring(std::unique_ptr wiring); - std::unique_ptr overlay(std::unique_ptr&& above, std::unique_ptr&& below); - const Wiring* get_wiring(std::string plugin_name, std::string type_name, std::string prefix); - const std::vector>& get_wirings(); - std::vector> parse_table(const toml::table& table); + void AddWirings(std::vector>& wirings, const std::string& source); + void AddWirings(const toml::table& table, const std::string& source); + void AddWirings(const std::string& filename); + + const Wiring* + GetWiring(const std::string& prefix) const; + + const std::vector& + GetWirings(const std::string& plugin_name, const std::string& type_name) const; + + const std::vector>& + GetWirings() const { return m_wirings; } + + static void Overlay(Wiring& above, const Wiring& below); }; -} // namespace services -} // namespace jana +} // namespace jana::services + diff --git a/src/libraries/JANA/Topology/JArrow.cc b/src/libraries/JANA/Topology/JArrow.cc new file mode 100644 index 000000000..3b0301fe5 --- /dev/null +++ b/src/libraries/JANA/Topology/JArrow.cc @@ -0,0 +1,116 @@ + +#include + +void JArrow::create_ports(size_t inputs, size_t outputs) { + m_ports.clear(); + for (size_t i=0; i= m_ports.size()) { + throw JException("Attempting to attach to a non-existent port! arrow=%s, port=%d", m_name.c_str(), port); + } + m_ports[port].queue = queue; +} + + +void JArrow::attach(JEventPool* pool, size_t port) { + // Place index is relative to whether it is an input or not + // Port index, however, is agnostic to whether it is an input or not + if (port >= m_ports.size()) { + throw JException("Attempting to attach to a non-existent place! arrow=%s, port=%d", m_name.c_str(), port); + } + m_ports[port].pool = pool; +} + + +JEvent* JArrow::pull(size_t port_index, size_t location_id) { + JEvent* event = nullptr; + auto& port = m_ports.at(port_index); + if (port.queue != nullptr) { + event = port.queue->Pop(location_id); + } + else if (port.pool != nullptr){ + event = port.pool->Pop( location_id); + } + else { + throw JException("Arrow %s: Port %d not wired!", m_name.c_str(), port_index); + } + // If pop() failed, the returned event is nullptr + return event; +} + + +void JArrow::push(OutputData& outputs, size_t output_count, size_t location_id) { + for (size_t output = 0; output < output_count; ++output) { + JEvent* event = outputs[output].first; + int port_index = outputs[output].second; + Port& port = m_ports.at(port_index); + if (port.queue != nullptr) { + port.queue->Push(event, location_id); + } + else if (port.pool != nullptr) { + if (!port.is_input) { + event->Clear(); + } + port.pool->Push(event, location_id); + } + else { + throw JException("Arrow %s: Port %d not wired!", m_name.c_str(), port_index); + } + } +} + +JArrow::FireResult JArrow::execute(size_t location_id) { + + auto start_total_time = std::chrono::steady_clock::now(); + if (m_next_visit_time > start_total_time) { + // If we haven't reached the next visit time, exit immediately + return FireResult::ComeBackLater; + } + + JEvent* input = nullptr; + if (m_next_input_port != -1) { + input = pull(m_next_input_port, location_id); + } + + if (input == nullptr && m_next_input_port != -1) { + // Failed to obtain the input we needed; arrow is NOT ready to fire + return FireResult::NotRunYet; + } + + // Obtained the input we needed; arrow is ready to fire + // Remember that `input` might be nullptr, in case arrow doesn't need any input event + + OutputData outputs; + size_t output_count; + JArrow::FireResult result = JArrow::FireResult::KeepGoing; + + fire(input, outputs, output_count, result); + + push(outputs, output_count, location_id); + + return result; +} + + +std::string to_string(JArrow::FireResult r) { + switch (r) { + case JArrow::FireResult::NotRunYet: return "NotRunYet"; + case JArrow::FireResult::KeepGoing: return "KeepGoing"; + case JArrow::FireResult::ComeBackLater: return "ComeBackLater"; + case JArrow::FireResult::Finished: return "Finished"; + default: return "Error"; + } +} + + + diff --git a/src/libraries/JANA/Topology/JArrow.h b/src/libraries/JANA/Topology/JArrow.h index 981dbef2a..e0dcbf011 100644 --- a/src/libraries/JANA/Topology/JArrow.h +++ b/src/libraries/JANA/Topology/JArrow.h @@ -3,299 +3,86 @@ // Subject to the terms in the LICENSE file found in the top-level directory. #pragma once -#include -#include #include #include -#include "JArrowMetrics.h" #include #include -#include -#include +#include +#include -#ifndef JANA2_ARROWDATA_MAX_SIZE -#define JANA2_ARROWDATA_MAX_SIZE 10 -#endif - -struct PlaceRefBase; - class JArrow { -private: - const std::string m_name; // Used for human understanding - const bool m_is_parallel; // Whether or not it is safe to parallelize - const bool m_is_source; // Whether or not this arrow should activate/drain the topology - bool m_is_sink; // Whether or not tnis arrow contributes to the final event count - JArrowMetrics m_metrics; // Performance information accumulated over all workers + friend class JTopologyBuilder; - mutable std::mutex m_arrow_mutex; // Protects access to arrow properties +public: + using OutputData = std::array, 2>; + enum class FireResult {NotRunYet, KeepGoing, ComeBackLater, Finished}; - // TODO: Get rid of me - size_t m_chunksize = 1; // Number of items to pop off the input queue at once + struct Port { + JEventQueue* queue = nullptr; + JEventPool* pool = nullptr; + bool is_input = false; + }; - friend class JScheduler; - std::vector m_listeners; // Downstream Arrows +private: + std::string m_name; // Used for human understanding + bool m_is_parallel; // Whether or not it is safe to parallelize + bool m_is_source; // Whether or not this arrow should activate/drain the topology + bool m_is_sink; // Whether or not tnis arrow contributes to the final event count protected: - // This is usable by subclasses. + using clock_t = std::chrono::steady_clock; + + int m_next_input_port=0; // -1 denotes "no input necessary", e.g. for barrier events + clock_t::time_point m_next_visit_time=clock_t::now(); + std::vector m_ports; JLogger m_logger; - friend class JTopologyBuilder; - std::vector m_places; // Will eventually supplant m_listeners, m_chunksize public: + const std::string& get_name() { return m_name; } + JLogger& get_logger() { return m_logger; } bool is_parallel() { return m_is_parallel; } bool is_source() { return m_is_source; } bool is_sink() { return m_is_sink; } + Port& get_port(size_t port_index) { return m_ports.at(port_index); } + int get_next_port_index() { return m_next_input_port; } - std::string get_name() { return m_name; } - - void set_logger(JLogger logger) { - m_logger = logger; - } - - void set_is_sink(bool is_sink) { - m_is_sink = is_sink; - } - - // TODO: Get rid of me - void set_chunksize(size_t chunksize) { - std::lock_guard lock(m_arrow_mutex); - m_chunksize = chunksize; - } - - // TODO: Get rid of me - size_t get_chunksize() const { - std::lock_guard lock(m_arrow_mutex); - return m_chunksize; - } + void set_name(std::string name) { m_name = name; } + void set_logger(JLogger logger) { m_logger = logger; } + void set_is_parallel(bool is_parallel) { m_is_parallel = is_parallel; } + void set_is_source(bool is_source) { m_is_source = is_source; } + void set_is_sink(bool is_sink) { m_is_sink = is_sink; } - // TODO: Metrics should be encapsulated so that only actions are to update, clear, or summarize - JArrowMetrics& get_metrics() { - return m_metrics; + JArrow() { + m_is_parallel = false; + m_is_source = false; + m_is_sink = false; } - JArrow(std::string name, bool is_parallel, bool is_source, bool is_sink, size_t chunksize=16) : - m_name(std::move(name)), m_is_parallel(is_parallel), m_is_source(is_source), m_is_sink(is_sink), m_chunksize(chunksize) { - - m_metrics.clear(); + JArrow(std::string name, bool is_parallel, bool is_source, bool is_sink) : + m_name(std::move(name)), m_is_parallel(is_parallel), m_is_source(is_source), m_is_sink(is_sink) { }; virtual ~JArrow() = default; virtual void initialize() { }; - virtual void execute(JArrowMetrics& result, size_t location_id) = 0; - - virtual void finalize() {}; - - // TODO: Make no longer virtual - virtual size_t get_pending(); - - // TODO: Get rid of me - virtual size_t get_threshold(); - - virtual void set_threshold(size_t /* threshold */); - - void attach(JArrow* downstream) { - m_listeners.push_back(downstream); - }; - - void attach(PlaceRefBase* place) { - if (std::find(m_places.begin(), m_places.end(), place) == m_places.end()) { - m_places.push_back(place); - } - }; -}; - -template -struct Data { - std::array items; - size_t item_count = 0; - size_t reserve_count = 0; - size_t location_id; - - Data(size_t location_id = 0) : location_id(location_id) { - items = {nullptr}; - } -}; + virtual FireResult execute(size_t location_id); -struct PlaceRefBase { - void* place_ref = nullptr; - bool is_queue = true; - bool is_input = false; - size_t min_item_count = 1; - size_t max_item_count = 1; + virtual void fire(JEvent*, OutputData&, size_t&, FireResult&) {}; - virtual size_t get_pending() { return 0; } - virtual size_t get_threshold() { return 0; } - virtual void set_threshold(size_t) {} -}; - -template -struct PlaceRef : public PlaceRefBase { - - PlaceRef(JArrow* parent) { - assert(parent != nullptr); - parent->attach(this); - } - - PlaceRef(JArrow* parent, bool is_input, size_t min_item_count, size_t max_item_count) { - assert(parent != nullptr); - parent->attach(this); - this->is_input = is_input; - this->min_item_count = min_item_count; - this->max_item_count = max_item_count; - } - - PlaceRef(JArrow* parent, JMailbox* queue, bool is_input, size_t min_item_count, size_t max_item_count) { - assert(parent != nullptr); - assert(queue != nullptr); - parent->attach(this); - this->place_ref = queue; - this->is_queue = true; - this->is_input = is_input; - this->min_item_count = min_item_count; - this->max_item_count = max_item_count; - } - - PlaceRef(JArrow* parent, JPool* pool, bool is_input, size_t min_item_count, size_t max_item_count) { - assert(parent != nullptr); - assert(pool != nullptr); - parent->attach(this); - this->place_ref = pool; - this->is_queue = false; - this->is_input = is_input; - this->min_item_count = min_item_count; - this->max_item_count = max_item_count; - } - - void set_queue(JMailbox* queue) { - assert(queue != nullptr); - this->place_ref = queue; - this->is_queue = true; - } - - void set_pool(JPool* pool) { - assert(pool != nullptr); - this->place_ref = pool; - this->is_queue = false; - } - - size_t get_pending() override { - assert(place_ref != nullptr); - if (is_input && is_queue) { - auto queue = static_cast*>(place_ref); - return queue->size(); - } - return 0; - } - - size_t get_threshold() override { - assert(place_ref != nullptr); - if (is_input && is_queue) { - auto queue = static_cast*>(place_ref); - return queue->get_threshold(); - } - return -1; - } - - void set_threshold(size_t threshold) override { - assert(place_ref != nullptr); - if (is_input && is_queue) { - auto queue = static_cast*>(place_ref); - queue->set_threshold(threshold); - } - } + virtual void finalize() {}; - bool pull(Data& data) { - assert(place_ref != nullptr); - if (is_input) { // Actually pull the data - if (is_queue) { - auto queue = static_cast*>(place_ref); - data.item_count = queue->pop_and_reserve(data.items.data(), min_item_count, max_item_count, data.location_id); - data.reserve_count = data.item_count; - return (data.item_count >= min_item_count); - } - else { - auto pool = static_cast*>(place_ref); - data.item_count = pool->pop(data.items.data(), min_item_count, max_item_count, data.location_id); - data.reserve_count = 0; - return (data.item_count >= min_item_count); - } - } - else { - if (is_queue) { - // Reserve a space on the output queue - data.item_count = 0; - auto queue = static_cast*>(place_ref); - data.reserve_count = queue->reserve(min_item_count, max_item_count, data.location_id); - return (data.reserve_count >= min_item_count); - } - else { - // No need to reserve on pool -- either there is space or limit_events_in_flight=false - data.item_count = 0; - data.reserve_count = 0; - return true; - } - } - } + void create_ports(size_t inputs, size_t outputs); - void revert(Data& data) { - assert(place_ref != nullptr); - if (is_queue) { - auto queue = static_cast*>(place_ref); - queue->push_and_unreserve(data.items.data(), data.item_count, data.reserve_count, data.location_id); - } - else { - if (is_input) { - auto pool = static_cast*>(place_ref); - pool->push(data.items.data(), data.item_count, data.location_id); - } - } - } + void attach(JEventQueue* queue, size_t port); + void attach(JEventPool* pool, size_t port); - size_t push(Data& data) { - assert(place_ref != nullptr); - if (is_queue) { - auto queue = static_cast*>(place_ref); - queue->push_and_unreserve(data.items.data(), data.item_count, data.reserve_count, data.location_id); - data.item_count = 0; - data.reserve_count = 0; - return is_input ? 0 : data.item_count; - } - else { - auto pool = static_cast*>(place_ref); - pool->push(data.items.data(), data.item_count, data.location_id); - data.item_count = 0; - data.reserve_count = 0; - return 1; - } - } + JEvent* pull(size_t input_port, size_t location_id); + void push(OutputData& outputs, size_t output_count, size_t location_id); }; -inline size_t JArrow::get_pending() { - size_t sum = 0; - for (PlaceRefBase* place : m_places) { - sum += place->get_pending(); - } - return sum; -} - -inline size_t JArrow::get_threshold() { - size_t result = -1; - for (PlaceRefBase* place : m_places) { - result = std::min(result, place->get_threshold()); - } - return result; - -} - -inline void JArrow::set_threshold(size_t threshold) { - for (PlaceRefBase* place : m_places) { - place->set_threshold(threshold); - } -} - +std::string to_string(JArrow::FireResult r); diff --git a/src/libraries/JANA/Topology/JArrowMetrics.h b/src/libraries/JANA/Topology/JArrowMetrics.h deleted file mode 100644 index fde19d6fc..000000000 --- a/src/libraries/JANA/Topology/JArrowMetrics.h +++ /dev/null @@ -1,181 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include - -class JArrowMetrics { - -public: - enum class Status {KeepGoing, ComeBackLater, Finished, NotRunYet, Error}; - using duration_t = std::chrono::steady_clock::duration; - -private: - mutable std::mutex m_mutex; - // Mutex is mutable so that we can lock before reading from a const ref - - Status m_last_status; - size_t m_total_message_count; - size_t m_last_message_count; - size_t m_total_queue_visits; - size_t m_last_queue_visits; - duration_t m_total_latency; - duration_t m_last_latency; - duration_t m_total_queue_latency; - duration_t m_last_queue_latency; - - - // TODO: We might want to add a timestamp, so that - // the 'last_*' measurements can reflect the most recent value, - // rather than the last-to-be-accumulated value. - -public: - void clear() { - - m_mutex.lock(); - m_last_status = Status::NotRunYet; - m_total_message_count = 0; - m_last_message_count = 0; - m_total_queue_visits = 0; - m_last_queue_visits = 0; - m_total_latency = duration_t::zero(); - m_last_latency = duration_t::zero(); - m_total_queue_latency = duration_t::zero(); - m_last_queue_latency = duration_t::zero(); - m_mutex.unlock(); - } - - void take(JArrowMetrics& other) { - - m_mutex.lock(); - other.m_mutex.lock(); - - if (other.m_last_message_count != 0) { - m_last_message_count = other.m_last_message_count; - m_last_latency = other.m_last_latency; - } - - m_last_status = other.m_last_status; - m_total_message_count += other.m_total_message_count; - m_total_queue_visits += other.m_total_queue_visits; - m_last_queue_visits = other.m_last_queue_visits; - m_total_latency += other.m_total_latency; - m_total_queue_latency += other.m_total_queue_latency; - m_last_queue_latency = other.m_last_queue_latency; - - other.m_last_status = Status::NotRunYet; - other.m_total_message_count = 0; - other.m_last_message_count = 0; - other.m_total_queue_visits = 0; - other.m_last_queue_visits = 0; - other.m_total_latency = duration_t::zero(); - other.m_last_latency = duration_t::zero(); - other.m_total_queue_latency = duration_t::zero(); - other.m_last_queue_latency = duration_t::zero(); - other.m_mutex.unlock(); - m_mutex.unlock(); - }; - - void update(const JArrowMetrics &other) { - - m_mutex.lock(); - other.m_mutex.lock(); - - if (other.m_last_message_count != 0) { - m_last_message_count = other.m_last_message_count; - m_last_latency = other.m_last_latency; - } - m_total_latency += other.m_total_latency; - m_last_status = other.m_last_status; - m_total_message_count += other.m_total_message_count; - m_total_queue_visits += other.m_total_queue_visits; - m_last_queue_visits = other.m_last_queue_visits; - m_total_queue_latency += other.m_total_queue_latency; - m_last_queue_latency = other.m_last_queue_latency; - other.m_mutex.unlock(); - m_mutex.unlock(); - }; - - void update_finished() { - m_mutex.lock(); - m_last_status = Status::Finished; - m_mutex.unlock(); - } - - void update(const Status& last_status, - const size_t& message_count_delta, - const size_t& queue_visit_delta, - const duration_t& latency_delta, - const duration_t& queue_latency_delta) { - - m_mutex.lock(); - m_last_status = last_status; - - if (message_count_delta > 0) { - // We don't want to lose our most recent latency numbers - // when the most recent execute() encounters an empty - // queue and consequently processes zero items. - m_last_message_count = message_count_delta; - m_last_latency = latency_delta; - } - m_total_message_count += message_count_delta; - m_total_queue_visits += queue_visit_delta; - m_last_queue_visits = queue_visit_delta; - m_total_latency += latency_delta; - m_total_queue_latency += queue_latency_delta; - m_last_queue_latency = queue_latency_delta; - m_mutex.unlock(); - - }; - - void get(Status& last_status, - size_t& total_message_count, - size_t& last_message_count, - size_t& total_queue_visits, - size_t& last_queue_visits, - duration_t& total_latency, - duration_t& last_latency, - duration_t& total_queue_latency, - duration_t& last_queue_latency) { - - m_mutex.lock(); - last_status = m_last_status; - total_message_count = m_total_message_count; - last_message_count = m_last_message_count; - total_queue_visits = m_total_queue_visits; - last_queue_visits = m_last_queue_visits; - total_latency = m_total_latency; - last_latency = m_last_latency; - total_queue_latency = m_total_queue_latency; - last_queue_latency = m_last_queue_latency; - m_mutex.unlock(); - } - - size_t get_total_message_count() { - std::lock_guard lock(m_mutex); - return m_total_message_count; - } - - Status get_last_status() { - std::lock_guard lock(m_mutex); - return m_last_status; - } - - void summarize() { - - } -}; - -inline std::string to_string(JArrowMetrics::Status h) { - switch (h) { - case JArrowMetrics::Status::KeepGoing: return "KeepGoing"; - case JArrowMetrics::Status::ComeBackLater: return "ComeBackLater"; - case JArrowMetrics::Status::Finished: return "Finished"; - case JArrowMetrics::Status::NotRunYet: return "NotRunYet"; - case JArrowMetrics::Status::Error: - default: return "Error"; - } -} - diff --git a/src/libraries/JANA/Topology/JEventMapArrow.cc b/src/libraries/JANA/Topology/JEventMapArrow.cc index 9713e3cb2..31686439a 100644 --- a/src/libraries/JANA/Topology/JEventMapArrow.cc +++ b/src/libraries/JANA/Topology/JEventMapArrow.cc @@ -10,16 +10,12 @@ #include -JEventMapArrow::JEventMapArrow(std::string name, - EventQueue *input_queue, - EventQueue *output_queue) - : JPipelineArrow(std::move(name), - true, - false, - false, - input_queue, - output_queue, - nullptr) {} +JEventMapArrow::JEventMapArrow(std::string name) { + set_name(name); + set_is_parallel(true); + create_ports(1, 1); + +} void JEventMapArrow::add_source(JEventSource* source) { m_sources.push_back(source); @@ -33,36 +29,53 @@ void JEventMapArrow::add_processor(JEventProcessor* processor) { m_procs.push_back(processor); } -void JEventMapArrow::process(Event* event, bool& success, JArrowMetrics::Status& status) { +void JEventMapArrow::fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { - LOG_DEBUG(m_logger) << "JEventMapArrow '" << get_name() << "': Starting event# " << (*event)->GetEventNumber() << LOG_END; + LOG_DEBUG(m_logger) << "Executing arrow " << get_name() << " for event# " << event->GetEventNumber() << LOG_END; for (JEventSource* source : m_sources) { - JCallGraphEntryMaker cg_entry(*(*event)->GetJCallGraphRecorder(), source->GetTypeName()); // times execution until this goes out of scope - source->Preprocess(**event); + JCallGraphEntryMaker cg_entry(*event->GetJCallGraphRecorder(), source->GetTypeName()); // times execution until this goes out of scope + source->Preprocess(*event); } for (JEventUnfolder* unfolder : m_unfolders) { - JCallGraphEntryMaker cg_entry(*(*event)->GetJCallGraphRecorder(), unfolder->GetTypeName()); // times execution until this goes out of scope - unfolder->Preprocess(**event); + JCallGraphEntryMaker cg_entry(*event->GetJCallGraphRecorder(), unfolder->GetTypeName()); // times execution until this goes out of scope + unfolder->Preprocess(*event); } for (JEventProcessor* processor : m_procs) { - JCallGraphEntryMaker cg_entry(*(*event)->GetJCallGraphRecorder(), processor->GetTypeName()); // times execution until this goes out of scope + JCallGraphEntryMaker cg_entry(*event->GetJCallGraphRecorder(), processor->GetTypeName()); // times execution until this goes out of scope if (processor->GetCallbackStyle() == JEventProcessor::CallbackStyle::LegacyMode) { - processor->DoLegacyProcess(*event); + processor->DoLegacyProcess(event->shared_from_this()); } else { processor->DoMap(*event); } } - LOG_DEBUG(m_logger) << "JEventMapArrow '" << get_name() << "': Finished event# " << (*event)->GetEventNumber() << LOG_END; - success = true; - status = JArrowMetrics::Status::KeepGoing; + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " for event# " << event->GetEventNumber() << LOG_END; + outputs[0] = {event, 1}; + output_count = 1; + status = JArrow::FireResult::KeepGoing; } void JEventMapArrow::initialize() { LOG_DEBUG(m_logger) << "Initializing arrow '" << get_name() << "'" << LOG_END; + for (auto processor : m_procs) { + if (processor->GetCallbackStyle() == JEventProcessor::CallbackStyle::LegacyMode) { + LOG_INFO(m_logger) << "Initializing JEventProcessor '" << processor->GetTypeName() << "'" << LOG_END; + processor->DoInitialize(); + LOG_INFO(m_logger) << "Initialized JEventProcessor '" << processor->GetTypeName() << "'" << LOG_END; + } + } + LOG_DEBUG(m_logger) << "Initialized arrow '" << get_name() << "'" << LOG_END; } void JEventMapArrow::finalize() { LOG_DEBUG(m_logger) << "Finalizing arrow '" << get_name() << "'" << LOG_END; + for (auto processor : m_procs) { + if (processor->GetCallbackStyle() == JEventProcessor::CallbackStyle::LegacyMode) { + LOG_DEBUG(m_logger) << "Finalizing JEventProcessor " << processor->GetTypeName() << LOG_END; + processor->DoFinalize(); + LOG_INFO(m_logger) << "Finalized JEventProcessor " << processor->GetTypeName() << LOG_END; + } + } + LOG_DEBUG(m_logger) << "Finalized arrow " << get_name() << LOG_END; } diff --git a/src/libraries/JANA/Topology/JEventMapArrow.h b/src/libraries/JANA/Topology/JEventMapArrow.h index a0d521bb8..5eb3f0c39 100644 --- a/src/libraries/JANA/Topology/JEventMapArrow.h +++ b/src/libraries/JANA/Topology/JEventMapArrow.h @@ -3,7 +3,7 @@ #pragma once -#include +#include class JEventPool; class JEventSource; @@ -11,10 +11,11 @@ class JEventUnfolder; class JEventProcessor; class JEvent; -using Event = std::shared_ptr; -using EventQueue = JMailbox; -class JEventMapArrow : public JPipelineArrow { +class JEventMapArrow : public JArrow { + +public: + enum PortIndex {EVENT_IN=0, EVENT_OUT=1}; private: std::vector m_sources; @@ -22,13 +23,13 @@ class JEventMapArrow : public JPipelineArrow { std::vector m_procs; public: - JEventMapArrow(std::string name, EventQueue *input_queue, EventQueue *output_queue); + JEventMapArrow(std::string name); void add_source(JEventSource* source); void add_unfolder(JEventUnfolder* unfolder); void add_processor(JEventProcessor* proc); - void process(Event* event, bool& success, JArrowMetrics::Status& status); + void fire(JEvent* input, OutputData& outputs, size_t& output_count, JArrow::FireResult& status); void initialize() final; void finalize() final; diff --git a/src/libraries/JANA/Topology/JEventPool.h b/src/libraries/JANA/Topology/JEventPool.h new file mode 100644 index 000000000..58c9e3bac --- /dev/null +++ b/src/libraries/JANA/Topology/JEventPool.h @@ -0,0 +1,82 @@ +// Copyright 2024, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. +// Author: Nathan Brei + +#pragma once + +#include +#include +#include + + +class JEventPool : public JEventQueue { +private: + std::vector> m_owned_events; + std::shared_ptr m_component_manager; + JEventLevel m_level; + JLogger m_logger; + +public: + inline JEventPool(std::shared_ptr component_manager, + size_t max_inflight_events, + size_t location_count, + JEventLevel level = JEventLevel::PhysicsEvent) + + : JEventQueue(max_inflight_events, location_count) + , m_component_manager(component_manager) + , m_level(level) { + + // Create JEvents and distribute them among the pools. + m_owned_events.reserve(max_inflight_events); + for (size_t evt_idx=0; evt_idx()); + auto evt = &m_owned_events.back(); + m_component_manager->configure_event(**evt); + (*evt)->SetLevel(m_level); + Push(evt->get(), evt_idx % location_count); + } + } + + void Scale(size_t capacity) override { + auto old_capacity = m_capacity; + if (capacity < old_capacity) { + // Downscaling a JEventPool is tricky because even draining the topology + // doesn't guarantee that all JEvents have been returned to the JEventPools by + // the time the topology pauses. Consider JEventUnfolder, which may very well hold + // on to a child event that it won't need because the parent event source has + // paused or finished. + // For now we don't reduce the size of the pools or queues because there's little + // penalty to keeping them around (the main penalty comes from event creation time + // and memory footprint, but if we are downscaling we already paid that) + return; + } + + // Resize queues to fit new capacity + m_capacity = capacity; + for (auto& local_queue: m_local_queues) { + local_queue->ringbuffer.resize(capacity, nullptr); + local_queue->capacity = capacity; + } + + // Create new JEvents, add to owned_events, and distribute to queues + m_owned_events.reserve(capacity); + + for (size_t evt_idx=old_capacity; evt_idx()); + auto evt = &m_owned_events.back(); + m_component_manager->configure_event(**evt); + (*evt)->SetLevel(m_level); + Push(evt->get(), evt_idx % GetLocationCount()); + } + } + + void Finalize() { + for (auto& evt : m_owned_events) { + evt->Finish(); + } + } + +}; + + diff --git a/src/libraries/JANA/Topology/JEventProcessorArrow.cc b/src/libraries/JANA/Topology/JEventProcessorArrow.cc deleted file mode 100644 index 5ff2a2361..000000000 --- a/src/libraries/JANA/Topology/JEventProcessorArrow.cc +++ /dev/null @@ -1,64 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#include -#include -#include -#include - - -JEventProcessorArrow::JEventProcessorArrow(std::string name, - EventQueue *input_queue, - EventQueue *output_queue, - JEventPool *pool) - : JPipelineArrow(std::move(name), - true, - false, - true, - input_queue, - output_queue, - pool) {} - -void JEventProcessorArrow::add_processor(JEventProcessor* processor) { - m_processors.push_back(processor); -} - -void JEventProcessorArrow::process(Event* event, bool& success, JArrowMetrics::Status& status) { - - - LOG_DEBUG(m_logger) << "JEventProcessorArrow '" << get_name() << "': Starting event# " << (*event)->GetEventNumber() << LOG_END; - for (JEventProcessor* processor : m_processors) { - // TODO: Move me into JEventProcessor::DoMap - JCallGraphEntryMaker cg_entry(*(*event)->GetJCallGraphRecorder(), processor->GetTypeName()); // times execution until this goes out of scope - if (processor->GetCallbackStyle() == JEventProcessor::CallbackStyle::LegacyMode) { - processor->DoLegacyProcess(*event); - } - else { - processor->DoMap(*event); - processor->DoTap(*event); - - } - } - LOG_DEBUG(m_logger) << "JEventProcessorArrow '" << get_name() << "': Finished event# " << (*event)->GetEventNumber() << LOG_END; - success = true; - status = JArrowMetrics::Status::KeepGoing; -} - -void JEventProcessorArrow::initialize() { - LOG_DEBUG(m_logger) << "Initializing arrow '" << get_name() << "'" << LOG_END; - for (auto processor : m_processors) { - processor->DoInitialize(); - LOG_INFO(m_logger) << "Initialized JEventProcessor '" << processor->GetTypeName() << "'" << LOG_END; - } -} - -void JEventProcessorArrow::finalize() { - LOG_DEBUG(m_logger) << "Finalizing arrow '" << get_name() << "'" << LOG_END; - for (auto processor : m_processors) { - processor->DoFinalize(); - LOG_INFO(m_logger) << "Finalized JEventProcessor '" << processor->GetTypeName() << "'" << LOG_END; - } -} - diff --git a/src/libraries/JANA/Topology/JEventProcessorArrow.h b/src/libraries/JANA/Topology/JEventProcessorArrow.h deleted file mode 100644 index 21b2f94b9..000000000 --- a/src/libraries/JANA/Topology/JEventProcessorArrow.h +++ /dev/null @@ -1,32 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include - -class JEventPool; - -using Event = std::shared_ptr; -using EventQueue = JMailbox; - -class JEventProcessorArrow : public JPipelineArrow { - -private: - std::vector m_processors; - -public: - JEventProcessorArrow(std::string name, - EventQueue *input_queue, - EventQueue *output_queue, - JEventPool *pool); - - void add_processor(JEventProcessor* processor); - - void process(Event* event, bool& success, JArrowMetrics::Status& status); - - void initialize() final; - void finalize() final; -}; - diff --git a/src/libraries/JANA/Topology/JEventQueue.h b/src/libraries/JANA/Topology/JEventQueue.h new file mode 100644 index 000000000..1211527fe --- /dev/null +++ b/src/libraries/JANA/Topology/JEventQueue.h @@ -0,0 +1,112 @@ + +// Copyright 2024, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. +// Author: Nathan Brei + +#pragma once +#include +#include + +#include +#include +#include + +// JEventQueue is a queue designed for communication between JArrows, with the following features: +// +// - Fixed-capacity during processing, since the max number of events in flight is predetermined and small +// - Ringbuffer-based, to avoid allocations during processing +// - Adjustable-capacity outside processing, since the max number of events in flight may change when the threadcount does +// - Locality-aware, so that events that live in one location (e.g. NUMA domain) can stay within that location +// - NOT thread-safe, because all queue accesses are protected by the JExecutionEngine mutex. +// +// - Previous versions of JEventQueue have allowed work stealing (taking events out of a different +// NUMA domain when none are otherwise available). The current version does not, though we may +// bring this feature back later. +// +/// To handle memory locality at different granularities, we introduce the concept of a location. +/// Each thread belongs to exactly one location, represented by contiguous unsigned +/// ints starting at 0. While JArrows are wired to one logical JEventQueue, threads interact with +/// the physical LocalQueue corresponding to their location. Locations prevent events from crossing +/// NUMA domains as they get picked up by different worker threads. + +class JEventQueue { + +protected: + struct alignas(JANA2_CACHE_LINE_BYTES) LocalQueue { + std::vector ringbuffer; + size_t size=0; + size_t capacity=0; + size_t front=0; + size_t back=0; + }; + + std::vector> m_local_queues; + size_t m_capacity; + +public: + inline JEventQueue(size_t initial_capacity, size_t locations_count) { + + assert(locations_count >= 1); + for (size_t location=0; location()); + } + m_capacity = initial_capacity; + Scale(initial_capacity); + } + + virtual ~JEventQueue() = default; + + virtual void Scale(size_t capacity) { + if (capacity < m_capacity) { + for (auto& local_queue : m_local_queues) { + if (local_queue->size != 0) { + throw JException("Attempted to shrink a non-empty JEventQueue. Please drain the topology before attempting to downscale."); + } + } + } + m_capacity = capacity; + for (auto& local_queue: m_local_queues) { + local_queue->ringbuffer.resize(capacity, nullptr); + local_queue->capacity = capacity; + } + } + + inline size_t GetLocationCount() { + return m_local_queues.size(); + } + + inline size_t GetSize(size_t location) { + auto& local_queue = m_local_queues[location]; + return local_queue->size; + } + + inline size_t GetCapacity() { + return m_capacity; + } + + inline void Push(JEvent* event, size_t location) { + auto& local_queue = *m_local_queues[location]; + if (local_queue.size == local_queue.capacity) { + throw JException("Attempted to push to a full JEventQueue. This probably means there is an error in your topology wiring"); + } + + local_queue.ringbuffer[local_queue.front] = event; + local_queue.front = (local_queue.front + 1) % local_queue.capacity; + local_queue.size += 1; + } + + inline JEvent* Pop(size_t location) { + auto& local_queue= *m_local_queues[location]; + if (local_queue.size == 0) { + return nullptr; + } + JEvent* result = local_queue.ringbuffer[local_queue.back]; + local_queue.ringbuffer[local_queue.back] = nullptr; + local_queue.back = (local_queue.back + 1) % local_queue.capacity; + local_queue.size -= 1; + return result; + }; + +}; + + diff --git a/src/libraries/JANA/Topology/JEventSourceArrow.cc b/src/libraries/JANA/Topology/JEventSourceArrow.cc index 98d98153a..69b24fbe9 100644 --- a/src/libraries/JANA/Topology/JEventSourceArrow.cc +++ b/src/libraries/JANA/Topology/JEventSourceArrow.cc @@ -6,50 +6,123 @@ #include #include #include -#include -JEventSourceArrow::JEventSourceArrow(std::string name, - std::vector sources, - EventQueue* output_queue, - JEventPool* pool - ) - : JPipelineArrow(name, false, true, false, nullptr, output_queue, pool), m_sources(sources) { +JEventSourceArrow::JEventSourceArrow(std::string name, std::vector sources) + : m_sources(sources) { + set_name(name); + set_is_source(true); + create_ports(1, 1); } -void JEventSourceArrow::process(Event* event, bool& success, JArrowMetrics::Status& arrow_status) { +void JEventSourceArrow::fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { - // If there are no sources available then we are automatically finished. - if (m_sources.empty()) { - success = false; - arrow_status = JArrowMetrics::Status::Finished; - return; + LOG_DEBUG(m_logger) << "Executing arrow " << get_name() << LOG_END; + + // First check to see if we need to handle a barrier event before attempting to emit another event + if (m_barrier_active) { + // A barrier event has been emitted by the source. + if (m_pending_barrier_event != nullptr) { + // This barrier event is pending until the topology drains + if (m_sources[m_current_source]->GetEmittedEventCount() - + m_sources[m_current_source]->GetFinishedEventCount() == 1) { + LOG_DEBUG(m_logger) << "JEventSourceArrow: Barrier event is in-flight" << LOG_END; + + // Topology has drained; only remaining in-flight event is the barrier event itself, + // which we have held on to until now + outputs[0] = {m_pending_barrier_event, 1}; + output_count = 1; + status = JArrow::FireResult::KeepGoing; + + m_pending_barrier_event = nullptr; + // There's not much for the thread team to do while the barrier event makes its way through the topology. + // Eventually we might be able to use this to communicate to the scheduler to not wake threads whose only + // available action is to hammer the JEventSourceArrow + return; + } + else { + // Topology has _not_ finished draining, all we can do is wait + LOG_DEBUG(m_logger) << "JEventSourceArrow: Waiting on pending barrier event" << LOG_END; + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " with result ComeBackLater"<< LOG_END; + + assert(event == nullptr); + output_count = 0; + status = JArrow::FireResult::ComeBackLater; + return; + } + } + else { + // This barrier event has already been sent into the topology and we need to wait + // until it is finished before emitting any more events + if (m_sources[m_current_source]->GetFinishedEventCount() == + m_sources[m_current_source]->GetEmittedEventCount()) { + + // Barrier event has finished. + LOG_DEBUG(m_logger) << "JEventSourceArrow: Barrier event finished, returning to normal operation" << LOG_END; + m_barrier_active = false; + m_next_input_port = 0; + + output_count = 0; + status = JArrow::FireResult::KeepGoing; + return; + } + else { + // Barrier event has NOT finished + LOG_DEBUG(m_logger) << "JEventSourceArrow: Waiting on in-flight barrier event" << LOG_END; + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " with result ComeBackLater"<< LOG_END; + assert(event == nullptr); + output_count = 0; + status = JArrow::FireResult::ComeBackLater; + return; + } + } } while (m_current_source < m_sources.size()) { - auto source_status = m_sources[m_current_source]->DoNext(*event); + auto source_status = m_sources[m_current_source]->DoNext(event->shared_from_this()); if (source_status == JEventSource::Result::FailureFinished) { + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " with result FailureFinished"<< LOG_END; m_current_source++; // TODO: Adjust nskip and nevents for the new source } else if (source_status == JEventSource::Result::FailureTryAgain){ // This JEventSource isn't finished yet, so we obtained either Success or TryAgainLater - success = false; - arrow_status = JArrowMetrics::Status::ComeBackLater; + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " with result ComeBackLater"<< LOG_END; + outputs[0] = {event, 0}; // Reject + output_count = 1; + status = JArrow::FireResult::ComeBackLater; + return; + } + else if (event->GetSequential()){ + // Source succeeded, but returned a barrier event + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " with result Success, holding back barrier event# " << event->GetEventNumber() << LOG_END; + m_pending_barrier_event = event; + m_barrier_active = true; + m_next_input_port = -1; // Stop popping events from the input queue until barrier event has finished + + // Arrow hangs on to the barrier event until the topology fully drains + output_count = 0; + status = JArrow::FireResult::KeepGoing; // Mysteriously livelocks if we set this to ComeBackLater?? return; } else { - success = true; - arrow_status = JArrowMetrics::Status::KeepGoing; + // Source succeeded, did NOT return a barrier event + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " with result Success, emitting event# " << event->GetEventNumber() << LOG_END; + outputs[0] = {event, 1}; // SUCCESS! + output_count = 1; + status = JArrow::FireResult::KeepGoing; return; } } - success = false; - arrow_status = JArrowMetrics::Status::Finished; + + // All event sources have finished now + outputs[0] = {event, 0}; // Reject event + output_count = 1; + status = JArrow::FireResult::Finished; } void JEventSourceArrow::initialize() { diff --git a/src/libraries/JANA/Topology/JEventSourceArrow.h b/src/libraries/JANA/Topology/JEventSourceArrow.h index fafb0d8b2..c33c784f0 100644 --- a/src/libraries/JANA/Topology/JEventSourceArrow.h +++ b/src/libraries/JANA/Topology/JEventSourceArrow.h @@ -3,22 +3,24 @@ // Subject to the terms in the LICENSE file found in the top-level directory. #pragma once -#include +#include -using Event = std::shared_ptr; -using EventQueue = JMailbox; -class JEventPool; -class JEventSourceArrow : public JPipelineArrow { +class JEventSourceArrow : public JArrow { +public: + enum PortIndex {EVENT_IN=0, EVENT_OUT=1}; + private: std::vector m_sources; size_t m_current_source = 0; + bool m_barrier_active = false; + JEvent* m_pending_barrier_event = nullptr; public: - JEventSourceArrow(std::string name, std::vector sources, EventQueue* output_queue, JEventPool* pool); + JEventSourceArrow(std::string name, std::vector sources); + void initialize() final; void finalize() final; - - void process(Event* event, bool& success, JArrowMetrics::Status& status); + void fire(JEvent* input, OutputData& outputs, size_t& output_count, JArrow::FireResult& status); }; diff --git a/src/libraries/JANA/Topology/JEventTapArrow.cc b/src/libraries/JANA/Topology/JEventTapArrow.cc index ee91db984..a71565a4e 100644 --- a/src/libraries/JANA/Topology/JEventTapArrow.cc +++ b/src/libraries/JANA/Topology/JEventTapArrow.cc @@ -3,40 +3,33 @@ #include -#include #include #include #include -JEventTapArrow::JEventTapArrow(std::string name, - EventQueue *input_queue, - EventQueue *output_queue, - JEventPool *pool) - : JPipelineArrow(std::move(name), - false, - false, - false, - input_queue, - output_queue, - pool) {} +JEventTapArrow::JEventTapArrow(std::string name) { + set_name(name); + create_ports(1,1); +} void JEventTapArrow::add_processor(JEventProcessor* proc) { m_procs.push_back(proc); } -void JEventTapArrow::process(Event* event, bool& success, JArrowMetrics::Status& status) { +void JEventTapArrow::fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { - LOG_DEBUG(m_logger) << "JEventTapArrow '" << get_name() << "': Starting event# " << (*event)->GetEventNumber() << LOG_END; + LOG_DEBUG(m_logger) << "Executing arrow " << get_name() << " for event# " << event->GetEventNumber() << LOG_END; for (JEventProcessor* proc : m_procs) { - JCallGraphEntryMaker cg_entry(*(*event)->GetJCallGraphRecorder(), proc->GetTypeName()); // times execution until this goes out of scope + JCallGraphEntryMaker cg_entry(*event->GetJCallGraphRecorder(), proc->GetTypeName()); // times execution until this goes out of scope if (proc->GetCallbackStyle() != JEventProcessor::CallbackStyle::LegacyMode) { proc->DoTap(*event); } } - LOG_DEBUG(m_logger) << "JEventTapArrow '" << get_name() << "': Finished event# " << (*event)->GetEventNumber() << LOG_END; - success = true; - status = JArrowMetrics::Status::KeepGoing; + outputs[0] = {event, 1}; + output_count = 1; + status = JArrow::FireResult::KeepGoing; + LOG_DEBUG(m_logger) << "Executed arrow " << get_name() << " for event# " << event->GetEventNumber() << LOG_END; } void JEventTapArrow::initialize() { diff --git a/src/libraries/JANA/Topology/JEventTapArrow.h b/src/libraries/JANA/Topology/JEventTapArrow.h index ad862411e..1a3975bc1 100644 --- a/src/libraries/JANA/Topology/JEventTapArrow.h +++ b/src/libraries/JANA/Topology/JEventTapArrow.h @@ -3,25 +3,25 @@ #pragma once -#include +#include -class JEventPool; class JEventProcessor; class JEvent; -using Event = std::shared_ptr; -using EventQueue = JMailbox; -class JEventTapArrow : public JPipelineArrow { +class JEventTapArrow : public JArrow { +public: + enum PortIndex {EVENT_IN=0, EVENT_OUT=1}; private: std::vector m_procs; public: - JEventTapArrow(std::string name, EventQueue *input_queue, EventQueue *output_queue, JEventPool *pool); + JEventTapArrow(std::string name); void add_processor(JEventProcessor* proc); - void process(Event* event, bool& success, JArrowMetrics::Status& status); + + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) final; void initialize() final; void finalize() final; }; diff --git a/src/libraries/JANA/Topology/JFoldArrow.h b/src/libraries/JANA/Topology/JFoldArrow.h index 8c1b027c1..c353277c3 100644 --- a/src/libraries/JANA/Topology/JFoldArrow.h +++ b/src/libraries/JANA/Topology/JFoldArrow.h @@ -3,206 +3,85 @@ #pragma once +#include #include -#include -#include class JFoldArrow : public JArrow { +public: + enum PortIndex {CHILD_IN=0, CHILD_OUT=1, PARENT_OUT=2}; + private: - using EventT = std::shared_ptr; + JEventFolder* m_folder = nullptr; - // TODO: Support user-provided folders - // JEventFolder* m_folder = nullptr; - JEventLevel m_parent_level; JEventLevel m_child_level; - PlaceRef m_child_in; - PlaceRef m_child_out; - PlaceRef m_parent_out; - public: JFoldArrow( std::string name, - //JEventFolder* folder, - JEventLevel parent_level, - JEventLevel child_level, - JMailbox* child_in, - JEventPool* child_out, - JMailbox* parent_out) - - : JArrow(std::move(name), false, false, false), - // m_folder(folder), - m_parent_level(parent_level), - m_child_level(child_level), - m_child_in(this, child_in, true, 1, 1), - m_child_out(this, child_out, false, 1, 1), - m_parent_out(this, parent_out, false, 1, 1) - { - } - - JFoldArrow( - std::string name, - //JEventFolder* folder, JEventLevel parent_level, - JEventLevel child_level, - JMailbox* child_in, - JMailbox* child_out, - JMailbox* parent_out) - - : JArrow(std::move(name), false, false, false), - // m_folder(folder), - m_parent_level(parent_level), - m_child_level(child_level), - m_child_in(this, child_in, true, 1, 1), - m_child_out(this, child_out, false, 1, 1), - m_parent_out(this, parent_out, false, 1, 1) - { - } + JEventLevel child_level) - JFoldArrow( - std::string name, - //JEventFolder* folder, - JEventLevel parent_level, - JEventLevel child_level, - JMailbox* child_in, - JEventPool* child_out, - JEventPool* parent_out) - - : JArrow(std::move(name), false, false, false), - // m_folder(folder), - m_parent_level(parent_level), - m_child_level(child_level), - m_child_in(this, child_in, true, 1, 1), - m_child_out(this, child_out, false, 1, 1), - m_parent_out(this, parent_out, false, 1, 1) + : m_parent_level(parent_level), + m_child_level(child_level) { + set_name(name); + create_ports(1, 2); + m_next_input_port = CHILD_IN; } - void attach_child_in(JMailbox* child_in) { - m_child_in.place_ref = child_in; - m_child_in.is_queue = true; - } - - void attach_child_out(JMailbox* child_out) { - m_child_out.place_ref = child_out; - m_child_out.is_queue = true; - } - - void attach_child_out(JEventPool* child_out) { - m_child_out.place_ref = child_out; - m_child_out.is_queue = false; + void set_folder(JEventFolder* folder) { + m_folder = folder; } - void attach_parent_out(JEventPool* parent_out) { - m_parent_out.place_ref = parent_out; - m_parent_out.is_queue = false; - } - - - void attach_parent_out(JMailbox* parent_out) { - m_parent_out.place_ref = parent_out; - m_parent_out.is_queue = true; - } - - void initialize() final { - /* if (m_folder != nullptr) { m_folder->DoInit(); - LOG_INFO(m_logger) << "Initialized JEventFolder '" << m_unfolder->GetTypeName() << "'" << LOG_END; + LOG_INFO(m_logger) << "Initialized JEventFolder '" << m_folder->GetTypeName() << "'" << LOG_END; } else { + LOG_INFO(m_logger) << "Initialized JEventFolder (trivial)" << LOG_END; } - */ - LOG_INFO(m_logger) << "Initialized JEventFolder (trivial)" << LOG_END; } void finalize() final { - /* if (m_folder != nullptr) { m_folder->DoFinish(); - LOG_INFO(m_logger) << "Finalized JEventFolder '" << m_unfolder->GetTypeName() << "'" << LOG_END; + LOG_INFO(m_logger) << "Finalized JEventFolder '" << m_folder->GetTypeName() << "'" << LOG_END; + } + else { + LOG_INFO(m_logger) << "Finalized JEventFolder (trivial)" << LOG_END; } - */ - LOG_INFO(m_logger) << "Finalized JEventFolder (trivial)" << LOG_END; } - bool try_pull_all(Data& ci, Data& co, Data& po) { - bool success; - success = m_child_in.pull(ci); - if (! success) { - return false; - } - success = m_child_out.pull(co); - if (! success) { - return false; + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) final { + + assert(m_next_input_port == CHILD_IN); + + // Check that child is at the correct event level + if (event->GetLevel() != m_child_level) { + throw JException("JFoldArrow received a child with the wrong event level"); } - success = m_parent_out.pull(po); - if (! success) { - return false; + + if (m_folder != nullptr) { + auto parent = const_cast(&event->GetParent(m_parent_level)); + m_folder->DoFold(*event, *parent); } - return true; - } - size_t push_all(Data& ci, Data& co, Data& po) { - size_t message_count = co.item_count; - m_child_in.push(ci); - m_child_out.push(co); - m_parent_out.push(po); - return message_count; - } + status = JArrow::FireResult::KeepGoing; + outputs[0] = {event, CHILD_OUT}; + output_count = 1; - void execute(JArrowMetrics& metrics, size_t location_id) final { - - auto start_total_time = std::chrono::steady_clock::now(); - - Data child_in_data {location_id}; - Data child_out_data {location_id}; - Data parent_out_data {location_id}; - - bool success = try_pull_all(child_in_data, child_out_data, parent_out_data); - if (success) { - - auto start_processing_time = std::chrono::steady_clock::now(); - auto child = child_in_data.items[0]; - child_in_data.items[0] = nullptr; - child_in_data.item_count = 0; - if (child->get()->GetLevel() != m_child_level) { - throw JException("JFoldArrow received a child with the wrong event level"); - } - - // TODO: Call folders here - auto* parent = child->get()->ReleaseParent(m_parent_level); - - // Put child on the output queue - child_out_data.items[0] = child; - child_out_data.item_count = 1; - - // Only recycle the parent once the reference count hits zero - if (parent != nullptr) { - parent_out_data.items[0] = parent; - parent_out_data.item_count = 1; - } - else { - parent_out_data.items[0] = nullptr; - parent_out_data.item_count = 0; - } - - auto end_processing_time = std::chrono::steady_clock::now(); - size_t events_processed = push_all(child_in_data, child_out_data, parent_out_data); - - auto end_total_time = std::chrono::steady_clock::now(); - auto latency = (end_processing_time - start_processing_time); - auto overhead = (end_total_time - start_total_time) - latency; - - metrics.update(JArrowMetrics::Status::KeepGoing, events_processed, 1, latency, overhead); - return; - } - else { - auto end_total_time = std::chrono::steady_clock::now(); - metrics.update(JArrowMetrics::Status::ComeBackLater, 0, 1, std::chrono::milliseconds(0), end_total_time - start_total_time); - return; + auto* released_parent = event->ReleaseParent(m_parent_level); + if (released_parent != nullptr) { + // JEvent::ReleaseParent() returns nullptr if there are remaining references + // to the parent event. If non-null, we are completely done with the parent + // and are free to return it to the pool. In the future we could have the pool + // itself handle the logic for releasing parents, in which case we could avoid + // trivial JEventFolders. + + outputs[1] = {released_parent, PARENT_OUT}; + output_count = 2; } } diff --git a/src/libraries/JANA/Topology/JJunctionArrow.h b/src/libraries/JANA/Topology/JJunctionArrow.h deleted file mode 100644 index fffd02893..000000000 --- a/src/libraries/JANA/Topology/JJunctionArrow.h +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2023, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once - -#include -#include -#include - - - -template -class JJunctionArrow : public JArrow { - -protected: - PlaceRef first_input {this}; - PlaceRef first_output {this}; - PlaceRef second_input {this}; - PlaceRef second_output {this}; - -public: - using Status = JArrowMetrics::Status; - - JJunctionArrow(std::string name, - bool is_parallel, - bool is_source, - bool is_sink - ) - : JArrow(std::move(name), is_parallel, is_source, is_sink) - { - } - - bool try_pull_all(Data& fi, Data& fo, Data& si, Data& so) { - - bool success; - success = first_input.pull(fi); - if (! success) { - return false; - } - success = first_output.pull(fo); - if (! success) { - first_input.revert(fi); - return false; - } - success = second_input.pull(si); - if (! success) { - first_input.revert(fi); - first_output.revert(fo); - return false; - } - success = second_output.pull(so); - if (! success) { - first_input.revert(fi); - first_output.revert(fo); - second_input.revert(si); - return false; - } - return true; - } - - size_t push_all(Data& fi, Data& fo, Data& si, Data& so) { - size_t message_count = 0; - message_count += first_input.push(fi); - message_count += first_output.push(fo); - message_count += second_input.push(si); - message_count += second_output.push(so); - return message_count; - } - - void execute(JArrowMetrics& result, size_t location_id) final { - - auto start_total_time = std::chrono::steady_clock::now(); - - Data first_input_data {location_id}; - Data first_output_data {location_id}; - Data second_input_data {location_id}; - Data second_output_data {location_id}; - - bool success = try_pull_all(first_input_data, first_output_data, second_input_data, second_output_data); - if (success) { - - auto start_processing_time = std::chrono::steady_clock::now(); - auto process_status = static_cast(this)->process(first_input_data, first_output_data, second_input_data, second_output_data); - auto end_processing_time = std::chrono::steady_clock::now(); - size_t events_processed = push_all(first_input_data, first_output_data, second_input_data, second_output_data); - - auto end_total_time = std::chrono::steady_clock::now(); - auto latency = (end_processing_time - start_processing_time); - auto overhead = (end_total_time - start_total_time) - latency; - result.update(process_status, events_processed, 1, latency, overhead); - return; - } - else { - auto end_total_time = std::chrono::steady_clock::now(); - result.update(JArrowMetrics::Status::ComeBackLater, 0, 1, std::chrono::milliseconds(0), end_total_time - start_total_time); - return; - } - } -}; - - diff --git a/src/libraries/JANA/Topology/JMailbox.h b/src/libraries/JANA/Topology/JMailbox.h deleted file mode 100644 index ef5256c14..000000000 --- a/src/libraries/JANA/Topology/JMailbox.h +++ /dev/null @@ -1,327 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include -#include -#include -#include - -/// JMailbox is a threadsafe event queue designed for communication between Arrows. -/// It is different from the standard data structure in the following ways: -/// - pushes and pops return a Status enum, handling the problem of .size() always being stale -/// - pops may fail but pushes may not -/// - pushes and pops are chunked, reducing contention and handling the failure case cleanly -/// - when the .reserve() method is used, the queue size is bounded -/// - the underlying queue may be shared by all threads, NUMA-domain-local, or thread-local -/// - the Arrow doesn't have to know anything about locality. -/// -/// To handle memory locality at different granularities, we introduce the concept of a location. -/// Each thread belongs to exactly one location, represented by contiguous unsigned -/// ints starting at 0. While JArrows are wired to one logical JMailbox, JWorkers interact with -/// the physical LocalQueue corresponding to their location. Locations prevent events from crossing -/// NUMA domains as they get picked up by different JWorker threads. -/// -/// \tparam T must be moveable. Usually this is unique_ptr. -/// -/// Improvements: -/// 1. Pad LocalQueue -/// 2. Enable work stealing - - -class JQueue { -protected: - size_t m_capacity; - size_t m_locations_count; - bool m_enable_work_stealing = false; - int m_id = 0; - JLogger m_logger; - -public: - inline size_t get_threshold() { return m_capacity; } - inline size_t get_locations_count() { return m_locations_count; } - inline bool is_work_stealing_enabled() { return m_enable_work_stealing; } - void set_logger(JLogger logger) { m_logger = logger; } - void set_id(int id) { m_id = id; } - - - inline JQueue(size_t threshold, size_t locations_count, bool enable_work_stealing) - : m_capacity(threshold), m_locations_count(locations_count), m_enable_work_stealing(enable_work_stealing) {} - virtual ~JQueue() = default; -}; - -template -class JMailbox : public JQueue { - - struct LocalQueue { - std::mutex mutex; - std::deque queue; - size_t reserved_count = 0; - }; - - // TODO: Copy these params into DLMB for better locality - std::unique_ptr m_queues; - -public: - - enum class Status {Ready, Congested, Empty, Full}; - - friend std::ostream& operator<<(std::ostream& os, const Status& s) { - switch (s) { - case Status::Ready: os << "Ready"; break; - case Status::Congested: os << "Congested"; break; - case Status::Empty: os << "Empty"; break; - case Status::Full: os << "Full"; break; - default: os << "Unknown"; break; - } - return os; - } - - - /// threshold: the (soft) maximum number of items in the queue at any time - /// locations_count: the number of locations. More locations = better NUMA performance, worse load balancing - /// enable_work_stealing: allow events to cross locations only when no other work is available. Improves aforementioned load balancing. - JMailbox(size_t threshold=100, size_t locations_count=1, bool enable_work_stealing=false) - : JQueue(threshold, locations_count, enable_work_stealing) { - - m_queues = std::unique_ptr(new LocalQueue[locations_count]); - } - - virtual ~JMailbox() { - //delete [] m_queues; - } - - // We can do this (for now) because we use a deque underneath, so threshold is 'soft' - inline void set_threshold(size_t threshold) { m_capacity = threshold; } - - /// size() counts the number of items in the queue across all locations - /// This should be used sparingly because it will mess up a bunch of caches. - /// Meant to be used by measure_perf() - size_t size() { - size_t result = 0; - for (size_t i = 0; i lock(m_queues[i].mutex); - result += m_queues[i].queue.size(); - } - return result; - }; - - /// size(location_id) counts the number of items in the queue for a particular location - /// Meant to be used by Scheduler::next_assignment() and measure_perf(), eventually - size_t size(size_t location_id) { - return m_queues[location_id].queue.size(); - } - - /// reserve(requested_count) keeps our queues bounded in size. The caller should - /// reserve their desired chunk size on the output queue first. The output - /// queue will return a reservation which is less than or equal to requested_count. - /// The caller may then request as many items from the input queue as have been - /// reserved on the output queue. Note that because the input queue may return - /// fewer items than requested, the caller must push their original reserved_count - /// alongside the items, to avoid a "reservation leak". - size_t reserve(size_t requested_count, size_t location_id = 0) { - - LocalQueue& mb = m_queues[location_id]; - std::lock_guard lock(mb.mutex); - size_t doable_count = m_capacity - mb.queue.size() - mb.reserved_count; - if (doable_count > 0) { - size_t reservation = std::min(doable_count, requested_count); - mb.reserved_count += reservation; - return reservation; - } - return 0; - }; - - /// push(items, reserved_count, location_id) This function will always - /// succeed, although it may exceed the threshold if the caller didn't reserve - /// space, and it may take a long time because it will wait on a mutex. - /// Note that if the caller had called reserve(), they must pass in the reserved_count here. - Status push(std::vector& buffer, size_t reserved_count = 0, size_t location_id = 0) { - - auto& mb = m_queues[location_id]; - std::lock_guard lock(mb.mutex); - mb.reserved_count -= reserved_count; - for (const T& t : buffer) { - mb.queue.push_back(std::move(t)); - } - buffer.clear(); - if (mb.queue.size() > m_capacity) { - return Status::Full; - } - return Status::Ready; - } - - - /// pop() will pop up to requested_count items for the desired location_id. - /// If many threads are contending for the queue, this will fail with Status::Contention, - /// in which case the caller should probably consult the Scheduler. - Status pop(std::vector& buffer, size_t requested_count, size_t location_id = 0) { - - auto& mb = m_queues[location_id]; - if (!mb.mutex.try_lock()) { - return Status::Congested; - } - auto nitems = std::min(requested_count, mb.queue.size()); - buffer.reserve(nitems); - for (size_t i=0; i= m_capacity) { - return Status::Full; - } - else if (size != 0) { - return Status::Ready; - } - return Status::Empty; - } - - - Status pop(T& item, bool& success, size_t location_id = 0) { - - success = false; - auto& mb = m_queues[location_id]; - if (!mb.mutex.try_lock()) { - return Status::Congested; - } - size_t nitems = mb.queue.size(); - if (nitems > 1) { - item = std::move(mb.queue.front()); - mb.queue.pop_front(); - success = true; - mb.mutex.unlock(); - return Status::Ready; - } - else if (nitems == 1) { - item = std::move(mb.queue.front()); - mb.queue.pop_front(); - success = true; - mb.mutex.unlock(); - return Status::Empty; - } - mb.mutex.unlock(); - return Status::Empty; - } - - - - - bool try_push(T* buffer, size_t count, size_t location_id = 0) { - auto& mb = m_queues[location_id]; - std::lock_guard lock(mb.mutex); - if (mb.queue.size() + count > m_capacity) return false; - for (size_t i=0; i lock(mb.mutex); - assert(reserved_count <= mb.reserved_count); - assert(mb.queue.size() + count <= m_capacity); - mb.reserved_count -= reserved_count; - for (size_t i=0; i lock(mb.mutex); - - if (mb.queue.size() < min_requested_count) return 0; - - auto nitems = std::min(max_requested_count, mb.queue.size()); - - for (size_t i=0; i lock(mb.mutex); - - if (mb.queue.size() < min_requested_count) return 0; - - auto nitems = std::min(max_requested_count, mb.queue.size()); - mb.reserved_count += nitems; - - for (size_t i=0; i lock(mb.mutex); - size_t available_count = m_capacity - mb.queue.size() - mb.reserved_count; - size_t count = std::min(available_count, max_requested_count); - if (count < min_requested_count) { - return 0; - } - mb.reserved_count += count; - return count; - }; - - void unreserve(size_t reserved_count, size_t location_id) { - - LocalQueue& mb = m_queues[location_id]; - std::lock_guard lock(mb.mutex); - assert(reserved_count <= mb.reserved_count); - mb.reserved_count -= reserved_count; - }; - -}; - -template <> -inline void JMailbox*>::push_and_unreserve(std::shared_ptr** buffer, size_t count, size_t reserved_count, size_t location_id) { - - auto& mb = m_queues[location_id]; - std::lock_guard lock(mb.mutex); - assert(reserved_count <= mb.reserved_count); - assert(mb.queue.size() + count <= m_capacity); - mb.reserved_count -= reserved_count; - for (size_t i=0; iget()->GetEventNumber() << LOG_END; - mb.queue.push_back(buffer[i]); - buffer[i] = nullptr; - } -} - -template <> -inline size_t JMailbox*>::pop_and_reserve(std::shared_ptr** buffer, size_t min_requested_count, size_t max_requested_count, size_t location_id) { - - auto& mb = m_queues[location_id]; - std::lock_guard lock(mb.mutex); - - if (mb.queue.size() < min_requested_count) return 0; - - auto nitems = std::min(max_requested_count, mb.queue.size()); - mb.reserved_count += nitems; - - for (size_t i=0; iget()->GetEventNumber() << LOG_END; - mb.queue.pop_front(); - } - return nitems; -} - - diff --git a/src/libraries/JANA/Topology/JPipelineArrow.h b/src/libraries/JANA/Topology/JPipelineArrow.h deleted file mode 100644 index 1ce6f0228..000000000 --- a/src/libraries/JANA/Topology/JPipelineArrow.h +++ /dev/null @@ -1,85 +0,0 @@ - -// Copyright 2023, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once - -#include -#include -#include - -template -class JPipelineArrow : public JArrow { -private: - PlaceRef m_input {this, true, 1, 1}; - PlaceRef m_output {this, false, 1, 1}; - -public: - JPipelineArrow(std::string name, - bool is_parallel, - bool is_source, - bool is_sink, - JMailbox* input_queue, - JMailbox* output_queue, - JPool* pool - ) - : JArrow(std::move(name), is_parallel, is_source, is_sink) { - - if (input_queue == nullptr) { - assert(pool != nullptr); - m_input.set_pool(pool); - } - else { - m_input.set_queue(input_queue); - } - if (output_queue == nullptr) { - assert(pool != nullptr); - m_output.set_pool(pool); - } - else { - m_output.set_queue(output_queue); - } - } - - void execute(JArrowMetrics& result, size_t location_id) final { - - auto start_total_time = std::chrono::steady_clock::now(); - - Data in_data {location_id}; - Data out_data {location_id}; - - bool success = m_input.pull(in_data) && m_output.pull(out_data); - if (!success) { - m_input.revert(in_data); - m_output.revert(out_data); - // TODO: Test that revert works properly - - auto end_total_time = std::chrono::steady_clock::now(); - result.update(JArrowMetrics::Status::ComeBackLater, 0, 1, std::chrono::milliseconds(0), end_total_time - start_total_time); - return; - } - - bool process_succeeded = true; - JArrowMetrics::Status process_status = JArrowMetrics::Status::KeepGoing; - assert(in_data.item_count == 1); - MessageT* event = in_data.items[0]; - - auto start_processing_time = std::chrono::steady_clock::now(); - static_cast(this)->process(event, process_succeeded, process_status); - auto end_processing_time = std::chrono::steady_clock::now(); - - if (process_succeeded) { - in_data.item_count = 0; - out_data.item_count = 1; - out_data.items[0] = event; - } - m_input.push(in_data); - m_output.push(out_data); - - // Publish metrics - auto end_total_time = std::chrono::steady_clock::now(); - auto latency = (end_processing_time - start_processing_time); - auto overhead = (end_total_time - start_total_time) - latency; - result.update(process_status, process_succeeded, 1, latency, overhead); - } -}; diff --git a/src/libraries/JANA/Topology/JPool.h b/src/libraries/JANA/Topology/JPool.h deleted file mode 100644 index 83bd2b437..000000000 --- a/src/libraries/JANA/Topology/JPool.h +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2023, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#pragma once -#include -#include -#include - - -class JPoolBase { -protected: - size_t m_pool_size; - size_t m_location_count; - bool m_limit_total_events_in_flight; -public: - JPoolBase( - size_t pool_size, - size_t location_count, - bool limit_total_events_in_flight) - : m_pool_size(pool_size) - , m_location_count(location_count) - , m_limit_total_events_in_flight(limit_total_events_in_flight) {} - - virtual ~JPoolBase() = default; -}; - -template -class JPool : public JPoolBase { -private: - struct alignas(JANA2_CACHE_LINE_BYTES) LocalPool { - std::mutex mutex; - std::vector available_items; - std::vector items; - }; - - std::unique_ptr m_pools; - -public: - JPool(size_t pool_size, - size_t location_count, - bool limit_total_events_in_flight) : JPoolBase(pool_size, location_count, limit_total_events_in_flight) - { - assert(m_location_count >= 1); - assert(m_pool_size > 0 || !m_limit_total_events_in_flight); - } - - virtual ~JPool() = default; - - void init() { - m_pools = std::unique_ptr(new LocalPool[m_location_count]()); - - for (size_t j=0; j(m_pool_size); // Default-construct everything in place - - for (T& item : m_pools[j].items) { - configure_item(&item); - m_pools[j].available_items.push_back(&item); - } - } - } - - virtual void configure_item(T*) { - } - - virtual void release_item(T*) { - } - - - T* get(size_t location=0) { - - assert(m_pools != nullptr); // If you hit this, you forgot to call init(). - LocalPool& pool = m_pools[location % m_location_count]; - std::lock_guard lock(pool.mutex); - - if (pool.available_items.empty()) { - if (m_limit_total_events_in_flight) { - return nullptr; - } - else { - auto t = new T; - configure_item(t); - return t; - } - } - else { - T* item = pool.available_items.back(); - pool.available_items.pop_back(); - return item; - } - } - - - void put(T* item, size_t location=0) { - - assert(m_pools != nullptr); // If you hit this, you forgot to call init(). - - // Do any necessary teardown within the item itself - release_item(item); - - // Consider each location starting with current one - for (size_t l = location; l= &(pool.items[0])) && (item <= &(pool.items[m_pool_size-1]))) { - std::lock_guard lock(pool.mutex); - pool.available_items.push_back(item); - return; - } - - } - // Otherwise it was allocated on the heap - delete item; - } - - // TODO: This is wrong. Do we use this anywhere? - size_t size() { return m_pool_size; } - - // TODO: Remove me - bool get_many(std::vector& dest, size_t count, size_t location=0) { - - assert(m_pools != nullptr); // If you hit this, you forgot to call init(). - - LocalPool& pool = m_pools[location % m_location_count]; - std::lock_guard lock(pool.mutex); - - if (m_limit_total_events_in_flight && pool.available_items.size() < count) { - return false; - } - else { - while (count > 0 && !pool.available_items.empty()) { - T* t = pool.available_items.back(); - pool.available_items.pop_back(); - dest.push_back(t); - count -= 1; - } - while (count > 0) { - auto t = new T; - configure_item(t); - dest.push_back(t); - count -= 1; - } - return true; - } - } - - // TODO: Remove me - void put_many(std::vector& finished_events, size_t location=0) { - for (T* item : finished_events) { - put(item, location); - } - } - - - size_t pop(T** dest, size_t min_count, size_t max_count, size_t location=0) { - - assert(m_pools != nullptr); // If you hit this, you forgot to call init(). - - LocalPool& pool = m_pools[location % m_location_count]; - std::lock_guard lock(pool.mutex); - - size_t available_count = pool.available_items.size(); - - if (m_limit_total_events_in_flight && available_count < min_count) { - // Exit immmediately if we can't reach the minimum - return 0; - } - if (m_limit_total_events_in_flight) { - // Return as many as we can. We aren't allowed to create any more - size_t count = std::min(available_count, max_count); - for (size_t i=0; i - -/// SubeventProcessor offers sub-event-level parallelism. The idea is to split parent -/// event S into independent subtasks T, and automatically bundling them with -/// bookkeeping information X onto a Queue. process :: T -> U handles the stateless, -/// parallel parts; its Arrow pushes messages on to a Queue, so that merge() :: S -> [U] -> V -/// "joins" all completed "subtasks" of type U corresponding to one parent of type S, (i.e. -/// a specific JEvent), back into a single entity of type V, (most likely the same JEvent as S, -/// only now containing more data) which is pushed onto a Queue, bookkeeping information now gone. -/// Note that there is no blocking and that our streaming paradigm is not compromised. - -/// Abstract class which is meant to extended by the user to contain all -/// subtask-related functions. (Data lives in a JObject instead) -/// Future versions might be templated for two reasons: -/// 1. To make the functions non-virtual, -/// 2. To replace the generic JObject pointer with something typesafe -/// Future versions could also recycle JObjects by using another Queue. - -template -struct JSubeventProcessor { - - std::string inputTag; - std::string outputTag; - - virtual OutputT* ProcessSubevent(InputT* input) = 0; - -}; - -template -struct SubeventWrapper { - - std::shared_ptr* parent; - SubeventT* data; - size_t id; - size_t total; - - SubeventWrapper(std::shared_ptr* parent, SubeventT* data, size_t id, size_t total) - : parent(std::move(parent)) - , data(data) - , id(id) - , total(total) {} -}; - - -template -class JSubeventArrow : public JArrow { - JSubeventProcessor* m_processor; - JMailbox>* m_inbox; - JMailbox>* m_outbox; -public: - JSubeventArrow(std::string name, - JSubeventProcessor* processor, - JMailbox>* inbox, - JMailbox>* outbox) - : JArrow(name, true, false, false), m_processor(processor), m_inbox(inbox), m_outbox(outbox) { - } - - size_t get_pending() final { return m_inbox->size(); }; - size_t get_threshold() final { return m_inbox->get_threshold(); }; - void set_threshold(size_t threshold) final { m_inbox->set_threshold(threshold);}; - - void execute(JArrowMetrics&, size_t location_id) override; -}; - -template -class JSplitArrow : public JArrow { - JSubeventProcessor* m_processor; - JMailbox*>* m_inbox; - JMailbox>* m_outbox; -public: - JSplitArrow(std::string name, - JSubeventProcessor* processor, - JMailbox*>* inbox, - JMailbox>* outbox) - : JArrow(name, true, false, false), m_processor(processor), m_inbox(inbox), m_outbox(outbox) { - } - - size_t get_pending() final { return m_inbox->size(); }; - size_t get_threshold() final { return m_inbox->get_threshold(); }; - void set_threshold(size_t threshold) final { m_inbox->set_threshold(threshold);}; - - void execute(JArrowMetrics&, size_t location_id) override; -}; - -template -class JMergeArrow : public JArrow { - JSubeventProcessor* m_processor; - JMailbox>* m_inbox; - JMailbox*>* m_outbox; - std::map*, size_t> m_in_progress; -public: - JMergeArrow(std::string name, - JSubeventProcessor* processor, - JMailbox>* inbox, - JMailbox*>* outbox) - : JArrow(name, false, false, false), m_processor(processor), m_inbox(inbox), m_outbox(outbox) { - } - - size_t get_pending() final { return m_inbox->size(); }; - size_t get_threshold() final { return m_inbox->get_threshold(); }; - void set_threshold(size_t threshold) final { m_inbox->set_threshold(threshold);}; - void execute(JArrowMetrics&, size_t location_id) override; -}; - - - -template -void JSplitArrow::execute(JArrowMetrics& result, size_t location_id) { - using InQueue = JMailbox*>; - using OutQueue = JMailbox>; - auto start_total_time = std::chrono::steady_clock::now(); - - std::shared_ptr* event = nullptr; - bool success; - size_t reserved_size = m_outbox->reserve(get_chunksize()); - size_t actual_size = reserved_size; - // TODO: Exit early if we don't have enough space on output queue - - auto in_status = m_inbox->pop(event, success, location_id); - auto start_latency_time = std::chrono::steady_clock::now(); - - std::vector> wrapped; - if (success) { - - // Construct prereqs - std::vector originals = (*event)->Get(m_processor->inputTag); - size_t i = 1; - actual_size = originals.size(); - - for (const InputT* original : originals) { - InputT* unconsted = const_cast(original); // TODO: Get constness right - wrapped.push_back(SubeventWrapper(event, unconsted, i++, actual_size)); - } - } - auto end_latency_time = std::chrono::steady_clock::now(); - auto out_status = OutQueue::Status::Ready; - - size_t output_size = wrapped.size(); - if (success) { - assert(m_outbox != nullptr); - out_status = m_outbox->push(wrapped, reserved_size, location_id); - } - auto end_queue_time = std::chrono::steady_clock::now(); - - JArrowMetrics::Status status; - if (in_status == InQueue::Status::Ready && out_status == OutQueue::Status::Ready) { - status = JArrowMetrics::Status::KeepGoing; - } - else { - status = JArrowMetrics::Status::ComeBackLater; - } - auto latency = (end_latency_time - start_latency_time); - auto overhead = (end_queue_time - start_total_time) - latency; - result.update(status, output_size, 1, latency, overhead); - -} -template -void JSubeventArrow::execute(JArrowMetrics& result, size_t location_id) { - using SubeventQueue = JMailbox>; - auto start_total_time = std::chrono::steady_clock::now(); - - // TODO: Think more carefully about subevent bucket size - std::vector> inputs; - size_t downstream_accepts = m_outbox->reserve(get_chunksize(), location_id); - auto in_status = m_inbox->pop(inputs, downstream_accepts, location_id); - auto start_latency_time = std::chrono::steady_clock::now(); - - std::vector> outputs; - for (const auto& input : inputs) { - auto output = m_processor->ProcessSubevent(input.data); - auto wrapped = SubeventWrapper(input.parent, output, input.id, input.total); - outputs.push_back(wrapped); - } - size_t outputs_size = outputs.size(); - auto end_latency_time = std::chrono::steady_clock::now(); - auto out_status = JMailbox>::Status::Ready; - - if (outputs_size > 0) { - assert(m_outbox != nullptr); - out_status = m_outbox->push(outputs, downstream_accepts, location_id); - } - auto end_queue_time = std::chrono::steady_clock::now(); - - JArrowMetrics::Status status; - if (in_status == SubeventQueue::Status::Ready && out_status == JMailbox>::Status::Ready) { - status = JArrowMetrics::Status::KeepGoing; - } - else { - status = JArrowMetrics::Status::ComeBackLater; - } - auto latency = (end_latency_time - start_latency_time); - auto overhead = (end_queue_time - start_total_time) - latency; - result.update(status, outputs_size, 1, latency, overhead); - -} -template -void JMergeArrow::execute(JArrowMetrics& result, size_t location_id) { - using InQueue = JMailbox>; - using OutQueue = JMailbox*>; - - auto start_total_time = std::chrono::steady_clock::now(); - - // TODO: Think more carefully about subevent bucket size - std::vector> inputs; - size_t downstream_accepts = m_outbox->reserve(get_chunksize(), location_id); - auto in_status = m_inbox->pop(inputs, downstream_accepts, location_id); - auto start_latency_time = std::chrono::steady_clock::now(); - - std::vector*> outputs; - for (const auto& input : inputs) { - LOG_TRACE(m_logger) << "JMergeArrow: Processing input with parent=" << input.parent << ", evt=" << (*(input.parent))->GetEventNumber() << ", sub=" << input.id << " and total=" << input.total << LOG_END; - // Problem: Are we sure we are updating the event in a way which is effectively thread-safe? - // Should we be doing this insert, or should the caller? - (*(input.parent))->template Insert(input.data); - if (input.total == 1) { - // Goes straight into "ready" - outputs.push_back(input.parent); - LOG_TRACE(m_logger) << "JMergeArrow: Finished parent=" << input.parent << ", evt=" << (*(input.parent))->GetEventNumber() << LOG_END; - } - else { - auto pair = m_in_progress.find(input.parent); - if (pair == m_in_progress.end()) { - m_in_progress[input.parent] = input.total-1; - } - else { - if (pair->second == 0) { // Event was already in the map. TODO: What happens when we turn off event recycling? - pair->second = input.total-1; - } - else if (pair->second == 1) { - pair->second -= 1; - outputs.push_back(input.parent); - LOG_TRACE(m_logger) << "JMergeArrow: Finished parent=" << input.parent << ", evt=" << (*(input.parent))->GetEventNumber() << LOG_END; - } - else { - pair->second -= 1; - } - } - } - } - LOG_DEBUG(m_logger) << "MergeArrow consumed " << inputs.size() << " subevents, produced " << outputs.size() << " events" << LOG_END; - auto end_latency_time = std::chrono::steady_clock::now(); - - auto outputs_size = outputs.size(); - auto out_status = m_outbox->push(outputs, downstream_accepts, location_id); - - auto end_queue_time = std::chrono::steady_clock::now(); - - JArrowMetrics::Status status; - if (in_status == InQueue::Status::Ready && out_status == OutQueue::Status::Ready && inputs.size() > 0) { - status = JArrowMetrics::Status::KeepGoing; - } - else { - status = JArrowMetrics::Status::ComeBackLater; - } - auto latency = (end_latency_time - start_latency_time); - auto overhead = (end_queue_time - start_total_time) - latency; - result.update(status, outputs_size, 1, latency, overhead); -} - - - diff --git a/src/libraries/JANA/Topology/JTopologyBuilder.cc b/src/libraries/JANA/Topology/JTopologyBuilder.cc index ff685575a..451b7fd82 100644 --- a/src/libraries/JANA/Topology/JTopologyBuilder.cc +++ b/src/libraries/JANA/Topology/JTopologyBuilder.cc @@ -6,15 +6,17 @@ #include "JTopologyBuilder.h" #include "JEventSourceArrow.h" -#include "JEventProcessorArrow.h" #include "JEventMapArrow.h" +#include "JEventTapArrow.h" #include "JUnfoldArrow.h" #include "JFoldArrow.h" +#include #include -using Event = std::shared_ptr; -using EventQueue = JMailbox; +JTopologyBuilder::JTopologyBuilder() { + SetPrefix("jana"); +} JTopologyBuilder::~JTopologyBuilder() { for (auto arrow : arrows) { @@ -42,12 +44,12 @@ std::string JTopologyBuilder::print_topology() { // Build index lookup for queues int i = 0; std::map lookup; - for (JQueue* queue : queues) { + for (JEventQueue* queue : queues) { lookup[queue] = i; i += 1; } // Build index lookup for pools - for (JPoolBase* pool : pools) { + for (JEventPool* pool : pools) { lookup[pool] = i; i += 1; } @@ -58,7 +60,7 @@ std::string JTopologyBuilder::print_topology() { for (JArrow* arrow : arrows) { show_row = true; - for (PlaceRefBase* place : arrow->m_places) { + for (JArrow::Port& port : arrow->m_ports) { if (show_row) { t | arrow->get_name(); t | arrow->is_parallel(); @@ -67,10 +69,11 @@ std::string JTopologyBuilder::print_topology() { else { t | "" | "" ; } + auto place_index = lookup[(port.queue!=nullptr) ? (void*) port.queue : (void*) port.pool]; - t | ((place->is_input) ? "Input ": "Output"); - t | ((place->is_queue) ? "Queue ": "Pool"); - t | lookup[place->place_ref]; + t | ((port.is_input) ? "Input ": "Output"); + t | ((port.queue != nullptr) ? "Queue ": "Pool"); + t | place_index; } } return t.Render(); @@ -89,28 +92,18 @@ void JTopologyBuilder::create_topology() { mapping.initialize(static_cast(m_affinity), static_cast(m_locality)); - event_pool = new JEventPool(m_components, - m_event_pool_size, - m_location_count, - m_limit_total_events_in_flight); - event_pool->init(); + event_pool = new JEventPool(m_components, m_max_inflight_events, m_location_count); if (m_configure_topology) { m_configure_topology(*this); LOG_WARN(GetLogger()) << "Found custom topology configurator! Modified arrow topology is: \n" << print_topology() << LOG_END; } else { - attach_top_level(JEventLevel::Run); + attach_level(JEventLevel::Run, nullptr, nullptr); LOG_INFO(GetLogger()) << "Arrow topology is:\n" << print_topology() << LOG_END; } - int id=0; - for (auto* queue : queues) { - queue->set_logger(m_queue_logger); - queue->set_id(id); - id += 1; - } for (auto* arrow : arrows) { - arrow->set_logger(m_arrow_logger); + arrow->set_logger(GetLogger()); } } @@ -123,252 +116,276 @@ void JTopologyBuilder::acquire_services(JServiceLocator *sl) { // We parse the 'nthreads' parameter two different ways for backwards compatibility. if (m_params->Exists("nthreads")) { if (m_params->GetParameterValue("nthreads") == "Ncores") { - m_event_pool_size = JCpuInfo::GetNumCpus(); + m_max_inflight_events = JCpuInfo::GetNumCpus(); } else { - m_event_pool_size = m_params->GetParameterValue("nthreads"); + m_max_inflight_events = m_params->GetParameterValue("nthreads"); } } - m_params->SetDefaultParameter("jana:event_pool_size", m_event_pool_size, - "Sets the initial size of the event pool. Having too few events starves the workers; having too many consumes memory and introduces overhead from extra factory initializations") - ->SetIsAdvanced(true); - m_params->SetDefaultParameter("jana:limit_total_events_in_flight", m_limit_total_events_in_flight, - "Controls whether the event pool is allowed to automatically grow beyond jana:event_pool_size") - ->SetIsAdvanced(true); - m_params->SetDefaultParameter("jana:event_queue_threshold", m_event_queue_threshold, - "Max number of events allowed on the main event queue. Higher => Better load balancing; Lower => Fewer events in flight") - ->SetIsAdvanced(true); - m_params->SetDefaultParameter("jana:event_source_chunksize", m_event_source_chunksize, - "Max number of events that a JEventSource may enqueue at once. Higher => less queue contention; Lower => better load balancing") - ->SetIsAdvanced(true); - m_params->SetDefaultParameter("jana:event_processor_chunksize", m_event_processor_chunksize, - "Max number of events that the JEventProcessors may dequeue at once. Higher => less queue contention; Lower => better load balancing") + m_params->SetDefaultParameter("jana:max_inflight_events", m_max_inflight_events, + "The number of events which may be in-flight at once. Should be at least `nthreads` to prevent starvation; more gives better load balancing.") ->SetIsAdvanced(true); + + /* m_params->SetDefaultParameter("jana:enable_stealing", m_enable_stealing, "Enable work stealing. Improves load balancing when jana:locality != 0; otherwise does nothing.") ->SetIsAdvanced(true); + */ m_params->SetDefaultParameter("jana:affinity", m_affinity, "Constrain worker thread CPU affinity. 0=Let the OS decide. 1=Avoid extra memory movement at the expense of using hyperthreads. 2=Avoid hyperthreads at the expense of extra memory movement") ->SetIsAdvanced(true); m_params->SetDefaultParameter("jana:locality", m_locality, "Constrain memory locality. 0=No constraint. 1=Events stay on the same socket. 2=Events stay on the same NUMA domain. 3=Events stay on same core. 4=Events stay on same cpu/hyperthread.") ->SetIsAdvanced(true); - - m_arrow_logger = m_logging->get_logger("JArrow"); - m_queue_logger = m_logging->get_logger("JQueue"); }; -void JTopologyBuilder::attach_lower_level(JEventLevel current_level, JUnfoldArrow* parent_unfolder, JFoldArrow* parent_folder, bool found_sink) { - - std::stringstream ss; - ss << current_level; - - LOG_DEBUG(GetLogger()) << "JTopologyBuilder: Attaching components at lower level = " << current_level << LOG_END; - - JEventPool* pool = new JEventPool(m_components, - m_event_pool_size, - m_location_count, - m_limit_total_events_in_flight, - current_level); - pool->init(); - pools.push_back(pool); // Transfers ownership +void JTopologyBuilder::connect(JArrow* upstream, size_t up_index, JArrow* downstream, size_t down_index) { + auto queue = new JEventQueue(m_max_inflight_events, mapping.get_loc_count()); + queues.push_back(queue); - std::vector sources_at_level; - for (JEventSource* source : m_components->get_evt_srces()) { - if (source->GetLevel() == current_level) { - sources_at_level.push_back(source); - } - } - std::vector procs_at_level; - for (JEventProcessor* proc : m_components->get_evt_procs()) { - if (proc->GetLevel() == current_level) { - procs_at_level.push_back(proc); + size_t i = 0; + for (JArrow::Port& port : upstream->m_ports) { + if (!port.is_input) { + if (i++ == up_index) { + // Found the correct output + port.queue = queue; + port.pool = nullptr; + } } } - std::vector unfolders_at_level; - for (JEventUnfolder* unfolder : m_components->get_unfolders()) { - if (unfolder->GetLevel() == current_level) { - unfolders_at_level.push_back(unfolder); + i = 0; + for (JArrow::Port& port : downstream->m_ports) { + if (port.is_input) { + if (i++ == down_index) { + // Found the correct input + port.queue = queue; + port.pool = nullptr; + } } } +} - if (sources_at_level.size() != 0) { - throw JException("Support for lower-level event sources coming soon!"); - } - if (unfolders_at_level.size() != 0) { - throw JException("Support for lower-level event unfolders coming soon!"); - } - if (procs_at_level.size() == 0) { - throw JException("For now we require you to provide at least one JEventProcessor"); - } - - auto q1 = new EventQueue(m_event_queue_threshold, mapping.get_loc_count(), m_enable_stealing); - queues.push_back(q1); - - auto q2 = new EventQueue(m_event_queue_threshold, mapping.get_loc_count(), m_enable_stealing); - queues.push_back(q2); - - auto* proc_arrow = new JEventProcessorArrow(ss.str()+"Tap", q1, q2, nullptr); - arrows.push_back(proc_arrow); - proc_arrow->set_chunksize(m_event_processor_chunksize); - proc_arrow->set_logger(m_arrow_logger); - if (found_sink) { - proc_arrow->set_is_sink(false); - } - - for (auto proc: procs_at_level) { - proc_arrow->add_processor(proc); +void JTopologyBuilder::connect_to_first_available(JArrow* upstream, std::vector downstreams) { + for (JArrow* downstream : downstreams) { + if (downstream != nullptr) { + // Arrows at the same level all connect at index 0 (even the input for the parent JFoldArrow) + connect(upstream, 0, downstream, 0); + return; + } } - - parent_unfolder->attach_child_in(pool); - parent_unfolder->attach_child_out(q1); - parent_folder->attach_child_in(q2); - parent_folder->attach_child_out(pool); - parent_unfolder->attach(proc_arrow); - proc_arrow->attach(parent_folder); } -void JTopologyBuilder::attach_top_level(JEventLevel current_level) { - +void JTopologyBuilder::attach_level(JEventLevel current_level, JUnfoldArrow* parent_unfolder, JFoldArrow* parent_folder) { std::stringstream ss; ss << current_level; auto level_str = ss.str(); + // Find all event sources at this level std::vector sources_at_level; for (JEventSource* source : m_components->get_evt_srces()) { if (source->GetLevel() == current_level) { sources_at_level.push_back(source); } } - if (sources_at_level.size() == 0) { - // Skip level entirely for now. Consider eventually supporting - // folding low levels into higher levels without corresponding unfold - LOG_TRACE(GetLogger()) << "JTopologyBuilder: No sources found at level " << current_level << ", skipping" << LOG_END; - JEventLevel next = next_level(current_level); - if (next == JEventLevel::None) { - LOG_WARN(GetLogger()) << "No sources found: Processing topology will be empty." << LOG_END; - return; - } - return attach_top_level(next); - } - LOG_DEBUG(GetLogger()) << "JTopologyBuilder: Attaching components at top level = " << current_level << LOG_END; - - // We've now found our top level. No matter what, we need an event pool for this level - JEventPool* pool_at_level = new JEventPool(m_components, - m_event_pool_size, - m_location_count, - m_limit_total_events_in_flight, - current_level); - pool_at_level->init(); - pools.push_back(pool_at_level); // Hand over ownership of the pool to the topology - - // There are two possibilities at this point: - // a. This is the only level, in which case we wire up the arrows and exit - // b. We have an unfolder/folder pair, in which case we wire everything up, and then recursively attach_lower_level(). - // We use the presence of an unfolder as our test for whether or not a lower level should be included. This is because - // the folder might be trivial and hence omitted by the user. (Note that some folder is always needed in order to return - // the higher-level event to the pool). - // The user always needs to provide an unfolder because I can't think of a trivial unfolder that would be useful. - + + // Find all unfolders at this level std::vector unfolders_at_level; for (JEventUnfolder* unfolder : m_components->get_unfolders()) { if (unfolder->GetLevel() == current_level) { unfolders_at_level.push_back(unfolder); } } + + // Find all processors at this level + std::vector mappable_procs_at_level; + std::vector tappable_procs_at_level; - std::vector procs_at_level; for (JEventProcessor* proc : m_components->get_evt_procs()) { if (proc->GetLevel() == current_level) { - procs_at_level.push_back(proc); + mappable_procs_at_level.push_back(proc); + if (proc->GetCallbackStyle() != JEventProcessor::CallbackStyle::LegacyMode) { + tappable_procs_at_level.push_back(proc); + } } } - if (unfolders_at_level.size() == 0) { - // No unfolders, so this is the only level - // Attach the source to the map/tap just like before - // - // We might want to print a friendly warning message communicating why any lower-level - // components are being ignored, like so: - // skip_lower_level(next_level(current_level)); - LOG_DEBUG(GetLogger()) << "JTopologyBuilder: No unfolders found at level " << current_level << ", finishing here." << LOG_END; - - auto queue = new EventQueue(m_event_queue_threshold, mapping.get_loc_count(), m_enable_stealing); - queues.push_back(queue); - - auto* src_arrow = new JEventSourceArrow(level_str+"Source", sources_at_level, queue, pool_at_level); - arrows.push_back(src_arrow); - src_arrow->set_chunksize(m_event_source_chunksize); - - auto* proc_arrow = new JEventProcessorArrow(level_str+"Tap", queue, nullptr, pool_at_level); - arrows.push_back(proc_arrow); - proc_arrow->set_chunksize(m_event_processor_chunksize); - - for (auto proc: procs_at_level) { - proc_arrow->add_processor(proc); + bool is_top_level = (parent_unfolder == nullptr); + if (is_top_level && sources_at_level.size() == 0) { + // Skip level entirely when no source is present. + LOG_TRACE(GetLogger()) << "JTopologyBuilder: No sources found at level " << current_level << ", skipping" << LOG_END; + JEventLevel next = next_level(current_level); + if (next == JEventLevel::None) { + LOG_WARN(GetLogger()) << "No sources found: Processing topology will be empty." << LOG_END; + return; } - src_arrow->attach(proc_arrow); + return attach_level(next, nullptr, nullptr); } - else if (unfolders_at_level.size() != 1) { - throw JException("At most one unfolder must be provided for each level in the event hierarchy!"); + + // Enforce constraints on what our builder will accept (at least for now) + if (!is_top_level && !sources_at_level.empty()) { + throw JException("Topology forbids event sources at lower event levels in the topology"); } - else { - - auto q1 = new EventQueue(m_event_queue_threshold, mapping.get_loc_count(), m_enable_stealing); - auto q2 = new EventQueue(m_event_queue_threshold, mapping.get_loc_count(), m_enable_stealing); + if ((parent_unfolder == nullptr && parent_folder != nullptr) || (parent_unfolder != nullptr && parent_folder == nullptr)) { + throw JException("Topology requires matching unfolder/folder arrow pairs"); + } + if (unfolders_at_level.size() > 1) { + throw JException("Multiple JEventUnfolders provided for level %s", level_str.c_str()); + } + // Another constraint is that the highest level of the topology has an event sources, but this is automatically handled by + // the level-skipping logic above + - queues.push_back(q1); - queues.push_back(q2); + // Fill out arrow grid from components at this event level + // -------------------------- + // 0. Pool + // -------------------------- + JEventPool* pool_at_level = new JEventPool(m_components, m_max_inflight_events, m_location_count, current_level); + pools.push_back(pool_at_level); // Hand over ownership of the pool to the topology - auto *src_arrow = new JEventSourceArrow(level_str+"Source", sources_at_level, q1, pool_at_level); + // -------------------------- + // 1. Source + // -------------------------- + JEventSourceArrow* src_arrow = nullptr; + bool need_source = !sources_at_level.empty(); + if (need_source) { + src_arrow = new JEventSourceArrow(level_str+"Source", sources_at_level); + src_arrow->attach(pool_at_level, JEventSourceArrow::EVENT_IN); + src_arrow->attach(pool_at_level, JEventSourceArrow::EVENT_OUT); arrows.push_back(src_arrow); - src_arrow->set_chunksize(m_event_source_chunksize); + } - auto *map_arrow = new JEventMapArrow(level_str+"Map", q1, q2);; - arrows.push_back(map_arrow); - map_arrow->set_chunksize(m_event_source_chunksize); - src_arrow->attach(map_arrow); + // -------------------------- + // 2. Map1 + // -------------------------- + bool have_parallel_sources = false; + for (JEventSource* source: sources_at_level) { + have_parallel_sources |= source->IsPreprocessEnabled(); + } + bool have_unfolder = !unfolders_at_level.empty(); + JEventMapArrow* map1_arrow = nullptr; + bool need_map1 = (have_parallel_sources || have_unfolder); + + if (need_map1) { + map1_arrow = new JEventMapArrow(level_str+"Map1"); + for (JEventSource* source: sources_at_level) { + if (source->IsPreprocessEnabled()) { + map1_arrow->add_source(source); + } + } + for (JEventUnfolder* unf: unfolders_at_level) { + map1_arrow->add_unfolder(unf); + } + map1_arrow->attach(pool_at_level, JEventMapArrow::EVENT_IN); + map1_arrow->attach(pool_at_level, JEventMapArrow::EVENT_OUT); + arrows.push_back(map1_arrow); + } - // TODO: We are using q2 temporarily knowing that it will be overwritten in attach_lower_level. - // It would be better to rejigger how we validate PlaceRefs and accept empty placerefs/fewer ctor args - auto *unfold_arrow = new JUnfoldArrow(level_str+"Unfold", unfolders_at_level[0], q2, pool_at_level, q2); + // -------------------------- + // 3. Unfold + // -------------------------- + JUnfoldArrow* unfold_arrow = nullptr; + bool need_unfold = have_unfolder; + if (need_unfold) { + unfold_arrow = new JUnfoldArrow(level_str+"Unfold", unfolders_at_level[0]); arrows.push_back(unfold_arrow); - unfold_arrow->set_chunksize(m_event_source_chunksize); - map_arrow->attach(unfold_arrow); - - // child_in, child_out, parent_out - auto *fold_arrow = new JFoldArrow(level_str+"Fold", current_level, unfolders_at_level[0]->GetChildLevel(), q2, pool_at_level, pool_at_level); - // TODO: Support user-provided folders - fold_arrow->set_chunksize(m_event_source_chunksize); - - bool found_sink = (procs_at_level.size() > 0); - attach_lower_level(unfolders_at_level[0]->GetChildLevel(), unfold_arrow, fold_arrow, found_sink); + } - // Push fold arrow back _after_ attach_lower_level so that arrows can be iterated over in order + // -------------------------- + // 4. Fold + // -------------------------- + JFoldArrow* fold_arrow = nullptr; + bool need_fold = have_unfolder; + if(need_fold) { + fold_arrow = new JFoldArrow(level_str+"Fold", current_level, unfolders_at_level[0]->GetChildLevel()); arrows.push_back(fold_arrow); + fold_arrow->attach(pool_at_level, JFoldArrow::PARENT_OUT); + } - if (procs_at_level.size() != 0) { + // -------------------------- + // 5. Map2 + // -------------------------- + JEventMapArrow* map2_arrow = nullptr; + bool need_map2 = !mappable_procs_at_level.empty(); + if (need_map2) { + map2_arrow = new JEventMapArrow(level_str+"Map2"); + for (JEventProcessor* proc : mappable_procs_at_level) { + map2_arrow->add_processor(proc); + map2_arrow->attach(pool_at_level, JEventMapArrow::EVENT_IN); + map2_arrow->attach(pool_at_level, JEventMapArrow::EVENT_OUT); + } + arrows.push_back(map2_arrow); + } - auto q3 = new EventQueue(m_event_queue_threshold, mapping.get_loc_count(), m_enable_stealing); - queues.push_back(q3); + // -------------------------- + // 6. Tap + // -------------------------- + JEventTapArrow* tap_arrow = nullptr; + bool need_tap = !tappable_procs_at_level.empty(); + if (need_tap) { + tap_arrow = new JEventTapArrow(level_str+"Tap"); + for (JEventProcessor* proc : tappable_procs_at_level) { + tap_arrow->add_processor(proc); + tap_arrow->attach(pool_at_level, JEventTapArrow::EVENT_IN); + tap_arrow->attach(pool_at_level, JEventTapArrow::EVENT_OUT); + } + arrows.push_back(tap_arrow); + } - auto* proc_arrow = new JEventProcessorArrow(level_str+"Tap", q3, nullptr, pool_at_level); - arrows.push_back(proc_arrow); - proc_arrow->set_chunksize(m_event_processor_chunksize); - for (auto proc: procs_at_level) { - proc_arrow->add_processor(proc); - } + // Now that we've set up our component grid, we can do wiring! + // -------------------------- + // 1. Source + // -------------------------- + if (parent_unfolder != nullptr) { + parent_unfolder->attach(pool_at_level, JUnfoldArrow::CHILD_IN); + connect_to_first_available(parent_unfolder, {map1_arrow, unfold_arrow, map2_arrow, tap_arrow, parent_folder}); + } + if (src_arrow != nullptr) { + connect_to_first_available(src_arrow, {map1_arrow, unfold_arrow, map2_arrow, tap_arrow, parent_folder}); + } + if (map1_arrow != nullptr) { + connect_to_first_available(map1_arrow, {unfold_arrow, map2_arrow, tap_arrow, parent_folder}); + } + if (fold_arrow != nullptr) { + connect_to_first_available(fold_arrow, {map2_arrow, tap_arrow, parent_folder}); + } + if (map2_arrow != nullptr) { + connect_to_first_available(map2_arrow, {tap_arrow, parent_folder}); + } + if (tap_arrow != nullptr) { + connect_to_first_available(tap_arrow, {parent_folder}); + } + if (parent_folder != nullptr) { + parent_folder->attach(pool_at_level, JFoldArrow::CHILD_OUT); + } - fold_arrow->attach_parent_out(q3); - fold_arrow->attach(proc_arrow); + // Finally, we recur over lower levels! + if (need_unfold) { + auto next_level = unfolders_at_level[0]->GetChildLevel(); + attach_level(next_level, unfold_arrow, fold_arrow); + } + else { + // This is the lowest level + // TODO: Improve logic for determining event counts for multilevel topologies + if (tap_arrow != nullptr) { + tap_arrow->set_is_sink(true); + } + else if (map2_arrow != nullptr) { + map2_arrow->set_is_sink(true); + } + else if (map1_arrow != nullptr) { + map1_arrow->set_is_sink(true); + } + else if (src_arrow != nullptr) { + src_arrow->set_is_sink(true); } } - } + + diff --git a/src/libraries/JANA/Topology/JTopologyBuilder.h b/src/libraries/JANA/Topology/JTopologyBuilder.h index a287d4631..685efc09f 100644 --- a/src/libraries/JANA/Topology/JTopologyBuilder.h +++ b/src/libraries/JANA/Topology/JTopologyBuilder.h @@ -7,60 +7,45 @@ #include #include #include -#include // TODO: Should't be here +#include +#include #include #include -#include class JParameterManager; -class JLoggingService; class JComponentManager; class JArrow; -class JQueue; -class JPoolBase; -class JQueue; class JFoldArrow; class JUnfoldArrow; -class JEventPool; class JTopologyBuilder : public JService { public: // Services Service m_params {this}; - Service m_logging {this}; std::shared_ptr m_components; // The topology itself std::vector arrows; - std::vector queues; // Queues shared between arrows - std::vector pools; // Pools shared between arrows + std::vector queues; // Queues shared between arrows + std::vector pools; // Pools shared between arrows // Topology configuration - size_t m_event_pool_size = 4; - size_t m_event_queue_threshold = 80; - size_t m_event_source_chunksize = 40; - size_t m_event_processor_chunksize = 1; + size_t m_max_inflight_events = 4; size_t m_location_count = 1; bool m_enable_stealing = false; - bool m_limit_total_events_in_flight = true; int m_affinity = 0; int m_locality = 0; // Things that probably shouldn't be here std::function m_configure_topology; JEventPool* event_pool = nullptr; // TODO: Move into pools eventually - JPerfMetrics metrics; JProcessorMapping mapping; - JLogger m_arrow_logger; - JLogger m_queue_logger; - public: - JTopologyBuilder() = default; - + JTopologyBuilder(); ~JTopologyBuilder() override; void acquire_services(JServiceLocator *sl) override; @@ -73,9 +58,9 @@ class JTopologyBuilder : public JService { void create_topology(); - void attach_lower_level(JEventLevel current_level, JUnfoldArrow* parent_unfolder, JFoldArrow* parent_folder, bool found_sink); - - void attach_top_level(JEventLevel current_level); + void attach_level(JEventLevel current_level, JUnfoldArrow* parent_unfolder, JFoldArrow* parent_folder); + void connect_to_first_available(JArrow* upstream, std::vector downstreams); + void connect(JArrow* upstream, size_t upstream_index, JArrow* downstream, size_t downstream_index); std::string print_topology(); diff --git a/src/libraries/JANA/Topology/JUnfoldArrow.h b/src/libraries/JANA/Topology/JUnfoldArrow.h index 1916c3980..5450b03f8 100644 --- a/src/libraries/JANA/Topology/JUnfoldArrow.h +++ b/src/libraries/JANA/Topology/JUnfoldArrow.h @@ -5,58 +5,23 @@ #include #include -#include class JUnfoldArrow : public JArrow { -private: - using EventT = std::shared_ptr; +public: + enum PortIndex {PARENT_IN=0, CHILD_IN=1, CHILD_OUT=2}; +private: JEventUnfolder* m_unfolder = nullptr; - EventT* m_parent_event = nullptr; - bool m_ready_to_fetch_parent = true; - - PlaceRef m_parent_in; - PlaceRef m_child_in; - PlaceRef m_child_out; + JEvent* m_parent_event = nullptr; + JEvent* m_child_event = nullptr; public: - - JUnfoldArrow( - std::string name, - JEventUnfolder* unfolder, - JMailbox* parent_in, - JEventPool* child_in, - JMailbox* child_out) - - : JArrow(std::move(name), false, false, false), - m_unfolder(unfolder), - m_parent_in(this, parent_in, true, 1, 1), - m_child_in(this, child_in, true, 1, 1), - m_child_out(this, child_out, false, 1, 1) - { - } - - void attach_parent_in(JMailbox* parent_in) { - m_parent_in.place_ref = parent_in; - m_parent_in.is_queue = true; - } - - void attach_child_in(JEventPool* child_in) { - m_child_in.place_ref = child_in; - m_child_in.is_queue = false; - } - - void attach_child_in(JMailbox* child_in) { - m_child_in.place_ref = child_in; - m_child_in.is_queue = true; - } - - void attach_child_out(JMailbox* child_out) { - m_child_out.place_ref = child_out; - m_child_out.is_queue = true; + JUnfoldArrow(std::string name, JEventUnfolder* unfolder) : m_unfolder(unfolder) { + set_name(name); + create_ports(2, 1); + m_next_input_port = PARENT_IN; } - void initialize() final { m_unfolder->DoInit(); LOG_INFO(m_logger) << "Initialized JEventUnfolder '" << m_unfolder->GetTypeName() << "'" << LOG_END; @@ -67,119 +32,87 @@ class JUnfoldArrow : public JArrow { LOG_INFO(m_logger) << "Finalized JEventUnfolder '" << m_unfolder->GetTypeName() << "'" << LOG_END; } - bool try_pull_all(Data& pi, Data& ci, Data& co) { - bool success; - success = m_parent_in.pull(pi); - if (! success) { - return false; + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) final { + + // Take whatever we were given + if (this->m_next_input_port == PARENT_IN) { + assert(m_parent_event == nullptr); + m_parent_event = event; + } + else if (this->m_next_input_port == CHILD_IN) { + assert(m_child_event == nullptr); + m_child_event = event; } - success = m_child_in.pull(ci); - if (! success) { - m_parent_in.push(pi); - return false; + else { + throw JException("Invalid input port for JEventUnfolder!"); } - success = m_child_out.pull(co); - if (! success) { - m_parent_in.push(pi); - m_child_in.push(ci); - return false; + + // Check if we should exit early because we don't have a parent event + if (m_parent_event == nullptr) { + m_next_input_port = PARENT_IN; + output_count = 0; + status = JArrow::FireResult::KeepGoing; + return; } - return true; - } - size_t push_all(Data& parent_in, Data& child_in, Data& child_out) { - size_t message_count = 0; - message_count += m_parent_in.push(parent_in); - message_count += m_child_in.push(child_in); - message_count += m_child_out.push(child_out); - return message_count; - } + // Check if we should exit early because we don't have a child event + if (m_child_event == nullptr) { + m_next_input_port = CHILD_IN; + output_count = 0; + status = JArrow::FireResult::KeepGoing; + return; + } + // At this point we know we have both inputs, so we can run the unfolder. First we validate + // that the events we received are at the correct level for the unfolder. Hopefully the only + // way to end up here is to override the JTopologyBuilder wiring and do it wrong - size_t get_pending() final { - size_t sum = 0; - for (PlaceRefBase* place : m_places) { - sum += place->get_pending(); + if (m_parent_event->GetLevel() != m_unfolder->GetLevel()) { + throw JException("JUnfolder: Expected parent with level %d, got %d", m_unfolder->GetLevel(), m_parent_event->GetLevel()); } - if (m_parent_event != nullptr) { - sum += 1; - // Handle the case of UnfoldArrow hanging on to a parent + + if (m_child_event->GetLevel() != m_unfolder->GetChildLevel()) { + throw JException("JUnfolder: Expected child with level %d, got %d", m_unfolder->GetChildLevel(), m_child_event->GetLevel()); } - return sum; - } + auto result = m_unfolder->DoUnfold(*m_parent_event, *m_child_event); + LOG_DEBUG(m_logger) << "Unfold succeeded: Parent event = " << m_parent_event->GetEventNumber() << ", child event = " << m_child_event->GetEventNumber() << LOG_END; + + if (result == JEventUnfolder::Result::KeepChildNextParent) { + m_parent_event->Release(); // Decrement the reference count so that this can be recycled + LOG_DEBUG(m_logger) << "Unfold finished with parent event = " << m_parent_event->GetEventNumber() << LOG_END; - void execute(JArrowMetrics& metrics, size_t location_id) final { - - auto start_total_time = std::chrono::steady_clock::now(); - - Data parent_in_data {location_id}; - Data child_in_data {location_id}; - Data child_out_data {location_id}; - - bool success = try_pull_all(parent_in_data, child_in_data, child_out_data); - if (success) { - - auto start_processing_time = std::chrono::steady_clock::now(); - if (m_ready_to_fetch_parent) { - m_ready_to_fetch_parent = false; - m_parent_in.min_item_count = 0; - m_parent_in.max_item_count = 0; - m_parent_event = parent_in_data.items[0]; - parent_in_data.items[0] = nullptr; - parent_in_data.item_count = 0; - } - auto child = child_in_data.items[0]; - child_in_data.items[0] = nullptr; - child_in_data.item_count = 0; - if (m_parent_event == nullptr) { - throw JException("Attempting to unfold without a valid parent event"); - } - if (m_parent_event->get()->GetLevel() != m_unfolder->GetLevel()) { - throw JException("JUnfolder: Expected parent with level %d, got %d", m_unfolder->GetLevel(), m_parent_event->get()->GetLevel()); - } - if (child->get()->GetLevel() != m_unfolder->GetChildLevel()) { - throw JException("JUnfolder: Expected child with level %d, got %d", m_unfolder->GetChildLevel(), child->get()->GetLevel()); - } - - auto status = m_unfolder->DoUnfold(*(m_parent_event->get()), *(child->get())); - - - // Join always succeeds (for now) - child->get()->SetParent(m_parent_event); - - LOG_DEBUG(m_logger) << "Unfold succeeded: Parent event = " << m_parent_event->get()->GetEventNumber() << ", child event = " << child->get()->GetEventNumber() << LOG_END; - // TODO: We'll need something more complicated for the streaming join case - - if (status == JEventUnfolder::Result::NextChildNextParent || status == JEventUnfolder::Result::KeepChildNextParent) { - LOG_DEBUG(m_logger) << "Unfold finished with parent event = " << m_parent_event->get()->GetEventNumber() << LOG_END; - m_ready_to_fetch_parent = true; - m_parent_event->get()->Release(); - m_parent_event = nullptr; - m_parent_in.min_item_count = 1; - m_parent_in.max_item_count = 1; - } - - child_out_data.items[0] = child; - child_out_data.item_count = 1; - - auto end_processing_time = std::chrono::steady_clock::now(); - size_t events_processed = push_all(parent_in_data, child_in_data, child_out_data); - - auto end_total_time = std::chrono::steady_clock::now(); - auto latency = (end_processing_time - start_processing_time); - auto overhead = (end_total_time - start_total_time) - latency; - - metrics.update(JArrowMetrics::Status::KeepGoing, events_processed, 1, latency, overhead); + m_parent_event = nullptr; + output_count = 0; + m_next_input_port = PARENT_IN; + status = JArrow::FireResult::KeepGoing; return; } - else { - auto end_total_time = std::chrono::steady_clock::now(); - metrics.update(JArrowMetrics::Status::ComeBackLater, 0, 1, std::chrono::milliseconds(0), end_total_time - start_total_time); + else if (result == JEventUnfolder::Result::NextChildKeepParent) { + m_child_event->SetParent(m_parent_event); + outputs[0] = {m_child_event, CHILD_OUT}; + output_count = 1; + m_child_event = nullptr; + m_next_input_port = CHILD_IN; + status = JArrow::FireResult::KeepGoing; + return; + } + else if (result == JEventUnfolder::Result::NextChildNextParent) { + m_child_event->SetParent(m_parent_event); + m_parent_event->Release(); // Decrement the reference count so that this can be recycled + outputs[0] = {m_child_event, CHILD_OUT}; + output_count = 1; + LOG_DEBUG(m_logger) << "Unfold finished with parent event = " << m_parent_event->GetEventNumber() << LOG_END; + m_child_event = nullptr; + m_parent_event = nullptr; + m_next_input_port = PARENT_IN; + status = JArrow::FireResult::KeepGoing; return; } + else { + throw JException("Unsupported (corrupt?) JEventUnfolder::Result"); + } } - }; diff --git a/src/libraries/JANA/Utils/JApplicationInspector.cc b/src/libraries/JANA/Utils/JApplicationInspector.cc index d75aac30a..26a520f71 100644 --- a/src/libraries/JANA/Utils/JApplicationInspector.cc +++ b/src/libraries/JANA/Utils/JApplicationInspector.cc @@ -1,10 +1,9 @@ #include "JApplicationInspector.h" -#include "JANA/Services/JComponentManager.h" #include "JANA/Components/JComponentSummary.h" #include "JANA/Topology/JTopologyBuilder.h" #include -#include +#include void PrintMenu() { @@ -32,9 +31,9 @@ void InspectTopology(JApplication* app) { } void Fire(JApplication* app, int arrow_id) { - auto engine = app->GetService(); - auto result = engine->execute_arrow(arrow_id); - std::cout << to_string(result) << std::endl; + auto engine = app->GetService(); + auto result = engine->Fire(arrow_id, 0); + std::cout << to_string(result); } void InspectComponents(JApplication* app) { @@ -79,10 +78,7 @@ void InspectCollection(JApplication* app, std::string collection_name) { void InspectApplication(JApplication* app) { - auto engine = app->GetService(); - engine->request_pause(); - engine->wait_until_paused(); - app->SetTimeoutEnabled(false); + auto engine = app->GetService(); PrintMenu(); while (true) { @@ -126,15 +122,16 @@ void InspectApplication(JApplication* app) { Fire(app, std::stoi(args[0])); } else if (token == "Resume" || token == "r") { - app->Run(false); + engine->RunTopology(); break; } else if ((token == "Scale" || token == "s") && (args.size() == 1)) { - app->Scale(std::stoi(args[0])); + engine->ScaleWorkers(std::stoi(args[0])); + engine->RunTopology(); break; } else if (token == "Quit" || token == "q") { - app->Quit(true); + engine->DrainTopology(); break; } else if (token == "Help" || token == "h") { diff --git a/src/libraries/JANA/Utils/JBacktrace.cc b/src/libraries/JANA/Utils/JBacktrace.cc new file mode 100644 index 000000000..7a76be559 --- /dev/null +++ b/src/libraries/JANA/Utils/JBacktrace.cc @@ -0,0 +1,112 @@ + +// Copyright 2020, Jefferson Science Associates, LLC. +// Subject to the terms in the LICENSE file found in the top-level directory. + + +#include "JBacktrace.h" + +#include +#include +#include +#include +#include +#include +#include +#include + + +void JBacktrace::Reset() { + m_ready = false; +} + +void JBacktrace::WaitForCapture() const { + while (!m_ready.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + +void JBacktrace::Capture(int frames_to_omit) { + m_frame_count = backtrace(m_buffer, MAX_FRAMES); + m_frames_to_omit = frames_to_omit; + m_ready.store(true, std::memory_order_release); +} + +void JBacktrace::Format(std::ostream& os) { + char** symbols = backtrace_symbols(m_buffer, m_frame_count); + // Skip the first few frames, which are inevitably signal handlers, JBacktrace ctors, or JException ctors + for (int i=m_frames_to_omit; i #include -#include -#include -#include -#include - - -inline std::ostream& make_backtrace(std::ostream& os) { - - try { - // Generate stack trace. - const int max_frames = 100; - void* backtrace_buffer[max_frames]; - int frame_count = backtrace(backtrace_buffer, max_frames); - char** symbols = backtrace_symbols(backtrace_buffer, frame_count); - // Skip the first two frames, because those are "make_backtrace" and "JException::JException" - for (int i=2; i - // We are going to get the symbol names from dl, so that we can demangle them - Dl_info dlinfo; - if (dladdr(backtrace_buffer[i], &dlinfo)) { // dladdr succeeded - if (dlinfo.dli_sname) { // dladdr found a corresponding symbol - int demangle_status = 0; - auto demangled_name = abi::__cxa_demangle(dlinfo.dli_sname, nullptr, nullptr, &demangle_status); +class JBacktrace { + static const int MAX_FRAMES = 100; + void* m_buffer[MAX_FRAMES]; + int m_frame_count = 0; + int m_frames_to_omit = 0; + std::atomic_bool m_ready {false}; - if (demangle_status == 0) { // Demangle succeeded, we have a symbol name! - name = std::string(demangled_name); - } - else { // Demangle failed, so we use mangled version - name = std::string(dlinfo.dli_sname); - } - free(demangled_name); - } - else { // Otherwise use the shared object name - name = std::string(dlinfo.dli_fname); - } - } - else { // dladdr failed for some reason, so we fall back on backtrace_symbols - name = symbols[i]; - } +public: + void WaitForCapture() const; + void Capture(int frames_to_omit=0); + void Reset(); + std::string ToString(); + void Format(std::ostream& os); + std::string AddrToLineInfo(const char* filename, size_t offset); - // Ignore frames where the name seems un-useful, particularly on macOS - //if( name.find("?") != string::npos ) continue; - //if( name == "0x0") continue; - //if( name == "_sigtramp" ) continue; - //if( name == "_pthread_body" ) continue; - //if( name == "thread_start" ) continue; +}; - os << std::string(i+5, ' ') + "`- " << name << " (" << std::hex << backtrace_buffer[i] << std::dec << ")" << std::endl; - } - - free(symbols); - } - catch (...) { - // If this fails (e.g. bad alloc), just keep going - } - return os; -} diff --git a/src/libraries/JANA/Utils/JCallGraphEntryMaker.h b/src/libraries/JANA/Utils/JCallGraphEntryMaker.h index 23448abb1..f0952148c 100644 --- a/src/libraries/JANA/Utils/JCallGraphEntryMaker.h +++ b/src/libraries/JANA/Utils/JCallGraphEntryMaker.h @@ -18,11 +18,13 @@ /// (and possibly other places). class JCallGraphEntryMaker{ public: + JCallGraphEntryMaker(JCallGraphRecorder &callgraphrecorder, JFactory *factory) : m_call_graph(callgraphrecorder), m_factory(factory){ m_call_graph.StartFactoryCall(m_factory->GetObjectName(), m_factory->GetTag()); } + JCallGraphEntryMaker(JCallGraphRecorder &callgraphrecorder, std::string name) : m_call_graph(callgraphrecorder) { - // (This is used mainly for JEventProcessors and called from JEventProcessorArrow::execute ) + // This is used mainly for JEventProcessors m_call_graph.StartFactoryCall(name, ""); } diff --git a/src/libraries/JANA/Utils/JCallGraphRecorder.h b/src/libraries/JANA/Utils/JCallGraphRecorder.h index 7fbdccc54..a17e98804 100644 --- a/src/libraries/JANA/Utils/JCallGraphRecorder.h +++ b/src/libraries/JANA/Utils/JCallGraphRecorder.h @@ -10,7 +10,6 @@ #include #include -#include // Note on tracking how/where Insert() objects came from. // diff --git a/src/libraries/JANA/Utils/JEventPool.h b/src/libraries/JANA/Utils/JEventPool.h deleted file mode 100644 index 3a3d6ebc5..000000000 --- a/src/libraries/JANA/Utils/JEventPool.h +++ /dev/null @@ -1,44 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#pragma once - -#include -#include -#include - - -class JEventPool : public JPool> { - - std::shared_ptr m_component_manager; - JEventLevel m_level; - -public: - inline JEventPool(std::shared_ptr component_manager, - size_t pool_size, - size_t location_count, - bool limit_total_events_in_flight, - JEventLevel level = JEventLevel::PhysicsEvent) - : JPool(pool_size, location_count, limit_total_events_in_flight) - , m_component_manager(component_manager) - , m_level(level) { - } - - void configure_item(std::shared_ptr* item) override { - (*item) = std::make_shared(); - m_component_manager->configure_event(**item); - item->get()->SetLevel(m_level); // This needs to happen _after_ configure_event - } - - void release_item(std::shared_ptr* item) override { - if (auto source = (*item)->GetJEventSource()) source->DoFinish(**item); - (*item)->mFactorySet->Release(); - (*item)->mInspector.Reset(); - (*item)->GetJCallGraphRecorder()->Reset(); - (*item)->Reset(); - } -}; - - diff --git a/src/libraries/JANA/Utils/JStringification.cc b/src/libraries/JANA/Utils/JStringification.cc index 364da2fd3..bf0986ab8 100644 --- a/src/libraries/JANA/Utils/JStringification.cc +++ b/src/libraries/JANA/Utils/JStringification.cc @@ -75,7 +75,7 @@ void JStringification::GetObjectSummaries(std::map } #endif }else{ - _DBG_<<"No factory found! object_name=" << object_name << std::endl; + LOG <<"No factory found! object_name=" << object_name << LOG_END; } } @@ -131,4 +131,4 @@ std::string JStringification::GetRootObjectMemberAsString(const TObject *tobj, c else if( type == "string" ) return GetAddrAsString(addr); return "unknown"; } -#endif // JANA2_HAVE_ROOT \ No newline at end of file +#endif // JANA2_HAVE_ROOT diff --git a/src/plugins/JTest/JTestCalibrationService.h b/src/plugins/JTest/JTestCalibrationService.h index 76dde7251..8c70c3fb2 100644 --- a/src/plugins/JTest/JTestCalibrationService.h +++ b/src/plugins/JTest/JTestCalibrationService.h @@ -5,9 +5,14 @@ #ifndef JANA2_JTESTCALIBRATIONSERVICE_H #define JANA2_JTESTCALIBRATIONSERVICE_H +#include + struct JTestCalibrationService: JService { + + Parameter m_calibration_value{this, "calibration_value", 7.0, "Dummy calibration value"}; + double getCalibration() { - return 7.0; + return *m_calibration_value; } }; diff --git a/src/plugins/JTest/JTestDataObjects.h b/src/plugins/JTest/JTestDataObjects.h index 0bcca09b6..5cc4a3ab6 100644 --- a/src/plugins/JTest/JTestDataObjects.h +++ b/src/plugins/JTest/JTestDataObjects.h @@ -45,4 +45,10 @@ struct JTestHistogramData : public JObject { JOBJECT_PUBLIC(JTestHistogramData) }; +struct JTestTrackAuxilliaryData : public JObject { + int something = 1; + float something2 = 2; + JOBJECT_PUBLIC(JTestTrackAuxilliaryData) +}; + #endif //JANA2_JTESTEVENTCONTEXTS_H diff --git a/src/plugins/JTest/JTestDisentangler.h b/src/plugins/JTest/JTestDisentangler.h index 155245cea..908f99a64 100644 --- a/src/plugins/JTest/JTestDisentangler.h +++ b/src/plugins/JTest/JTestDisentangler.h @@ -15,44 +15,39 @@ class JTestDisentangler : public JFactoryT { - size_t m_cputime_ms = 20; - size_t m_write_bytes = 500000; - double m_cputime_spread = 0.25; - double m_write_spread = 0.25; - JBenchUtils m_bench_utils = JBenchUtils(); + Parameter m_cputime_ms {this, "cputime_ms", 20, "Time spent during disentangling" }; + Parameter m_write_bytes {this, "bytes", 500000, "Bytes written during disentangling"}; + Parameter m_cputime_spread {this, "cputime_spread", 0.25, "Spread of time spent during disentangling"}; + Parameter m_write_spread {this, "bytes_spread", 0.25, "Spread of bytes written during disentangling"}; - std::shared_ptr m_calibration_service; + Service m_calibration_service {this}; -public: + JBenchUtils m_bench_utils; - void Init() override { - auto app = GetApplication(); - assert (app != nullptr); - app->SetDefaultParameter("jtest:disentangler_ms", m_cputime_ms, "Time spent during disentangling"); - app->SetDefaultParameter("jtest:disentangler_spread", m_cputime_spread, "Spread of time spent during disentangling"); - app->SetDefaultParameter("jtest:disentangler_bytes", m_write_bytes, "Bytes written during disentangling"); - app->SetDefaultParameter("jtest:disentangler_bytes_spread", m_write_spread, "Spread of bytes written during disentangling"); +public: - // Retrieve calibration service from JApp - m_calibration_service = app->GetService(); + JTestDisentangler() { + SetPrefix("jtest:disentangler"); + SetTypeName(NAME_OF_THIS); } void Process(const std::shared_ptr &aEvent) override { m_bench_utils.set_seed(aEvent->GetEventNumber(), NAME_OF_THIS); + // Read (large) entangled event data auto eed = aEvent->GetSingle(); m_bench_utils.read_memory(*eed->buffer); // Read calibration data - m_calibration_service->getCalibration(); + auto calib = m_calibration_service->getCalibration(); // Do a little bit of computation - m_bench_utils.consume_cpu_ms(m_cputime_ms, m_cputime_spread); + m_bench_utils.consume_cpu_ms(*m_cputime_ms + calib, *m_cputime_spread); // Write (large) event data auto ed = new JTestEventData; - m_bench_utils.write_memory(ed->buffer, m_write_bytes, m_write_spread); + m_bench_utils.write_memory(ed->buffer, *m_write_bytes, *m_write_spread); Insert(ed); } }; diff --git a/src/plugins/JTest/JTestParser.h b/src/plugins/JTest/JTestParser.h index 51bf6bd43..48855b7ef 100644 --- a/src/plugins/JTest/JTestParser.h +++ b/src/plugins/JTest/JTestParser.h @@ -14,16 +14,16 @@ #include - #include "JTestDataObjects.h" class JTestParser : public JEventSource { - size_t m_cputime_ms = 0; - size_t m_write_bytes = 2000000; - double m_cputime_spread = 0.25; - double m_write_spread = 0.25; + Parameter m_cputime_ms {this, "cputime_ms", 0, "Time spent during parsing" }; + Parameter m_write_bytes {this, "bytes", 2000000, "Bytes written during parsing"}; + Parameter m_cputime_spread {this, "cputime_spread", 0.25, "Spread of time spent during parsing"}; + Parameter m_write_spread {this, "bytes_spread", 0.25, "Spread of bytes written during parsing"}; + JBenchUtils m_bench_utils = JBenchUtils(); std::shared_ptr> m_latest_entangled_buffer; @@ -32,6 +32,7 @@ class JTestParser : public JEventSource { public: JTestParser() { + SetPrefix("jtest:parser"); SetTypeName(NAME_OF_THIS); SetCallbackStyle(CallbackStyle::ExpertMode); } @@ -40,14 +41,6 @@ class JTestParser : public JEventSource { return "JTest Fake Event Source"; } - void Init() override { - auto app = GetApplication(); - app->SetDefaultParameter("jtest:parser_ms", m_cputime_ms, "Time spent during parsing"); - app->SetDefaultParameter("jtest:parser_spread", m_cputime_spread, "Spread of time spent during parsing"); - app->SetDefaultParameter("jtest:parser_bytes", m_write_bytes, "Bytes written during parsing"); - app->SetDefaultParameter("jtest:parser_bytes_spread", m_write_spread, "Spread of bytes written during parsing"); - } - void Open() override { } @@ -62,11 +55,11 @@ class JTestParser : public JEventSource { if ((prev_m_events_generated % 40) == 0) { // "Read" new entangled event every 40 events m_latest_entangled_buffer = std::shared_ptr>(new std::vector); - m_bench_utils.write_memory(*m_latest_entangled_buffer, m_write_bytes, m_write_spread); + m_bench_utils.write_memory(*m_latest_entangled_buffer, *m_write_bytes, *m_write_spread); } // Spin the CPU - m_bench_utils.consume_cpu_ms(m_cputime_ms, m_cputime_spread); + m_bench_utils.consume_cpu_ms(*m_cputime_ms, *m_cputime_spread); // Emit a shared pointer to the entangled event buffer auto eec = new JTestEntangledEventData; diff --git a/src/plugins/JTest/JTestPlotter.h b/src/plugins/JTest/JTestPlotter.h index a37928b9d..88f0ed929 100644 --- a/src/plugins/JTest/JTestPlotter.h +++ b/src/plugins/JTest/JTestPlotter.h @@ -5,56 +5,44 @@ #ifndef JTestEventProcessor_h #define JTestEventProcessor_h -#include #include #include -#include "JTestTracker.h" -#include +#include "JTestDataObjects.h" class JTestPlotter : public JEventProcessor { - size_t m_cputime_ms = 0; - size_t m_write_bytes = 1000; - double m_cputime_spread = 0.25; - double m_write_spread = 0.25; - JBenchUtils m_bench_utils = JBenchUtils(); - std::mutex m_mutex; + Parameter m_cputime_ms {this, "cputime_ms", 0, "Time spent during plotting" }; + Parameter m_write_bytes {this, "bytes", 1000, "Bytes written during plotting"}; + Parameter m_cputime_spread {this, "cputime_spread", 0.25, "Spread of time spent during plotting"}; + Parameter m_write_spread {this, "bytes_spread", 0.25, "Spread of bytes written during plotting"}; + + Input m_track_data {this}; + Input m_track_aux_data {this}; + + JBenchUtils m_bench_utils; public: JTestPlotter() { SetPrefix("jtest:plotter"); SetTypeName(NAME_OF_THIS); + SetCallbackStyle(CallbackStyle::ExpertMode); } - void Init() override { - auto app = GetApplication(); - app->SetDefaultParameter("jtest:plotter_ms", m_cputime_ms, "Time spent during plotting"); - app->SetDefaultParameter("jtest:plotter_spread", m_cputime_spread, "Spread of time spent during plotting"); - app->SetDefaultParameter("jtest:plotter_bytes", m_write_bytes, "Bytes written during plotting"); - app->SetDefaultParameter("jtest:plotter_bytes_spread", m_write_spread, "Spread of bytes written during plotting"); - } + void Process(const JEvent& event) override { - void Process(const std::shared_ptr& event) override { + m_bench_utils.set_seed(event.GetEventNumber(), typeid(*this).name()); - m_bench_utils.set_seed(event->GetEventNumber(), typeid(*this).name()); // Read the track data - auto td = event->GetSingle(); - m_bench_utils.read_memory(td->buffer); - - // Read the extra data objects inserted by JTestTracker - event->Get(); - - // Everything that happens after here is in a critical section - std::lock_guard lock(m_mutex); + m_bench_utils.read_memory(m_track_data->at(0)->buffer); // Consume CPU - m_bench_utils.consume_cpu_ms(m_cputime_ms, m_cputime_spread); + m_bench_utils.consume_cpu_ms(*m_cputime_ms + m_track_aux_data->at(0)->something, *m_cputime_spread); // Write the histogram data auto hd = new JTestHistogramData; - m_bench_utils.write_memory(hd->buffer, m_write_bytes, m_write_spread); - event->Insert(hd); + m_bench_utils.write_memory(hd->buffer, *m_write_bytes, *m_write_spread); + event.Insert(hd); } }; diff --git a/src/plugins/JTest/JTestTracker.h b/src/plugins/JTest/JTestTracker.h index 3438d8fdb..99cddc1cc 100644 --- a/src/plugins/JTest/JTestTracker.h +++ b/src/plugins/JTest/JTestTracker.h @@ -13,26 +13,19 @@ class JTestTracker : public JFactoryT { - size_t m_cputime_ms = 200; - size_t m_write_bytes = 1000; - double m_cputime_spread = 0.25; - double m_write_spread = 0.25; - JBenchUtils m_bench_utils = JBenchUtils(); + Parameter m_segfault {this, "segfault", false, "Event #7 always segfaults" }; + Parameter m_timeout {this, "timeout", false, "Event #22 always times out" }; + Parameter m_cputime_ms {this, "cputime_ms", 200, "Time spent during tracking" }; + Parameter m_write_bytes {this, "bytes", 1000, "Bytes written during tracking"}; + Parameter m_cputime_spread {this, "cputime_spread", 0.25, "Spread of time spent during tracking"}; + Parameter m_write_spread {this, "bytes_spread", 0.25, "Spread of bytes written during tracking"}; -public: - - struct JTestTrackAuxilliaryData{ - int something = 1; - float something2 = 2; - }; + JBenchUtils m_bench_utils; - void Init() override { - - auto app = GetApplication(); - app->SetDefaultParameter("jtest:tracker_ms", m_cputime_ms, "Time spent during tracking"); - app->SetDefaultParameter("jtest:tracker_spread", m_cputime_spread, "Spread of time spent during tracking"); - app->SetDefaultParameter("jtest:tracker_bytes", m_write_bytes, "Bytes written during tracking"); - app->SetDefaultParameter("jtest:tracker_bytes_spread", m_write_spread, "Spread of bytes written during tracking"); +public: + JTestTracker() { + SetPrefix("jtest:tracker"); + SetTypeName(NAME_OF_THIS); } void Process(const std::shared_ptr &aEvent) override { @@ -43,11 +36,25 @@ class JTestTracker : public JFactoryT { m_bench_utils.read_memory(ed->buffer); // Do lots of computation - m_bench_utils.consume_cpu_ms(m_cputime_ms, m_cputime_spread); + if (*m_timeout) { + if (aEvent->GetEventNumber() == 22) { + m_bench_utils.consume_cpu_ms(1000000); + } + } + else { + m_bench_utils.consume_cpu_ms(*m_cputime_ms, *m_cputime_spread); + } + + + if(*m_segfault && aEvent->GetEventNumber() == 7) { + // Trigger a segfault on purpose + JTestTrackData* d = nullptr; + d->buffer[0] = 22; + } // Write (small) track data auto td = new JTestTrackData; - m_bench_utils.write_memory(td->buffer, m_write_bytes, m_write_spread); + m_bench_utils.write_memory(td->buffer, *m_write_bytes, *m_write_spread); Insert(td); // Insert some additional objects diff --git a/src/plugins/janacontrol/JControlZMQ.cc b/src/plugins/janacontrol/JControlZMQ.cc index ba6a16d23..f370ec54d 100644 --- a/src/plugins/janacontrol/JControlZMQ.cc +++ b/src/plugins/janacontrol/JControlZMQ.cc @@ -258,7 +258,7 @@ void JControlZMQ::ServerLoop() ss << "OK"; }else if( vals[0]=="resume" ){ //------------------ resume - _japp->Resume(); + _japp->Run(false); ss << "OK"; }else if( vals[0]=="debug_mode" ){ //------------------ debug_mode diff --git a/src/programs/perf_tests/PerfTests.cc b/src/programs/perf_tests/PerfTests.cc index 974a57731..4395a12af 100644 --- a/src/programs/perf_tests/PerfTests.cc +++ b/src/programs/perf_tests/PerfTests.cc @@ -27,7 +27,8 @@ int main() { // entangled "block of 40": dis * 40 auto params = new JParameterManager; - params->SetParameter("log:off", "JApplication,JPluginLoader,JArrowProcessingController,JArrow,JParameterManager"); + params->SetParameter("jana:loglevel", "off"); + // Log levels get set as soon as JApp gets constructed params->SetParameter("jtest:write_csv", false); params->SetParameter("jtest:parser_ms", 2); @@ -36,7 +37,7 @@ int main() { params->SetParameter("benchmark:resultsdir", "perftest_fake_halldrecon"); JApplication app(params); - auto logger = app.GetService()->get_logger("PerfTests"); + auto logger = params->GetLogger("PerfTests"); app.AddPlugin("JTest"); LOG_INFO(logger) << "Running JTest tuned to imitate halld_recon" << LOG_END; @@ -48,7 +49,8 @@ int main() { { auto params = new JParameterManager; - params->SetParameter("log:off", "JApplication,JPluginLoader,JArrowProcessingController,JArrow,JParameterManager"); + params->SetParameter("jana:loglevel", "off"); + // Log levels get set as soon as JApp gets constructed params->SetParameter("jtest:write_csv", false); @@ -75,7 +77,7 @@ int main() { params->SetParameter("benchmark:resultsdir", "perftest_pure_overhead"); JApplication app(params); - auto logger = app.GetService()->get_logger("PerfTests"); + auto logger = params->GetLogger("PerfTests"); app.AddPlugin("JTest"); LOG_INFO(logger) << "Running JTest with all sleeps and computations turned off" << LOG_END; @@ -90,9 +92,9 @@ int main() { ExampleHitCollection c; auto params = new JParameterManager; - params->SetParameter("log:off", "JApplication,JPluginLoader,JArrowProcessingController,JArrow"); // Log levels get set as soon as JApp gets constructed XD + params->SetParameter("jana:loglevel", "off"); JApplication app(params); - auto logger = app.GetService()->get_logger("PerfTests"); + auto logger = params->GetLogger("PerfTests"); // TODO: Add Podio sources, processors, and factories just like JTest LOG_INFO(logger) << "Running PODIO stress test" << LOG_END; JBenchmarker benchmarker(&app); diff --git a/src/programs/unit_tests/CMakeLists.txt b/src/programs/unit_tests/CMakeLists.txt index 9b34afecd..bc5284b17 100644 --- a/src/programs/unit_tests/CMakeLists.txt +++ b/src/programs/unit_tests/CMakeLists.txt @@ -1,12 +1,11 @@ set(TEST_SOURCES - Topology/ArrowTests.cc + Topology/JArrowTests.cc Topology/JPoolTests.cc Topology/MultiLevelTopologyTests.cc Topology/QueueTests.cc - Topology/SubeventTests.cc - Topology/TopologyTests.cc + #Topology/SubeventTests.cc Components/BarrierEventTests.cc Components/JObjectTests.cc @@ -21,6 +20,7 @@ set(TEST_SOURCES Components/JEventTests.cc Components/JFactoryDefTagsTests.cc Components/JFactoryTests.cc + Components/JFactoryGeneratorTests.cc Components/JMultiFactoryTests.cc Components/UnfoldTests.cc Components/UserExceptionTests.cc @@ -29,11 +29,10 @@ set(TEST_SOURCES Services/JParameterManagerTests.cc Services/JWiringServiceTests.cc - Engine/ArrowActivationTests.cc Engine/ScaleTests.cc - Engine/SchedulerTests.cc Engine/TerminationTests.cc Engine/TimeoutTests.cc + Engine/JExecutionEngineTests.cc Utils/JAutoactivableTests.cc Utils/JEventGroupTests.cc diff --git a/src/programs/unit_tests/Components/BarrierEventTests.cc b/src/programs/unit_tests/Components/BarrierEventTests.cc index 8dce85b45..e3a7a467c 100644 --- a/src/programs/unit_tests/Components/BarrierEventTests.cc +++ b/src/programs/unit_tests/Components/BarrierEventTests.cc @@ -2,18 +2,120 @@ // Copyright 2021, Jefferson Science Associates, LLC. // Subject to the terms in the LICENSE file found in the top-level directory. -#include "BarrierEventTests.h" +#include +#include +#include +#include "JANA/Utils/JBenchUtils.h" #include "catch.hpp" +int global_resource = 0; + + +struct BarrierSource : public JEventSource { + + JBenchUtils bench; + + BarrierSource() { + SetCallbackStyle(CallbackStyle::ExpertMode); + } + + void Open() override { + } + + Result Emit(JEvent& event) override { + + auto event_nr = GetEmittedEventCount() + 1; + event.SetEventNumber(event_nr); + + if (event_nr % 10 == 0) { + LOG_INFO(GetLogger()) << "Emitting barrier event " << event_nr << LOG_END; + event.SetSequential(true); + } + else { + LOG_INFO(GetLogger()) << "Emitting non-barrier event " << event_nr << LOG_END; + } + bench.consume_cpu_ms(50, 0, false); + return Result::Success; + } +}; + + +struct LegacyBarrierProcessor : public JEventProcessor { + + JBenchUtils bench; + std::mutex m_my_mutex; + + + void Process(const std::shared_ptr& event) override { + + bench.consume_cpu_ms(200, 0, true); + + std::lock_guard lock(m_my_mutex); + + if (event->GetSequential()) { + LOG_INFO(GetLogger()) << "Processing barrier event = " << event->GetEventNumber() << ", writing global var = " << global_resource+1 << LOG_END; + REQUIRE(global_resource == ((event->GetEventNumber() - 1) / 10)); + global_resource += 1; + } + else { + LOG_INFO(GetLogger()) << "Processing non-barrier event = " << event->GetEventNumber() << ", reading global var = " << global_resource << LOG_END; + REQUIRE(global_resource == (event->GetEventNumber() / 10)); + } + bench.consume_cpu_ms(100, 0, true); + } +}; + + +struct BarrierProcessor : public JEventProcessor { + + JBenchUtils bench; + + BarrierProcessor() { + SetCallbackStyle(CallbackStyle::ExpertMode); + } + void ProcessParallel(const JEvent&) override { + bench.consume_cpu_ms(200, 0, false); + } + + void Process(const JEvent& event) override { + + if (event.GetSequential()) { + LOG_INFO(GetLogger()) << "Processing barrier event = " << event.GetEventNumber() << ", writing global var = " << global_resource+1 << LOG_END; + REQUIRE(global_resource == ((event.GetEventNumber() - 1) / 10)); + global_resource += 1; + } + else { + LOG_INFO(GetLogger()) << "Processing non-barrier event = " << event.GetEventNumber() << ", reading global var = " << global_resource << LOG_END; + REQUIRE(global_resource == (event.GetEventNumber() / 10)); + } + bench.consume_cpu_ms(100, 0, false); + } +}; + + TEST_CASE("BarrierEventTests") { - SECTION("Basic Barrier") { - JApplication app; - app.Add(new BarrierProcessor); - app.Add(new BarrierSource); - app.SetParameterValue("nthreads", 4); - app.SetParameterValue("jana:event_source_chunksize", 1); - app.SetParameterValue("jana:event_processor_chunksize", 1); - app.Run(true); - } + global_resource = 0; + JApplication app; + app.Add(new BarrierProcessor); + app.Add(new BarrierSource); + app.SetParameterValue("nthreads", 4); + app.SetParameterValue("jana:nevents", 40); + //app.SetParameterValue("jana:log:show_threadstamp", true); + //app.SetParameterValue("jana:loglevel", "debug"); + app.Run(true); }; + +TEST_CASE("BarrierEventTests_Legacy") { + global_resource = 0; + JApplication app; + app.Add(new LegacyBarrierProcessor); + app.Add(new BarrierSource); + app.SetParameterValue("nthreads", 4); + app.SetParameterValue("jana:nevents", 40); + //app.SetParameterValue("jana:log:show_threadstamp", true); + //app.SetParameterValue("jana:loglevel", "debug"); + app.Run(true); +}; + + diff --git a/src/programs/unit_tests/Components/BarrierEventTests.h b/src/programs/unit_tests/Components/BarrierEventTests.h deleted file mode 100644 index 5843c9854..000000000 --- a/src/programs/unit_tests/Components/BarrierEventTests.h +++ /dev/null @@ -1,64 +0,0 @@ - -// Copyright 2021, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#ifndef JANA2_BARRIEREVENTTESTS_H -#define JANA2_BARRIEREVENTTESTS_H - -#include -#include -#include - -int global_resource = 0; - -class BarrierSource : public JEventSource { - int event_count=0; - -public: - - BarrierSource() { - SetCallbackStyle(CallbackStyle::ExpertMode); - } - - void Open() override { - } - - Result Emit(JEvent& event) override { - event_count++; - - if (event_count >= 100) { - return Result::FailureFinished; - } - - LOG << "Emitting event " << event_count << LOG_END; - event.SetEventNumber(event_count); - - if (event_count % 10 == 0) { - event.SetSequential(true); - } - return Result::Success; - } -}; - - - -struct BarrierProcessor : public JEventProcessor { - -public: - BarrierProcessor() { - SetCallbackStyle(CallbackStyle::ExpertMode); - } - void Process(const JEvent& event) override { - - if (event.GetSequential()) { - global_resource += 1; - LOG << "Barrier event = " << event.GetEventNumber() << ", writing global var = " << global_resource << LOG_END; - } - else { - LOG << "Processing non-barrier event = " << event.GetEventNumber() << ", reading global var = " << global_resource << LOG_END; - } - } -}; - -#endif //JANA2_BARRIEREVENTTESTS_H diff --git a/src/programs/unit_tests/Components/GetObjectsTests.cc b/src/programs/unit_tests/Components/GetObjectsTests.cc index 05e8399d2..e5476e881 100644 --- a/src/programs/unit_tests/Components/GetObjectsTests.cc +++ b/src/programs/unit_tests/Components/GetObjectsTests.cc @@ -2,48 +2,106 @@ // Subject to the terms in the LICENSE file found in the top-level directory. #include +#include +#include +#include "JANA/JFactoryGenerator.h" #include "catch.hpp" -struct Obj1 : public JObject { int data; }; -struct Obj2 : public JObject { int data; }; -struct Obj3 : public JObject { int data; }; -struct Obj4 : public JObject { int data; }; +namespace jana::components::getobjects_tests { -class Src : public JEventSource { +struct Obj : public JObject { int data; }; + +struct Src : public JEventSource { Src() { + EnableGetObjects(); SetCallbackStyle(CallbackStyle::ExpertMode); } - Result Emit(JEvent& event) override { - auto obj = new Obj1; - obj->data = 21; - event.Insert(obj); + Result Emit(JEvent&) override { return Result::Success; } bool GetObjects(const std::shared_ptr&, JFactory* fac) override { - if (fac->GetObjectName() == "Obj2") { - auto obj = new Obj2; + + LOG_INFO(GetLogger()) << "GetObjects: Fac has object name '" << fac->GetObjectName() << "' and type name '" << fac->GetTypeName() << "'" << LOG_END; + + //if (fac->GetObjectName() == "jana::components::getobjects_tests::Obj") { + auto typed_fac = dynamic_cast*>(fac); + if (typed_fac != nullptr) { + auto obj = new Obj; obj->data = 22; - fac->Insert(obj); + typed_fac->Insert(obj); return true; } return false; } }; -class Fac : public JFactoryT { +struct Fac : public JFactoryT { + void Process(const std::shared_ptr&) override { + auto obj = new Obj; + obj->data = 23; + Insert(obj); + } +}; + +struct RFac : public JFactoryT { + RFac() { + SetFactoryFlag(REGENERATE); + } void Process(const std::shared_ptr&) override { - auto obj = new Obj3; + auto obj = new Obj; obj->data = 23; Insert(obj); } }; -TEST_CASE("GetObjectsTests") { - JEvent event; - JFactorySet* fs = new JFactorySet; - fs->Add(new Fac); - event.SetFactorySet(fs); - event.GetJCallGraphRecorder()->SetEnabled(true); +struct Proc : public JEventProcessor { + bool from_getobjects=true; + void Process(const std::shared_ptr& event) override { + auto objs = event->Get(); + REQUIRE(objs.size() == 1); + if (from_getobjects) { + REQUIRE(objs[0]->data == 22); + } + else { + REQUIRE(objs[0]->data == 23); + } + } +}; + +TEST_CASE("GetObjectsTests_NoFac") { + JApplication app; + app.SetParameterValue("jana:loglevel", "warn"); + app.SetParameterValue("jana:nevents", 1); + app.Add(new Src); + auto proc = new Proc; + proc->from_getobjects = true; + app.Add(proc); + app.Add(new JFactoryGeneratorT>); + app.Run(); +} + +TEST_CASE("GetObjectsTests_OverrideFac") { + JApplication app; + app.SetParameterValue("jana:loglevel", "warn"); + app.SetParameterValue("jana:nevents", 1); + app.Add(new Src); + app.Add(new JFactoryGeneratorT); + auto proc = new Proc; + proc->from_getobjects = true; + app.Add(proc); + app.Run(); +} +TEST_CASE("GetObjectsTests_Regenerate") { + JApplication app; + app.SetParameterValue("jana:loglevel", "warn"); + app.SetParameterValue("jana:nevents", 1); + app.Add(new Src); + app.Add(new JFactoryGeneratorT); + auto proc = new Proc; + proc->from_getobjects = false; + app.Add(proc); + app.Run(); } +} // namespace ... diff --git a/src/programs/unit_tests/Components/JEventGetAllTests.cc b/src/programs/unit_tests/Components/JEventGetAllTests.cc index 4e66dbeb8..84531678c 100644 --- a/src/programs/unit_tests/Components/JEventGetAllTests.cc +++ b/src/programs/unit_tests/Components/JEventGetAllTests.cc @@ -159,7 +159,6 @@ TEST_CASE("JFactoryGetAs") { TEST_CASE("JEventGetAllChildren") { auto event = std::make_shared(); - event->SetFactorySet(new JFactorySet); SECTION("Single-item JEvent::Insert() can be retrieved via JEvent::GetAllChildren()") { auto b = new Base(22); diff --git a/src/programs/unit_tests/Components/JEventProcessorSequentialTests.cc b/src/programs/unit_tests/Components/JEventProcessorSequentialTests.cc index 31aaebb16..1137642de 100644 --- a/src/programs/unit_tests/Components/JEventProcessorSequentialTests.cc +++ b/src/programs/unit_tests/Components/JEventProcessorSequentialTests.cc @@ -49,10 +49,8 @@ TEST_CASE("JEventProcessorSequentialRootTests") { app.Add(new JEventSource()); app.SetParameterValue("nthreads", 4); app.SetParameterValue("jana:nevents", 4); - app.SetParameterValue("jana:event_source_chunksize", 1); - app.SetParameterValue("jana:event_processor_chunksize", 1); - app.SetParameterValue("log:global", "OFF"); - app.SetParameterValue("log:warn", "JScheduler,JArrow,JArrowProcessingController"); + //app.SetParameterValue("jana:loglevel", "debug"); + //app.SetParameterValue("jana:log:show_threadstamp", true); auto proc = new MyRootProcessor; app.Add(proc); app.Run(true); @@ -136,10 +134,8 @@ TEST_CASE("JEventProcessorSequentialTests") { app.Add(new JEventSource()); app.SetParameterValue("nthreads", 4); app.SetParameterValue("jana:nevents", 4); - app.SetParameterValue("jana:event_source_chunksize", 1); - app.SetParameterValue("jana:event_processor_chunksize", 1); - app.SetParameterValue("log:global", "OFF"); - app.SetParameterValue("log:warn", "JScheduler,JArrow,JArrowProcessingController"); + //app.SetParameterValue("jana:loglevel", "debug"); + //app.SetParameterValue("jana:log:show_threadstamp", 1); auto proc = new MySeqProcessor; app.Add(proc); app.Run(true); diff --git a/src/programs/unit_tests/Components/JEventProcessorTests.cc b/src/programs/unit_tests/Components/JEventProcessorTests.cc index 71d836602..ced408ab6 100644 --- a/src/programs/unit_tests/Components/JEventProcessorTests.cc +++ b/src/programs/unit_tests/Components/JEventProcessorTests.cc @@ -4,6 +4,9 @@ #include #include +namespace jana::jeventprocessortests { + + struct MyEventProcessor : public JEventProcessor { int init_count = 0; int process_count = 0; @@ -18,14 +21,17 @@ struct MyEventProcessor : public JEventProcessor { (*destroy_count)++; } void Init() override { + REQUIRE(GetApplication() != nullptr); LOG_INFO(GetLogger()) << "Init() called" << LOG_END; init_count++; } void Process(const JEvent&) override { + REQUIRE(GetApplication() != nullptr); process_count++; LOG_INFO(GetLogger()) << "Process() called" << LOG_END; } void Finish() override { + REQUIRE(GetApplication() != nullptr); LOG_INFO(GetLogger()) << "Finish() called" << LOG_END; finish_count++; } @@ -41,8 +47,7 @@ TEST_CASE("JEventProcessor_ExpertMode_ProcessCount") { { JApplication app; - app.SetParameterValue("log:global", "off"); - app.SetParameterValue("log:info", "MyEventProcessor"); + app.SetParameterValue("jana:loglevel", "off"); app.SetParameterValue("jana:nevents", 5); app.Add(new JEventSource); @@ -83,3 +88,110 @@ TEST_CASE("JEventProcessor_Exception") { REQUIRE(found_throw == true); } + + +struct SourceWithRunNumberChange : public JEventSource { + SourceWithRunNumberChange() { + SetCallbackStyle(CallbackStyle::ExpertMode); + } + Result Emit(JEvent& event) { + if (GetEmittedEventCount() < 2) { + event.SetRunNumber(48); + } + else { + event.SetRunNumber(49); + } + return Result::Success; + } +}; + + +struct ProcWithExceptionsAndLogging : public JEventProcessor { + Parameter except_on_init {this, "except_on_init", false, "Except on init"}; + Parameter except_on_beginrun {this, "except_on_beginrun", false, "Except on beginrun"}; + Parameter except_on_process {this, "except_on_process", false, "Except on process"}; + Parameter except_on_endrun {this, "except_on_endrun", false, "Except on endrun"}; + Parameter except_on_finish {this, "except_on_finish", false, "Except on finish"}; + + std::vector log; + + void Init() override { + LOG_INFO(GetLogger()) << "ProcWithExceptionsAndLogging::Init" << LOG_END; + log.push_back("init"); + if (*except_on_init) throw std::runtime_error("Mystery"); + } + void BeginRun(const std::shared_ptr&) override { + LOG_INFO(GetLogger()) << "ProcWithExceptionsAndLogging::BeginRun" << LOG_END; + log.push_back("beginrun"); + if (*except_on_beginrun) throw std::runtime_error("Mystery"); + } + void Process(const std::shared_ptr&) override { + LOG_INFO(GetLogger()) << "ProcWithExceptionsAndLogging::Process" << LOG_END; + log.push_back("process"); + if (*except_on_process) throw std::runtime_error("Mystery"); + } + void EndRun() override { + LOG_INFO(GetLogger()) << "ProcWithExceptionsAndLogging::EndRun" << LOG_END; + log.push_back("endrun"); + if (*except_on_endrun) throw std::runtime_error("Mystery"); + } + void Finish() override { + LOG_INFO(GetLogger()) << "ProcWithExceptionsAndLogging::Finish" << LOG_END; + log.push_back("finish"); + if (*except_on_finish) throw std::runtime_error("Mystery"); + } +}; + +TEST_CASE("JEventProcessor_CallbackSequence") { + JApplication app; + auto sut = new ProcWithExceptionsAndLogging; + app.Add(sut); + // JApplication takes ownership of sut, so our pointer will become invalid when JApplication is destroyed + + SECTION("NoRunNumber") { + app.Add(new JEventSource); + app.SetParameterValue("jana:nevents", 2); + app.Run(); + REQUIRE(sut->log.size() == 6); + REQUIRE(sut->log.at(0) == "init"); + REQUIRE(sut->log.at(1) == "beginrun"); + REQUIRE(sut->log.at(2) == "process"); + REQUIRE(sut->log.at(3) == "process"); + REQUIRE(sut->log.at(4) == "endrun"); + REQUIRE(sut->log.at(5) == "finish"); + } + SECTION("ConstantRunNumber") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.Run(); + REQUIRE(sut->log.size() == 6); + REQUIRE(sut->log.at(0) == "init"); + REQUIRE(sut->log.at(1) == "beginrun"); + REQUIRE(sut->log.at(2) == "process"); + REQUIRE(sut->log.at(3) == "process"); + REQUIRE(sut->log.at(4) == "endrun"); + REQUIRE(sut->log.at(5) == "finish"); + } + SECTION("MultipleRunNumbers") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 5); + app.Run(); + REQUIRE(sut->log.size() == 11); + REQUIRE(sut->log.at(0) == "init"); + REQUIRE(sut->log.at(1) == "beginrun"); + REQUIRE(sut->log.at(2) == "process"); + REQUIRE(sut->log.at(3) == "process"); + REQUIRE(sut->log.at(4) == "endrun"); + REQUIRE(sut->log.at(5) == "beginrun"); + REQUIRE(sut->log.at(6) == "process"); + REQUIRE(sut->log.at(7) == "process"); + REQUIRE(sut->log.at(8) == "process"); + REQUIRE(sut->log.at(9) == "endrun"); + REQUIRE(sut->log.at(10) == "finish"); + } +} + + + + +} // namespace jana::jeventprocessortests diff --git a/src/programs/unit_tests/Components/JEventSourceTests.cc b/src/programs/unit_tests/Components/JEventSourceTests.cc index 2ef433887..e0e79d9a4 100644 --- a/src/programs/unit_tests/Components/JEventSourceTests.cc +++ b/src/programs/unit_tests/Components/JEventSourceTests.cc @@ -10,13 +10,14 @@ struct MyEventSource : public JEventSource { size_t events_in_file = 5; void Open() override { + REQUIRE(GetApplication() != nullptr); LOG_INFO(GetLogger()) << "Open() called" << LOG_END; open_count++; } Result Emit(JEvent&) override { emit_count++; - - if (GetEventCount() >= events_in_file) { + REQUIRE(GetApplication() != nullptr); + if (GetEmittedEventCount() >= events_in_file) { LOG_INFO(GetLogger()) << "Emit() called, returning FailureFinished" << LOG_END; return Result::FailureFinished; } @@ -24,6 +25,7 @@ struct MyEventSource : public JEventSource { return Result::Success; } void Close() override { + REQUIRE(GetApplication() != nullptr); LOG_INFO(GetLogger()) << "Close() called" << LOG_END; close_count++; } @@ -36,8 +38,7 @@ TEST_CASE("JEventSource_ExpertMode_EmitCount") { sut->SetTypeName("MyEventSource"); JApplication app; - app.SetParameterValue("log:global", "off"); - app.SetParameterValue("log:info", "MyEventSource"); + app.SetParameterValue("jana:loglevel", "off"); app.Add(sut); SECTION("ShutsSelfOff") { @@ -45,7 +46,7 @@ TEST_CASE("JEventSource_ExpertMode_EmitCount") { app.Run(); REQUIRE(sut->open_count == 1); REQUIRE(sut->emit_count == 6); // Emit called 5 times successfully and fails on the 6th - REQUIRE(sut->GetEventCount() == 5); // Emits 5 events successfully (including skipped) + REQUIRE(sut->GetEmittedEventCount() == 5); // Emits 5 events successfully (including skipped) REQUIRE(sut->close_count == 1); } @@ -55,7 +56,7 @@ TEST_CASE("JEventSource_ExpertMode_EmitCount") { app.Run(); REQUIRE(sut->open_count == 1); REQUIRE(sut->emit_count == 3); // Emit called 3 times successfully - REQUIRE(sut->GetEventCount() == 3); // Nevents limit discovered outside Emit + REQUIRE(sut->GetEmittedEventCount() == 3); // Nevents limit discovered outside Emit REQUIRE(sut->close_count == 1); } @@ -65,7 +66,7 @@ TEST_CASE("JEventSource_ExpertMode_EmitCount") { app.Run(); REQUIRE(sut->open_count == 1); REQUIRE(sut->emit_count == 6); // Emit called 5 times successfully and fails on the 6th - REQUIRE(sut->GetEventCount() == 5); // 5 events successfully emitted, 3 of which were (presumably) skipped + REQUIRE(sut->GetEmittedEventCount() == 5); // 5 events successfully emitted, 3 of which were (presumably) skipped REQUIRE(sut->close_count == 1); } } diff --git a/src/programs/unit_tests/Components/JEventTests.cc b/src/programs/unit_tests/Components/JEventTests.cc index 5e536b5a6..c2c31bd9d 100644 --- a/src/programs/unit_tests/Components/JEventTests.cc +++ b/src/programs/unit_tests/Components/JEventTests.cc @@ -13,7 +13,6 @@ TEST_CASE("JEventInsertTests") { auto event = std::make_shared(); - event->SetFactorySet(new JFactorySet); SECTION("Single-item JEvent::Insert() can be retrieved via JEvent::Get()") { auto input = new FakeJObject(22); @@ -85,7 +84,6 @@ TEST_CASE("JEventInsertTests") { bool deleted = false; obj->deleted = &deleted; JEvent* event_ptr = new JEvent(); - event_ptr->SetFactorySet(new JFactorySet); event_ptr->Insert(obj, "tag"); REQUIRE(deleted == false); delete event_ptr; diff --git a/src/programs/unit_tests/Components/JFactoryDefTagsTests.cc b/src/programs/unit_tests/Components/JFactoryDefTagsTests.cc index a85e93455..20d610f74 100644 --- a/src/programs/unit_tests/Components/JFactoryDefTagsTests.cc +++ b/src/programs/unit_tests/Components/JFactoryDefTagsTests.cc @@ -93,10 +93,7 @@ TEST_CASE("MediumDefTags") { app.Add(new JFactoryGeneratorT); app.Add(new JFactoryGeneratorT); app.SetParameterValue("DEFTAG:deftagstest::Obj", "tagB"); - app.Initialize(); - auto event = std::make_shared(); - auto jcm = app.GetService(); - jcm->configure_event(*event); + auto event = std::make_shared(&app); auto objs = event->Get(); REQUIRE(objs[0]->E == 33.3); } diff --git a/src/programs/unit_tests/Components/JFactoryGeneratorTests.cc b/src/programs/unit_tests/Components/JFactoryGeneratorTests.cc new file mode 100644 index 000000000..dcbaa10f1 --- /dev/null +++ b/src/programs/unit_tests/Components/JFactoryGeneratorTests.cc @@ -0,0 +1,42 @@ + + +#include +#include +#include +#include + + +// ----------------------------------- +// UserDefined +// ----------------------------------- +namespace jana::components::jfactorygeneratortests_userdefined { + +struct MyData { int x; }; + +class MyFac : public JFactoryT { + void Init() override { + GetApplication(); // This will throw if nullptr + } + void Process(const std::shared_ptr&) override { + std::vector results; + results.push_back(new MyData {22}); + Set(results); + } +}; + +class MyFacGen : public JFactoryGenerator { + + void GenerateFactories(JFactorySet *factory_set) override { + factory_set->Add(new MyFac); + } +}; + +TEST_CASE("JFactoryGeneratorTests_UserDefined") { + JApplication app; + app.Add(new MyFacGen()); + auto event = std::make_shared(&app); + + auto data = event->Get(); + REQUIRE(data.at(0)->x == 22); +} +} diff --git a/src/programs/unit_tests/Components/JFactoryTests.cc b/src/programs/unit_tests/Components/JFactoryTests.cc index 436edbc79..07ada4b21 100644 --- a/src/programs/unit_tests/Components/JFactoryTests.cc +++ b/src/programs/unit_tests/Components/JFactoryTests.cc @@ -3,6 +3,8 @@ // Subject to the terms in the LICENSE file found in the top-level directory. +#include "JANA/Components/JComponentFwd.h" +#include "JANA/JFactory.h" #include "catch.hpp" #include "JFactoryTests.h" @@ -39,22 +41,22 @@ TEST_CASE("JFactoryTests") { } -/* SECTION("If no factory is present and nothing inserted, GetObjects called") { // The event hasn't been given any DummyFactory, nor has anything been inserted. // Instead, the JEvent knows to go down to the JEventSource. auto event = std::make_shared(); - DummySource sut; + JFactoryTestDummySource sut; + + // Empty factory needed for GetObjects() to work + event->GetFactorySet()->Add(new JFactoryT()); event->SetJEventSource(&sut); - event->SetFactorySet(new JFactorySet()); - auto data = event->Get(""); + auto data = event->Get(""); REQUIRE(data[0]->data == 8); REQUIRE(data[1]->data == 88); } -*/ SECTION("ChangeRun called only when run number changes") { auto event = std::make_shared(); JFactoryTestDummyFactory sut; @@ -249,6 +251,54 @@ TEST_CASE("JFactory_Exception") { REQUIRE(found_throw == true); } +struct ExceptingInitFactory : public JFactoryT { + void Init() override { + throw std::runtime_error("Exception in Init"); + } + void Process(const std::shared_ptr&) override { + throw std::runtime_error("Exception in Process"); + } +}; + +TEST_CASE("JFactoryTests_ExceptingInitCalledTwice") { + JApplication app; + app.SetParameterValue("jana:loglevel", "error"); + app.Add(new JFactoryGeneratorT()); + auto event = std::make_shared(&app); + + bool found_throw = false; + try { + event->Get(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::Init"); + REQUIRE(ex.message == "Exception in Init"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "ExceptingInitFactory"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); + + // Second time around should except from Init again, NOT Process() + found_throw = false; + try { + event->Get(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::Init"); + REQUIRE(ex.message == "Exception in Init"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "ExceptingInitFactory"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); +} + + struct MyLoggedFactory : public JFactoryT { MyLoggedFactory() { SetPrefix("myfac"); @@ -262,9 +312,258 @@ TEST_CASE("JFactory_Logger") { JApplication app; app.Add(new JEventSource); app.Add(new JFactoryGeneratorT()); - app.SetParameterValue("log:debug", "myfac"); + app.SetParameterValue("jana:loglevel", "off"); + app.SetParameterValue("myfac:loglevel", "debug"); app.SetParameterValue("jana:nevents", "1"); app.SetParameterValue("autoactivate", "JFactoryTestDummyObject"); app.Run(); } +std::vector factory_with_finish_log; + +struct SourceWithRunNumberChange : public JEventSource { + SourceWithRunNumberChange() { + SetCallbackStyle(CallbackStyle::ExpertMode); + } + Result Emit(JEvent& event) { + if (GetEmittedEventCount() < 2) { + event.SetRunNumber(48); + } + else { + event.SetRunNumber(49); + } + return Result::Success; + } +}; + +struct FactoryWithFinish : public JFactoryT { + Parameter except_on_init {this, "except_on_init", false, "Except on init"}; + Parameter except_on_beginrun {this, "except_on_beginrun", false, "Except on beginrun"}; + Parameter except_on_process {this, "except_on_process", false, "Except on process"}; + Parameter except_on_endrun {this, "except_on_endrun", false, "Except on endrun"}; + Parameter except_on_finish {this, "except_on_finish", false, "Except on finish"}; + + void Init() override { + LOG_INFO(GetLogger()) << "FactoryWithFinish::Init: " << this << LOG_END; + factory_with_finish_log.push_back("init"); + if (*except_on_init) throw std::runtime_error("Mystery"); + } + void BeginRun(const std::shared_ptr&) override { + LOG_INFO(GetLogger()) << "FactoryWithFinish::BeginRun: " << this << LOG_END; + factory_with_finish_log.push_back("beginrun"); + if (*except_on_beginrun) throw std::runtime_error("Mystery"); + } + void Process(const std::shared_ptr&) override { + LOG_INFO(GetLogger()) << "FactoryWithFinish::Process: " << this << LOG_END; + factory_with_finish_log.push_back("process"); + if (*except_on_process) throw std::runtime_error("Mystery"); + } + void EndRun() override { + LOG_INFO(GetLogger()) << "FactoryWithFinish::EndRun: " << this << LOG_END; + factory_with_finish_log.push_back("endrun"); + if (*except_on_endrun) throw std::runtime_error("Mystery"); + } + void Finish() override { + LOG_INFO(GetLogger()) << "FactoryWithFinish::Finish: " << this << LOG_END; + factory_with_finish_log.push_back("finish"); + if (*except_on_finish) throw std::runtime_error("Mystery"); + } +}; + +TEST_CASE("JFactory_CallbackSequence") { + JApplication app; + app.Add(new JFactoryGeneratorT()); + app.SetParameterValue("autoactivate", "JFactoryTestDummyObject"); + app.SetParameterValue("jana:max_inflight_events", 1); + + SECTION("NoRunNumber") { + app.Add(new JEventSource); + app.SetParameterValue("jana:nevents", 2); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + app.Run(); + for (auto& s : factory_with_finish_log) { + std::cout << s << std::endl; + } + REQUIRE(factory_with_finish_log.size() == 6); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + REQUIRE(factory_with_finish_log.at(2) == "process"); + REQUIRE(factory_with_finish_log.at(3) == "process"); + REQUIRE(factory_with_finish_log.at(4) == "endrun"); + REQUIRE(factory_with_finish_log.at(5) == "finish"); + } + SECTION("ConstantRunNumber") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + app.Run(); + REQUIRE(factory_with_finish_log.size() == 6); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + REQUIRE(factory_with_finish_log.at(2) == "process"); + REQUIRE(factory_with_finish_log.at(3) == "process"); + REQUIRE(factory_with_finish_log.at(4) == "endrun"); + REQUIRE(factory_with_finish_log.at(5) == "finish"); + } + SECTION("MultipleRunNumbers") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 5); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + app.Run(); + REQUIRE(factory_with_finish_log.size() == 11); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + REQUIRE(factory_with_finish_log.at(2) == "process"); + REQUIRE(factory_with_finish_log.at(3) == "process"); + REQUIRE(factory_with_finish_log.at(4) == "endrun"); + REQUIRE(factory_with_finish_log.at(5) == "beginrun"); + REQUIRE(factory_with_finish_log.at(6) == "process"); + REQUIRE(factory_with_finish_log.at(7) == "process"); + REQUIRE(factory_with_finish_log.at(8) == "process"); + REQUIRE(factory_with_finish_log.at(9) == "endrun"); + REQUIRE(factory_with_finish_log.at(10) == "finish"); + } +} + +TEST_CASE("JFactory_ExceptionHandling") { + JApplication app; + app.Add(new JFactoryGeneratorT()); + app.SetParameterValue("autoactivate", "JFactoryTestDummyObject"); + app.SetParameterValue("jana:max_inflight_events", 1); + + SECTION("ExceptOnInit") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.SetParameterValue("JFactoryTestDummyObject:except_on_init", true); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + bool found_throw = false; + try { + app.Run(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::Init"); + REQUIRE(ex.message == "Mystery"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "FactoryWithFinish"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); + REQUIRE(factory_with_finish_log.size() == 1); + REQUIRE(factory_with_finish_log.at(0) == "init"); + } + + SECTION("ExceptOnBeginRun") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.SetParameterValue("JFactoryTestDummyObject:except_on_beginrun", true); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + bool found_throw = false; + try { + app.Run(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::BeginRun"); + REQUIRE(ex.message == "Mystery"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "FactoryWithFinish"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); + REQUIRE(factory_with_finish_log.size() == 2); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + } + + SECTION("ExceptOnProcess") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.SetParameterValue("JFactoryTestDummyObject:except_on_process", true); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + bool found_throw = false; + try { + app.Run(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::Process"); + REQUIRE(ex.message == "Mystery"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "FactoryWithFinish"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); + REQUIRE(factory_with_finish_log.size() == 3); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + REQUIRE(factory_with_finish_log.at(2) == "process"); + } + + SECTION("ExceptOnEndRun") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.SetParameterValue("JFactoryTestDummyObject:except_on_endrun", true); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + bool found_throw = false; + try { + app.Run(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::EndRun"); + REQUIRE(ex.message == "Mystery"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "FactoryWithFinish"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); + REQUIRE(factory_with_finish_log.size() == 5); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + REQUIRE(factory_with_finish_log.at(2) == "process"); + REQUIRE(factory_with_finish_log.at(3) == "process"); + REQUIRE(factory_with_finish_log.at(4) == "endrun"); + } + SECTION("ExceptOnFinish") { + app.Add(new SourceWithRunNumberChange); + app.SetParameterValue("jana:nevents", 2); + app.SetParameterValue("JFactoryTestDummyObject:except_on_finish", true); + app.Initialize(); // This init()s a throwaway JFactoryT, which we immediately clear from the log + factory_with_finish_log.clear(); + bool found_throw = false; + try { + app.Run(); + } + catch(JException& ex) { + LOG << ex << LOG_END; + REQUIRE(ex.function_name == "JFactory::Finish"); + REQUIRE(ex.message == "Mystery"); + REQUIRE(ex.exception_type == "std::runtime_error"); + REQUIRE(ex.type_name == "FactoryWithFinish"); + REQUIRE(ex.instance_name == "JFactoryTestDummyObject"); + found_throw = true; + } + REQUIRE(found_throw == true); + REQUIRE(factory_with_finish_log.size() == 6); + REQUIRE(factory_with_finish_log.at(0) == "init"); + REQUIRE(factory_with_finish_log.at(1) == "beginrun"); + REQUIRE(factory_with_finish_log.at(2) == "process"); + REQUIRE(factory_with_finish_log.at(3) == "process"); + REQUIRE(factory_with_finish_log.at(4) == "endrun"); + REQUIRE(factory_with_finish_log.at(5) == "finish"); + } +} + + diff --git a/src/programs/unit_tests/Components/JFactoryTests.h b/src/programs/unit_tests/Components/JFactoryTests.h index eb3c98801..e700b86c4 100644 --- a/src/programs/unit_tests/Components/JFactoryTests.h +++ b/src/programs/unit_tests/Components/JFactoryTests.h @@ -71,6 +71,7 @@ struct JFactoryTestDummySource: public JEventSource { JFactoryTestDummySource() { SetCallbackStyle(CallbackStyle::ExpertMode); + EnableGetObjects(); } Result Emit(JEvent&) override { diff --git a/src/programs/unit_tests/Components/JMultiFactoryTests.cc b/src/programs/unit_tests/Components/JMultiFactoryTests.cc index bba559bbc..df201fb1f 100644 --- a/src/programs/unit_tests/Components/JMultiFactoryTests.cc +++ b/src/programs/unit_tests/Components/JMultiFactoryTests.cc @@ -37,6 +37,7 @@ struct MyMultifactory : public JMultifactory { } void Process(const std::shared_ptr&) override { + REQUIRE(GetApplication() != nullptr); m_process_call_count += 1; std::vector as; std::vector bs; @@ -81,10 +82,7 @@ TEST_CASE("MultiFactoryTests") { SECTION("Multifactories work with JFactoryGeneratorT") { app.Add(new JFactoryGeneratorT()); - app.Initialize(); - auto jcm = app.GetService(); auto event = std::make_shared(&app); - jcm->configure_event(*event); auto as = event->Get("first"); REQUIRE(as.size() == 2); REQUIRE(as[1]->x == 5.5); @@ -92,10 +90,7 @@ TEST_CASE("MultiFactoryTests") { SECTION("Test that multifactory Process() is only called once") { app.Add(new JFactoryGeneratorT()); - app.Initialize(); - auto jcm = app.GetService(); auto event = std::make_shared(&app); - jcm->configure_event(*event); auto helper_fac = dynamic_cast*>(event->GetFactory("first")); REQUIRE(helper_fac != nullptr); diff --git a/src/programs/unit_tests/Components/NEventNSkipTests.cc b/src/programs/unit_tests/Components/NEventNSkipTests.cc index 03ebbc690..3bf880cfe 100644 --- a/src/programs/unit_tests/Components/NEventNSkipTests.cc +++ b/src/programs/unit_tests/Components/NEventNSkipTests.cc @@ -83,7 +83,6 @@ TEST_CASE("NEventNSkipTests") { TEST_CASE("JEventSourceArrow with multiple JEventSources") { JApplication app; - app.SetParameterValue("log:info","JArrow,JArrowProcessingController"); auto source1 = new NEventNSkipBoundedSource(); auto source2 = new NEventNSkipBoundedSource(); auto source3 = new NEventNSkipBoundedSource(); @@ -111,9 +110,9 @@ TEST_CASE("JEventSourceArrow with multiple JEventSources") { REQUIRE(source1->close_count == 1); REQUIRE(source2->close_count == 1); REQUIRE(source3->close_count == 1); - REQUIRE(source1->GetEventCount() == 9); - REQUIRE(source2->GetEventCount() == 13); - REQUIRE(source3->GetEventCount() == 7); + REQUIRE(source1->GetEmittedEventCount() == 9); + REQUIRE(source2->GetEmittedEventCount() == 13); + REQUIRE(source3->GetEmittedEventCount() == 7); REQUIRE(app.GetNEventsProcessed() == 9+13+7); } @@ -137,9 +136,9 @@ TEST_CASE("JEventSourceArrow with multiple JEventSources") { REQUIRE(source1->close_count == 1); REQUIRE(source2->close_count == 1); REQUIRE(source3->close_count == 1); - REQUIRE(source1->GetEventCount() == 9); // 3 dropped, 6 emitted - REQUIRE(source2->GetEventCount() == 12); // 3 dropped, 9 emitted - REQUIRE(source3->GetEventCount() == 7); // 3 dropped, 4 emitted + REQUIRE(source1->GetEmittedEventCount() == 9); // 3 dropped, 6 emitted + REQUIRE(source2->GetEmittedEventCount() == 12); // 3 dropped, 9 emitted + REQUIRE(source3->GetEmittedEventCount() == 7); // 3 dropped, 4 emitted REQUIRE(app.GetNEventsProcessed() == 19); } @@ -165,9 +164,9 @@ TEST_CASE("JEventSourceArrow with multiple JEventSources") { REQUIRE(source1->close_count == 1); REQUIRE(source2->close_count == 1); REQUIRE(source3->close_count == 1); - REQUIRE(source1->GetEventCount() == 6); // 2 dropped, 4 emitted - REQUIRE(source2->GetEventCount() == 13); // 13 emitted - REQUIRE(source3->GetEventCount() == 4); // 4 emitted + REQUIRE(source1->GetEmittedEventCount() == 6); // 2 dropped, 4 emitted + REQUIRE(source2->GetEmittedEventCount() == 13); // 13 emitted + REQUIRE(source3->GetEmittedEventCount() == 4); // 4 emitted REQUIRE(app.GetNEventsProcessed() == 21); } diff --git a/src/programs/unit_tests/Components/PodioTests.cc b/src/programs/unit_tests/Components/PodioTests.cc index c0bedb560..50b0a0d4a 100644 --- a/src/programs/unit_tests/Components/PodioTests.cc +++ b/src/programs/unit_tests/Components/PodioTests.cc @@ -4,6 +4,7 @@ #include #include #include +#include namespace podiotests { @@ -150,11 +151,9 @@ struct TestFac : public JFactoryPodioT { TEST_CASE("JFactoryPodioT::Init gets called") { JApplication app; + app.Add(new JFactoryGeneratorT()); auto event = std::make_shared(&app); - auto fs = new JFactorySet; - fs->Add(new jana2_tests_podiotests_init::TestFac); - event->SetFactorySet(fs); - event->GetFactorySet()->Release(); // Simulate a trip to the JEventPool + event->Clear(); // Simulate a trip to the JEventPool auto r = event->GetCollectionBase("clusters"); REQUIRE(r != nullptr); diff --git a/src/programs/unit_tests/Components/UnfoldTests.cc b/src/programs/unit_tests/Components/UnfoldTests.cc index 59a0a7a87..0cf7f7ba4 100644 --- a/src/programs/unit_tests/Components/UnfoldTests.cc +++ b/src/programs/unit_tests/Components/UnfoldTests.cc @@ -6,8 +6,6 @@ namespace jana { namespace unfoldtests { -using EventT = std::shared_ptr; - struct TestUnfolder : public JEventUnfolder { mutable std::vector preprocessed_event_nrs; @@ -46,31 +44,31 @@ TEST_CASE("UnfoldTests_Basic") { app.Initialize(); auto jcm = app.GetService(); - JEventPool parent_pool {jcm, 5, 1, true, JEventLevel::Timeslice}; // size=5, locations=1, limit_total_events_in_flight=true - JEventPool child_pool {jcm, 5, 1, true, JEventLevel::PhysicsEvent}; - JMailbox parent_queue {3}; // size - JMailbox child_queue {3}; - - parent_pool.init(); - child_pool.init(); + JEventPool parent_pool {jcm, 5, 1, JEventLevel::Timeslice}; + JEventPool child_pool {jcm, 5, 1, JEventLevel::PhysicsEvent}; + JEventQueue parent_queue {3, 1}; + JEventQueue child_queue {3, 1}; - auto ts1 = parent_pool.get(); - (*ts1)->SetEventNumber(17); + auto ts1 = parent_pool.Pop(0); + ts1->SetEventNumber(17); - auto ts2 = parent_pool.get(); - (*ts2)->SetEventNumber(28); + auto ts2 = parent_pool.Pop(0); + ts2->SetEventNumber(28); - parent_queue.try_push(&ts1, 1); - parent_queue.try_push(&ts2, 1); + parent_queue.Push(ts1, 0); + parent_queue.Push(ts2, 0); TestUnfolder unfolder; - JUnfoldArrow arrow("sut", &unfolder, &parent_queue, &child_pool, &child_queue); + JUnfoldArrow arrow("sut", &unfolder); + arrow.attach(&parent_queue, JUnfoldArrow::PARENT_IN); + arrow.attach(&child_pool, JUnfoldArrow::CHILD_IN); + arrow.attach(&child_queue, JUnfoldArrow::CHILD_OUT); - JArrowMetrics m; arrow.initialize(); - arrow.execute(m, 0); - REQUIRE(m.get_last_status() == JArrowMetrics::Status::KeepGoing); - REQUIRE(child_queue.size() == 1); + arrow.execute( 0); // First call to execute() picks up the parent and exits early + auto result = arrow.execute( 0); // Second call to execute() picks up the child, calls Unfold(), and emits the newly parented child + REQUIRE(result == JArrow::FireResult::KeepGoing); + REQUIRE(child_queue.GetSize(0) == 1); REQUIRE(unfolder.preprocessed_event_nrs.size() == 0); REQUIRE(unfolder.unfolded_parent_nrs.size() == 1); REQUIRE(unfolder.unfolded_parent_nrs[0] == 17); @@ -86,115 +84,113 @@ TEST_CASE("FoldArrowTests") { JApplication app; app.Initialize(); auto jcm = app.GetService(); - // We only use these to obtain preconfigured JEvents - JEventPool parent_pool {jcm, 5, 1, true, JEventLevel::Timeslice}; // size=5, locations=1, limit_total_events_in_flight=true - JEventPool child_pool {jcm, 5, 1, true, JEventLevel::PhysicsEvent}; - parent_pool.init(); - child_pool.init(); - + JEventPool parent_pool {jcm, 5, 1, JEventLevel::Timeslice}; + JEventPool child_pool {jcm, 5, 1, JEventLevel::PhysicsEvent}; // We set up our test cases by putting events on these queues - JMailbox*> child_in; - JMailbox*> child_out; - JMailbox*> parent_out; - - JFoldArrow arrow("sut", JEventLevel::Timeslice, JEventLevel::PhysicsEvent, &child_in, &child_out, &parent_out); - JArrowMetrics metrics; + JEventQueue child_in(5, 1); + JEventQueue child_out(5, 1); + JEventQueue parent_out(5, 1); + + JFoldArrow arrow("sut", JEventLevel::Timeslice, JEventLevel::PhysicsEvent); + arrow.attach(&child_in, JFoldArrow::CHILD_IN); + arrow.attach(&child_out, JFoldArrow::CHILD_OUT); + arrow.attach(&parent_out, JFoldArrow::PARENT_OUT); arrow.initialize(); SECTION("One-to-one relationship between timeslices and events") { - auto ts1 = parent_pool.get(); - (*ts1)->SetEventNumber(17); - REQUIRE(ts1->get()->GetLevel() == JEventLevel::Timeslice); + auto ts1 = parent_pool.Pop(0); + ts1->SetEventNumber(17); + REQUIRE(ts1->GetLevel() == JEventLevel::Timeslice); - auto ts2 = parent_pool.get(); - (*ts2)->SetEventNumber(28); + auto ts2 = parent_pool.Pop(0); + ts2->SetEventNumber(28); - auto evt1 = child_pool.get(); - (*evt1)->SetEventNumber(111); + auto evt1 = child_pool.Pop(0); + evt1->SetEventNumber(111); - auto evt2 = child_pool.get(); - (*evt2)->SetEventNumber(112); + auto evt2 = child_pool.Pop(0); + evt2->SetEventNumber(112); - evt1->get()->SetParent(ts1); - ts1->get()->Release(); // One-to-one - child_in.try_push(&evt1, 1, 0); + evt1->SetParent(ts1); + ts1->Release(); // One-to-one + child_in.Push(evt1, 0); - evt2->get()->SetParent(ts2); - ts2->get()->Release(); // One-to-one - child_in.try_push(&evt2, 1, 0); + evt2->SetParent(ts2); + ts2->Release(); // One-to-one + child_in.Push(evt2, 0); - arrow.execute(metrics, 0); + arrow.execute(0); - REQUIRE(child_in.size() == 1); - REQUIRE(child_out.size() == 1); - REQUIRE(parent_out.size() == 1); + REQUIRE(child_in.GetSize(0) == 1); + REQUIRE(child_out.GetSize(0) == 1); + REQUIRE(parent_out.GetSize(0) == 1); } SECTION("One-to-two relationship between timeslices and events") { - auto ts1 = parent_pool.get(); - (*ts1)->SetEventNumber(17); - REQUIRE(ts1->get()->GetLevel() == JEventLevel::Timeslice); + auto ts1 = parent_pool.Pop(0); + ts1->SetEventNumber(17); + REQUIRE(ts1->GetLevel() == JEventLevel::Timeslice); - auto ts2 = parent_pool.get(); - (*ts2)->SetEventNumber(28); + auto ts2 = parent_pool.Pop(0); + ts2->SetEventNumber(28); - auto evt1 = child_pool.get(); - (*evt1)->SetEventNumber(111); + auto evt1 = child_pool.Pop(0); + evt1->SetEventNumber(111); - auto evt2 = child_pool.get(); - (*evt2)->SetEventNumber(112); + auto evt2 = child_pool.Pop(0); + evt2->SetEventNumber(112); - auto evt3 = child_pool.get(); - (*evt3)->SetEventNumber(113); + auto evt3 = child_pool.Pop(0); + evt3->SetEventNumber(113); - auto evt4 = child_pool.get(); - (*evt4)->SetEventNumber(114); + auto evt4 = child_pool.Pop(0); + evt4->SetEventNumber(114); - evt1->get()->SetParent(ts1); - evt2->get()->SetParent(ts1); - ts1->get()->Release(); // One-to-two + evt1->SetParent(ts1); + evt2->SetParent(ts1); + ts1->Release(); // One-to-two - evt3->get()->SetParent(ts2); - evt4->get()->SetParent(ts2); - ts2->get()->Release(); // One-to-two + evt3->SetParent(ts2); + evt4->SetParent(ts2); + ts2->Release(); // One-to-two - child_in.try_push(&evt1, 1, 0); - child_in.try_push(&evt2, 1, 0); - child_in.try_push(&evt3, 1, 0); - child_in.try_push(&evt4, 1, 0); + child_in.Push(evt1, 0); + child_in.Push(evt2, 0); + child_in.Push(evt3, 0); + child_in.Push(evt4, 0); - arrow.execute(metrics, 0); + arrow.execute(0); - REQUIRE(child_in.size() == 3); - REQUIRE(child_out.size() == 1); - REQUIRE(parent_out.size() == 0); + REQUIRE(child_in.GetSize(0) == 3); + REQUIRE(child_out.GetSize(0) == 1); + REQUIRE(parent_out.GetSize(0) == 0); - arrow.execute(metrics, 0); + arrow.execute(0); - REQUIRE(child_in.size() == 2); - REQUIRE(child_out.size() == 2); - REQUIRE(parent_out.size() == 1); + REQUIRE(child_in.GetSize(0) == 2); + REQUIRE(child_out.GetSize(0) == 2); + REQUIRE(parent_out.GetSize(0) == 1); - arrow.execute(metrics, 0); + arrow.execute(0); - REQUIRE(child_in.size() == 1); - REQUIRE(child_out.size() == 3); - REQUIRE(parent_out.size() == 1); + REQUIRE(child_in.GetSize(0) == 1); + REQUIRE(child_out.GetSize(0) == 3); + REQUIRE(parent_out.GetSize(0) == 1); - arrow.execute(metrics, 0); + arrow.execute(0); - REQUIRE(child_in.size() == 0); - REQUIRE(child_out.size() == 4); - REQUIRE(parent_out.size() == 2); + REQUIRE(child_in.GetSize(0) == 0); + REQUIRE(child_out.GetSize(0) == 4); + REQUIRE(parent_out.GetSize(0) == 2); } diff --git a/src/programs/unit_tests/Components/UserExceptionTests.cc b/src/programs/unit_tests/Components/UserExceptionTests.cc index 4593ef781..2ed1fe541 100644 --- a/src/programs/unit_tests/Components/UserExceptionTests.cc +++ b/src/programs/unit_tests/Components/UserExceptionTests.cc @@ -10,7 +10,7 @@ TEST_CASE("UserExceptionTests") { JApplication app; - app.SetParameterValue("log:debug","JApplication,JScheduler,JArrowProcessingController,JWorker,JArrow"); + app.SetParameterValue("jana:loglevel","debug"); app.SetParameterValue("jana:extended_report", 0); SECTION("JEventSource::Open() excepts") { diff --git a/src/programs/unit_tests/Engine/ArrowActivationTests.cc b/src/programs/unit_tests/Engine/ArrowActivationTests.cc deleted file mode 100644 index 80ef5d860..000000000 --- a/src/programs/unit_tests/Engine/ArrowActivationTests.cc +++ /dev/null @@ -1,139 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#include "catch.hpp" - -#include "../Topology/TestTopologyComponents.h" -#include - - -JArrowMetrics::Status steppe(JArrow* arrow) { - JArrowMetrics metrics; - arrow->execute(metrics, 0); - return metrics.get_last_status(); -} - -TEST_CASE("ArrowActivationTests") { - - auto q1 = new JMailbox(); - auto q2 = new JMailbox(); - auto q3 = new JMailbox(); - - auto p1 = new JPool(0,1,false); - auto p2 = new JPool(0,1,false); - p1->init(); - p2->init(); - - MultByTwoProcessor processor; - - auto emit_rand_ints = new RandIntSource("emit_rand_ints", p1, q1); - auto multiply_by_two = new MapArrow("multiply_by_two", processor, q1, q2); - auto subtract_one = new SubOneProcessor("subtract_one", q2, q3); - auto sum_everything = new SumSink("sum_everything", q3, p2); - - auto topology = std::make_shared(); - - emit_rand_ints->attach(multiply_by_two); - multiply_by_two->attach(subtract_one); - subtract_one->attach(sum_everything); - - topology->arrows.push_back(emit_rand_ints); - topology->arrows.push_back(multiply_by_two); - topology->arrows.push_back(subtract_one); - topology->arrows.push_back(sum_everything); - - topology->pools.push_back(p1); - topology->pools.push_back(p2); - - auto logger = JLogger(JLogger::Level::OFF); - topology->SetLogger(logger); - emit_rand_ints->set_logger(logger); - multiply_by_two->set_logger(logger); - subtract_one->set_logger(logger); - sum_everything->set_logger(logger); - emit_rand_ints->set_chunksize(1); - - JScheduler scheduler(topology); - scheduler.logger = logger; - - - SECTION("At first, everything is deactivated and all queues are empty") { - - REQUIRE(q1->size() == 0); - REQUIRE(q2->size() == 0); - REQUIRE(q3->size() == 0); - - JScheduler::TopologyState state = scheduler.get_topology_state(); - - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Uninitialized); - - REQUIRE(emit_rand_ints->get_pending() == 0); - REQUIRE(multiply_by_two->get_pending() == 0); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - } - - SECTION("As a message propagates, arrows and queues downstream automatically activate") { - - JScheduler::TopologyState state = scheduler.get_topology_state(); - - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Uninitialized); - - scheduler.run_topology(1); - state = scheduler.get_topology_state(); - // TODO: Check that initialize has been called, but not finalize - - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Active); - } - - SECTION("Deactivation") { - - emit_rand_ints->emit_limit = 1; - - JScheduler::TopologyState state = scheduler.get_topology_state(); - - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Uninitialized); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Uninitialized); - - scheduler.run_topology(1); - state = scheduler.get_topology_state(); - - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Active); - - auto result = steppe(emit_rand_ints); - REQUIRE(result == JArrowMetrics::Status::Finished); - - scheduler.next_assignment(0, emit_rand_ints, result); - state = scheduler.get_topology_state(); - - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Active); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Active); - - } - -} // TEST_CASE - - - - - - - diff --git a/src/programs/unit_tests/Engine/JExecutionEngineTests.cc b/src/programs/unit_tests/Engine/JExecutionEngineTests.cc new file mode 100644 index 000000000..8d3f1fd88 --- /dev/null +++ b/src/programs/unit_tests/Engine/JExecutionEngineTests.cc @@ -0,0 +1,293 @@ + +#define JANA2_TESTCASE + +#include "JANA/Engine/JExecutionEngine.h" +#include "JANA/Utils/JBenchUtils.h" +#include + +#include +#include +#include +#include +#include + +namespace jana::engine::tests { + +struct TestData { int x; }; +struct TestSource : public JEventSource { + JBenchUtils bench; + TestSource() { + SetCallbackStyle(CallbackStyle::ExpertMode); + } + Result Emit(JEvent& event) override { + event.Insert(new TestData {.x=(int) GetEmittedEventCount() * 2}, "src"); + bench.consume_cpu_ms(100); + return Result::Success; + } +}; +struct TestProc : public JEventProcessor { + JBenchUtils bench; + TestProc() { + SetCallbackStyle(CallbackStyle::ExpertMode); + } + void ProcessParallel(const JEvent& event) override { + JBenchUtils local_bench; + local_bench.consume_cpu_ms(300); + auto src_x = event.Get("src").at(0)->x; + event.Insert(new TestData {.x=src_x + 1}, "map"); + } + void Process(const JEvent& event) override { + bench.consume_cpu_ms(200); + auto map_x = event.Get("map").at(0)->x; + REQUIRE(map_x == (int)event.GetEventNumber()*2 + 1); + } +}; + + +TEST_CASE("JExecutionEngine_StateMachine") { + + JApplication app; + app.Add(new TestSource()); + app.Add(new TestProc()); + app.Initialize(); + auto sut = app.GetService(); + + SECTION("ManualFinish") { + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Paused); + sut->ScaleWorkers(0); + sut->RunTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Running); + sut->PauseTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Pausing); + + // Need to trigger the Pause since there are no workers to do so + auto worker = sut->RegisterWorker(); + JExecutionEngine::Task task; + sut->ExchangeTask(task, worker.worker_id, true); + + sut->RunSupervisor(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Paused); + sut->FinishTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Finished); + } + + SECTION("AutoFinish") { + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Paused); + sut->ScaleWorkers(0); + sut->RunTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Running); + sut->PauseTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Pausing); + + // Need to trigger the Pause since there are no workers to do so + auto worker = sut->RegisterWorker(); + JExecutionEngine::Task task; + sut->ExchangeTask(task, worker.worker_id, true); + + sut->RunSupervisor(); + sut->FinishTopology(); + + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Finished); + } +} + + +TEST_CASE("JExecutionEngine_ExternalWorkers") { + JApplication app; + app.SetParameterValue("jana:nevents", 1); + app.Add(new TestSource()); + app.Add(new TestProc()); + app.Initialize(); + auto sut = app.GetService(); + + + SECTION("SelfTermination") { + + auto worker = sut->RegisterWorker(); + REQUIRE(worker.worker_id == 0); // Not threadsafe doing this + + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Paused); + REQUIRE(sut->GetPerf().thread_count == 1); + sut->RunTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Running); + + JExecutionEngine::Task task; + + sut->ExchangeTask(task, worker.worker_id); + REQUIRE(task.arrow != nullptr); + REQUIRE(task.arrow->get_name() == "PhysicsEventSource"); // Only task available at this point! + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Running); + REQUIRE(sut->GetPerf().event_count == 0); + + task.arrow->fire(task.input_event, task.outputs, task.output_count, task.status); + REQUIRE(task.output_count == 1); + REQUIRE(task.status == JArrow::FireResult::KeepGoing); + + sut->ExchangeTask(task, worker.worker_id); + REQUIRE(task.arrow != nullptr); + REQUIRE(task.arrow->get_name() == "PhysicsEventSource"); // This will fail due to jana:nevents + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Running); + REQUIRE(sut->GetPerf().event_count == 0); + + task.arrow->fire(task.input_event, task.outputs, task.output_count, task.status); + REQUIRE(task.output_count == 1); + REQUIRE(task.outputs[0].second == 0); // Failure => return to pool + REQUIRE(task.status == JArrow::FireResult::Finished); + + sut->ExchangeTask(task, worker.worker_id); + REQUIRE(task.arrow != nullptr); + REQUIRE(task.arrow->get_name() == "PhysicsEventMap2"); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Draining); + REQUIRE(sut->GetPerf().event_count == 0); + + task.arrow->fire(task.input_event, task.outputs, task.output_count, task.status); + REQUIRE(task.output_count == 1); + REQUIRE(task.status == JArrow::FireResult::KeepGoing); + + sut->ExchangeTask(task, worker.worker_id); + REQUIRE(task.arrow != nullptr); + REQUIRE(task.arrow->get_name() == "PhysicsEventTap"); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Draining); + REQUIRE(sut->GetPerf().event_count == 0); + + task.arrow->fire(task.input_event, task.outputs, task.output_count, task.status); + REQUIRE(task.output_count == 1); + REQUIRE(task.status == JArrow::FireResult::KeepGoing); + + sut->ExchangeTask(task, worker.worker_id, true); + REQUIRE(task.arrow == nullptr); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Paused); + REQUIRE(sut->GetPerf().event_count == 1); + + sut->RunSupervisor(); + sut->FinishTopology(); + REQUIRE(sut->GetRunStatus() == JExecutionEngine::RunStatus::Finished); + } +} + + +TEST_CASE("JExecutionEngine_ScaleWorkers") { + JApplication app; + app.SetParameterValue("jana:nevents", 10); + app.SetParameterValue("jana:loglevel", "debug"); + app.Add(new TestSource()); + app.Add(new TestProc()); + app.Initialize(); + auto sut = app.GetService(); + + SECTION("SingleWorker") { + REQUIRE(sut->GetPerf().thread_count == 0); + sut->ScaleWorkers(1); + REQUIRE(sut->GetPerf().thread_count == 1); + sut->ScaleWorkers(0); + REQUIRE(sut->GetPerf().thread_count == 0); + } +} + + +TEST_CASE("JExecutionEngine_RunSingleEvent") { + JApplication app; + app.SetParameterValue("jana:nevents", 3); + app.SetParameterValue("jana:loglevel", "debug"); + app.Add(new TestSource()); + app.Add(new TestProc()); + app.Initialize(); + auto sut = app.GetService(); + + SECTION("SingleWorker") { + REQUIRE(sut->GetPerf().thread_count == 0); + sut->ScaleWorkers(1); + sut->RunTopology(); + + sut->RunSupervisor(); + REQUIRE(sut->GetPerf().thread_count == 1); + REQUIRE(sut->GetPerf().runstatus == JExecutionEngine::RunStatus::Paused); + + sut->FinishTopology(); + REQUIRE(sut->GetPerf().runstatus == JExecutionEngine::RunStatus::Finished); + REQUIRE(sut->GetPerf().event_count == 3); + + REQUIRE(sut->GetPerf().thread_count == 1); + sut->ScaleWorkers(0); + REQUIRE(sut->GetPerf().thread_count == 0); + } + + SECTION("MultipleWorker") { + REQUIRE(sut->GetPerf().thread_count == 0); + sut->ScaleWorkers(4); + REQUIRE(sut->GetPerf().thread_count == 4); + + sut->RunTopology(); + + sut->RunSupervisor(); + REQUIRE(sut->GetPerf().thread_count == 4); + REQUIRE(sut->GetPerf().runstatus == JExecutionEngine::RunStatus::Paused); + + sut->FinishTopology(); + REQUIRE(sut->GetPerf().runstatus == JExecutionEngine::RunStatus::Finished); + REQUIRE(sut->GetPerf().event_count == 3); + + REQUIRE(sut->GetPerf().thread_count == 4); + sut->ScaleWorkers(0); + REQUIRE(sut->GetPerf().thread_count == 0); + } +} + +TEST_CASE("JExecutionEngine_ExternalPause") { + JApplication app; + app.SetParameterValue("jana:loglevel", "info"); + app.Add(new TestSource()); + app.Add(new TestProc()); + app.Initialize(); + auto sut = app.GetService(); + + REQUIRE(sut->GetPerf().thread_count == 0); + sut->ScaleWorkers(4); + REQUIRE(sut->GetPerf().thread_count == 4); + + SECTION("PauseImmediately") { + sut->RunTopology(); + sut->PauseTopology(); + sut->RunSupervisor(); + + auto perf = sut->GetPerf(); + REQUIRE(perf.thread_count == 4); + REQUIRE(perf.runstatus == JExecutionEngine::RunStatus::Paused); + + sut->FinishTopology(); + + perf = sut->GetPerf(); + REQUIRE(perf.runstatus == JExecutionEngine::RunStatus::Finished); + REQUIRE(perf.event_count == 0); + REQUIRE(perf.thread_count == 4); + + sut->ScaleWorkers(0); + + perf = sut->GetPerf(); + REQUIRE(perf.thread_count == 0); + } + + SECTION("RunForABit") { + sut->RunTopology(); + std::thread t([&](){ + std::this_thread::sleep_for(std::chrono::seconds(3)); + sut->PauseTopology(); + }); + + sut->RunSupervisor(); + t.join(); + + auto perf = sut->GetPerf(); + REQUIRE(perf.thread_count == 4); + REQUIRE(perf.runstatus == JExecutionEngine::RunStatus::Paused); + REQUIRE(perf.event_count > 5); + + sut->ScaleWorkers(0); + REQUIRE(sut->GetPerf().thread_count == 0); + } +} + +} // jana::engine::tests + + + diff --git a/src/programs/unit_tests/Engine/ScaleTests.cc b/src/programs/unit_tests/Engine/ScaleTests.cc index e30962d67..4102b460a 100644 --- a/src/programs/unit_tests/Engine/ScaleTests.cc +++ b/src/programs/unit_tests/Engine/ScaleTests.cc @@ -2,7 +2,7 @@ #include #include -#include +#include #include "ScaleTests.h" @@ -10,87 +10,91 @@ TEST_CASE("NThreads") { JApplication app; app.SetParameterValue("jana:nevents",3); - app.SetParameterValue("log:global", "OFF"); - app.SetParameterValue("log:warn", "JScheduler,JArrow,JWorker,JArrowProcessingController"); + app.SetParameterValue("jana:loglevel","warn"); app.Add(new scaletest::DummySource); SECTION("If nthreads not provided, default to 1") { - app.Run(true); + app.Run(false); auto threads = app.GetNThreads(); REQUIRE(threads == 1); + app.Stop(true); } SECTION("If nthreads=Ncores, use ncores") { auto ncores = JCpuInfo::GetNumCpus(); app.SetParameterValue("nthreads", "Ncores"); - app.Run(true); + app.Run(false); auto threads = app.GetNThreads(); REQUIRE(threads == ncores); + app.Stop(true); } SECTION("If nthreads is something else, use that") { app.SetParameterValue("nthreads", 17); - app.Run(true); + app.Run(false); auto threads = app.GetNThreads(); REQUIRE(threads == 17); + app.Stop(true); } } TEST_CASE("ScaleNWorkerUpdate") { JApplication app; app.SetParameterValue("nthreads",4); - app.SetParameterValue("log:global", "OFF"); - app.SetParameterValue("log:warn", "JScheduler,JArrow,JWorker,JArrowProcessingController"); + app.SetParameterValue("jana:loglevel","warn"); app.Add(new scaletest::DummySource); app.Add(new scaletest::DummyProcessor); + app.Add(new JFactoryGeneratorT()); app.Run(false); auto threads = app.GetNThreads(); REQUIRE(threads == 4); - app.Scale(8); // Scale blocks until workers have fired up - - // Ideally we could just do this: - // threads = app.GetNThreads(); - // However, we can't, because JApplication caches performance metrics based off of a ticker interval, and - // Scale() doesn't invalidate the cache. We don't have a clean mechanism to manually force a cache invalidation - // from JApplication yet. So for now we will obtain the thread count directly from the JArrowProcessingController. - - auto pc = app.GetService(); - auto perf_summary = pc->measure_performance(); - threads = perf_summary->thread_count; + app.Stop(true, false); + app.Scale(8); // Scale blocks until workers have fired up + threads = app.GetNThreads(); REQUIRE(threads == 8); + app.Stop(true); } -TEST_CASE("ScaleThroughputImprovement", "[.][performance]") { +TEST_CASE("ScaleThroughputImprovement") { JApplication app; - app.SetParameterValue("log:global", "INFO"); - // app.SetParameterValue("log:warn", "JScheduler,JArrow,JWorker,JArrowProcessingController"); - app.SetTicker(false); + app.SetParameterValue("jana:loglevel", "INFO"); + //app.SetTicker(false); app.Add(new scaletest::DummySource); app.Add(new scaletest::DummyProcessor); - // app.SetParameterValue("benchmark:minthreads", 1); - // app.SetParameterValue("benchmark:maxthreads", 5); - // app.SetParameterValue("benchmark:threadstep", 2); - // app.SetParameterValue("benchmark:nsamples", 3); + app.Add(new JFactoryGeneratorT()); app.Initialize(); - auto japc = app.GetService(); - app.Run(false); + auto jee = app.GetService(); + + jee->ScaleWorkers(1); + jee->RunTopology(); std::this_thread::sleep_for(std::chrono::seconds(5)); - auto throughput_hz_1 = japc->measure_performance()->latest_throughput_hz; - japc->print_report(); + auto throughput_hz_1 = jee->GetPerf().throughput_hz; std::cout << "nthreads=1: throughput_hz=" << throughput_hz_1 << std::endl; - app.Scale(2); + jee->PauseTopology(); + jee->RunSupervisor(); + + jee->ScaleWorkers(2); + jee->RunTopology(); std::this_thread::sleep_for(std::chrono::seconds(5)); - auto throughput_hz_2 = japc->measure_performance()->latest_throughput_hz; - japc->print_report(); + auto throughput_hz_2 = jee->GetPerf().throughput_hz; std::cout << "nthreads=2: throughput_hz=" << throughput_hz_2 << std::endl; - app.Scale(4); + REQUIRE(jee->GetPerf().runstatus == JExecutionEngine::RunStatus::Running); + jee->PauseTopology(); + jee->RunSupervisor(); + + jee->ScaleWorkers(4); + jee->RunTopology(); std::this_thread::sleep_for(std::chrono::seconds(5)); - auto throughput_hz_4 = japc->measure_performance()->latest_throughput_hz; - japc->print_report(); + auto throughput_hz_4 = jee->GetPerf().throughput_hz; + jee->PauseTopology(); + jee->RunSupervisor(); std::cout << "nthreads=4: throughput_hz=" << throughput_hz_4 << std::endl; + + jee->FinishTopology(); + REQUIRE(throughput_hz_2 > throughput_hz_1*1.5); REQUIRE(throughput_hz_4 > throughput_hz_2*1.25); } diff --git a/src/programs/unit_tests/Engine/ScaleTests.h b/src/programs/unit_tests/Engine/ScaleTests.h index 6369f5376..adb296626 100644 --- a/src/programs/unit_tests/Engine/ScaleTests.h +++ b/src/programs/unit_tests/Engine/ScaleTests.h @@ -19,7 +19,6 @@ struct DummySource : public JEventSource { Result Emit(JEvent& event) override { m_bench_utils.set_seed(event.GetEventNumber(), NAME_OF_THIS); m_bench_utils.consume_cpu_ms(20); - std::this_thread::sleep_for(std::chrono::nanoseconds(1)); return Result::Success; } @@ -27,15 +26,30 @@ struct DummySource : public JEventSource { JBenchUtils m_bench_utils = JBenchUtils(); }; +struct DummyData {int x;}; +struct DummyFactory : public JFactoryT { + JBenchUtils m_bench_utils = JBenchUtils(); + + void Process(const std::shared_ptr& event) override { + m_bench_utils.set_seed(event->GetEventNumber(), NAME_OF_THIS); + m_bench_utils.consume_cpu_ms(70); + Insert(new DummyData {22}); + } +}; + struct DummyProcessor : public JEventProcessor { DummyProcessor() { SetCallbackStyle(CallbackStyle::ExpertMode); } + + void ProcessParallel(const JEvent& event) override { + event.Get(); + } + void Process(const JEvent& event) override { m_bench_utils.set_seed(event.GetEventNumber(), NAME_OF_THIS); - m_bench_utils.consume_cpu_ms(100); - std::this_thread::sleep_for(std::chrono::nanoseconds(1)); + m_bench_utils.consume_cpu_ms(10); } private: diff --git a/src/programs/unit_tests/Engine/SchedulerTests.cc b/src/programs/unit_tests/Engine/SchedulerTests.cc deleted file mode 100644 index 0136fc2ad..000000000 --- a/src/programs/unit_tests/Engine/SchedulerTests.cc +++ /dev/null @@ -1,229 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - - -#include "catch.hpp" - -#include -#include - -#include "../Topology/TestTopologyComponents.h" - -TEST_CASE("SchedulerTests") { - - auto q1 = new JMailbox(); - auto q2 = new JMailbox(); - auto q3 = new JMailbox(); - - auto p1 = new JPool(0,1,false); - auto p2 = new JPool(0,1,false); - p1->init(); - p2->init(); - - MultByTwoProcessor processor; - - auto emit_rand_ints = new RandIntSource("emit_rand_ints", p1, q1); - auto multiply_by_two = new MapArrow("multiply_by_two", processor, q1, q2); - auto subtract_one = new SubOneProcessor("subtract_one", q2, q3); - auto sum_everything = new SumSink("sum_everything", q3, p2); - - auto topology = std::make_shared(); - - emit_rand_ints->attach(multiply_by_two); - multiply_by_two->attach(subtract_one); - subtract_one->attach(sum_everything); - - topology->arrows.push_back(emit_rand_ints); - topology->arrows.push_back(multiply_by_two); - topology->arrows.push_back(subtract_one); - topology->arrows.push_back(sum_everything); - - auto logger = JLogger(JLogger::Level::INFO); - topology->SetLogger(logger); - emit_rand_ints->set_logger(logger); - multiply_by_two->set_logger(logger); - subtract_one->set_logger(logger); - sum_everything->set_logger(logger); - - emit_rand_ints->set_chunksize(1); - - JScheduler scheduler(topology); - scheduler.logger = logger; - - scheduler.run_topology(1); - - JArrow* assignment; - JArrowMetrics::Status last_result; - - - SECTION("When run sequentially, RRS returns nullptr => topology finished") { - - last_result = JArrowMetrics::Status::ComeBackLater; - assignment = nullptr; - do { - assignment = scheduler.next_assignment(0, assignment, last_result); - if (assignment != nullptr) { - JArrowMetrics metrics; - assignment->execute(metrics, 0); - last_result = metrics.get_last_status(); - } - } while (assignment != nullptr); - - JScheduler::TopologyState state = scheduler.get_topology_state(); - REQUIRE(state.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(state.arrow_states[1].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(state.arrow_states[2].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(state.arrow_states[3].status == JScheduler::ArrowStatus::Finalized); - } - - SECTION("When run sequentially, topology finished => RRS returns nullptr") { - - last_result = JArrowMetrics::Status::ComeBackLater; - assignment = nullptr; - - for (int i=0; i<80; ++i) { - // 20 events in source which need to pass through 4 arrows - - assignment = scheduler.next_assignment(0, assignment, last_result); - REQUIRE(assignment != nullptr); - JArrowMetrics metrics; - assignment->execute(metrics, 0); - last_result = metrics.get_last_status(); - } - assignment = scheduler.next_assignment(0, assignment, last_result); - REQUIRE(assignment == nullptr); - } -} - -TEST_CASE("SchedulerRoundRobinBehaviorTests") { - - auto q1 = new JMailbox(); - auto q2 = new JMailbox(); - auto q3 = new JMailbox(); - - auto p1 = new JPool(0,1,false); - auto p2 = new JPool(0,1,false); - p1->init(); - p2->init(); - - MultByTwoProcessor processor; - - auto emit_rand_ints = new RandIntSource("emit_rand_ints", p1, q1); - auto multiply_by_two = new MapArrow("multiply_by_two", processor, q1, q2); - auto subtract_one = new SubOneProcessor("subtract_one", q2, q3); - auto sum_everything = new SumSink("sum_everything", q3, p2); - - auto topology = std::make_shared(); - - emit_rand_ints->attach(multiply_by_two); - multiply_by_two->attach(subtract_one); - subtract_one->attach(sum_everything); - - topology->arrows.push_back(emit_rand_ints); - topology->arrows.push_back(multiply_by_two); - topology->arrows.push_back(subtract_one); - topology->arrows.push_back(sum_everything); - - auto logger = JLogger(JLogger::Level::INFO); - topology->SetLogger(logger); - emit_rand_ints->set_logger(logger); - multiply_by_two->set_logger(logger); - subtract_one->set_logger(logger); - sum_everything->set_logger(logger); - - emit_rand_ints->set_chunksize(1); - - JScheduler scheduler(topology); - scheduler.logger = logger; - scheduler.run_topology(1); - - auto last_result = JArrowMetrics::Status::ComeBackLater; - JArrow* assignment = nullptr; - - SECTION("When there is only one worker, who always encounters ComeBackLater...") { - - LOG_INFO(logger) << "---------------------------------------" << LOG_END; - std::string ordering[] = {"emit_rand_ints", - "multiply_by_two", - "subtract_one", - "sum_everything"}; - - for (int i = 0; i < 10; ++i) { - assignment = scheduler.next_assignment(0, assignment, last_result); - - // The assignments go round-robin - std::cout << "Assignment is " << assignment->get_name() << std::endl; - REQUIRE(assignment != nullptr); - REQUIRE(assignment->get_name() == ordering[i % 4]); - } - } - - - SECTION("When a team of workers start off with (nullptr, ComeBackLater)...") { - - LOG_INFO(logger) << "---------------------------------------" << LOG_END; - std::map assignment_counts; - - - for (int i = 0; i < 10; ++i) { - assignment = nullptr; - last_result = JArrowMetrics::Status::ComeBackLater; - assignment = scheduler.next_assignment(i, assignment, last_result); - - // They all receive a nonnull assignment - REQUIRE (assignment != nullptr); - - assignment_counts[assignment->get_name()]++; - } - - // The sequential arrows only get assigned once - REQUIRE(assignment_counts["emit_rand_ints"] == 1); - REQUIRE(assignment_counts["sum_everything"] == 1); - - // The parallel arrows get assigned many times, evenly - REQUIRE(assignment_counts["subtract_one"] == 4); - REQUIRE(assignment_counts["multiply_by_two"] == 4); - - } - - - SECTION("When all arrows are assigned, and a sequential arrow comes back...") { - - - LOG_INFO(logger) << "--------------------------------------" << LOG_END; - - scheduler.next_assignment(0, nullptr, JArrowMetrics::Status::ComeBackLater); - scheduler.next_assignment(1, nullptr, JArrowMetrics::Status::ComeBackLater); - scheduler.next_assignment(2, nullptr, JArrowMetrics::Status::ComeBackLater); - auto sum_everything_arrow = scheduler.next_assignment(3, nullptr, JArrowMetrics::Status::ComeBackLater); - - // Last assignment returned sequential arrow "sum_everything" - REQUIRE(sum_everything_arrow != nullptr); - REQUIRE(sum_everything_arrow->get_name() == "sum_everything"); - - std::map assignment_counts; - - // We return the sequential arrow to the scheduler - assignment = sum_everything_arrow; - last_result = JArrowMetrics::Status::ComeBackLater; - auto arrow = scheduler.next_assignment(3, assignment, last_result); - - assignment_counts[arrow->get_name()]++; - - for (int i = 0; i < 8; ++i) { - assignment = nullptr; - last_result = JArrowMetrics::Status::ComeBackLater; - arrow = scheduler.next_assignment(i, assignment, last_result); - assignment_counts[arrow->get_name()]++; - } - - // Once scheduler receives a sequential arrow back, it will offer it back out - // but only once, if we don't return it - - REQUIRE(assignment_counts["sum_everything"] == 1); - } - -} - - diff --git a/src/programs/unit_tests/Engine/TerminationTests.cc b/src/programs/unit_tests/Engine/TerminationTests.cc index 4e939f92b..bcbd18f5a 100644 --- a/src/programs/unit_tests/Engine/TerminationTests.cc +++ b/src/programs/unit_tests/Engine/TerminationTests.cc @@ -4,7 +4,6 @@ #include "TerminationTests.h" #include "catch.hpp" -#include #include #include @@ -13,10 +12,9 @@ TEST_CASE("TerminationTests") { JApplication app; - app.SetParameterValue("log:global", "OFF"); + app.SetParameterValue("jana:loglevel", "warn"); auto processor = new CountingProcessor(); app.Add(processor); - app.SetParameterValue("jana:extended_report", 0); SECTION("Manual termination") { @@ -27,7 +25,7 @@ TEST_CASE("TerminationTests") { app.Stop(true); REQUIRE(source->event_count > 0); REQUIRE(processor->finish_call_count == 1); - REQUIRE(app.GetNEventsProcessed() == source->event_count); + REQUIRE(app.GetNEventsProcessed() == source->GetEmittedEventCount()); } SECTION("Self termination") { @@ -38,7 +36,7 @@ TEST_CASE("TerminationTests") { REQUIRE(source->event_count == 10); REQUIRE(processor->processed_count == 10); REQUIRE(processor->finish_call_count == 1); - REQUIRE(app.GetNEventsProcessed() == source->event_count); + REQUIRE(app.GetNEventsProcessed() == source->GetEmittedEventCount()); } SECTION("Interrupted during JEventSource::Open()") { @@ -47,7 +45,7 @@ TEST_CASE("TerminationTests") { app.Add(source); app.Run(true); REQUIRE(processor->finish_call_count == 1); - REQUIRE(app.GetNEventsProcessed() == source->GetEventCount()); + REQUIRE(app.GetNEventsProcessed() == source->GetEmittedEventCount()); // We don't know how many events will emit before Stop() request propagates } diff --git a/src/programs/unit_tests/Services/JParameterManagerTests.cc b/src/programs/unit_tests/Services/JParameterManagerTests.cc index 8c75bf0a1..4d0d96019 100644 --- a/src/programs/unit_tests/Services/JParameterManagerTests.cc +++ b/src/programs/unit_tests/Services/JParameterManagerTests.cc @@ -432,6 +432,49 @@ TEST_CASE("JParameterManager_Strictness") { +TEST_CASE("JParameterManager_ConflictingDefaults") { + + int x1 = 3; + int x2 = 4; + int x3 = 3; + int x4 = 4; + + JParameterManager sut; + sut.SetLogger(JLogger()); + + // Simulate a FactorySet containing two JFactories, each of which declares the same parameter with different default values + auto p1 = sut.SetDefaultParameter("my_param_name", x1, "Tests how conflicting defaults are handled"); + REQUIRE(p1->HasDefault() == true); + REQUIRE(p1->IsDefault() == true); + REQUIRE(p1->GetDefault() == "3"); // Should be the _latest_ default found + REQUIRE(p1->GetValue() == "3"); // Should be the _latest_ default value + REQUIRE(x1 == 3); + auto p2 = sut.SetDefaultParameter("my_param_name", x2, "Tests how conflicting defaults are handled"); + REQUIRE(p2->HasDefault() == true); + REQUIRE(p2->IsDefault() == true); + REQUIRE(p2->GetDefault() == "4"); // Should be the _latest_ default found + REQUIRE(p2->GetValue() == "4"); // Should be the _latest_ default value + REQUIRE(x2 == 4); + + // Simulate a _second_ FactorySet containing fresh instances of the same two JFactories, + auto p3 = sut.SetDefaultParameter("my_param_name", x3, "Tests how conflicting defaults are handled"); + REQUIRE(p3->HasDefault() == true); + REQUIRE(p3->IsDefault() == true); + REQUIRE(p3->GetDefault() == "3"); // Should be the _latest_ default found + REQUIRE(p3->GetValue() == "3"); // Should be the _latest_ default value + REQUIRE(x3 == 3); + auto p4 = sut.SetDefaultParameter("my_param_name", x4, "Tests how conflicting defaults are handled"); + REQUIRE(p4->HasDefault() == true); + REQUIRE(p4->IsDefault() == true); + REQUIRE(p4->GetDefault() == "4"); // Should be the _latest_ default value found + REQUIRE(p4->GetValue() == "4"); // Should be the _latest_ default value + REQUIRE(x4 == 4); + + sut.PrintParameters(2,1); +} + + + diff --git a/src/programs/unit_tests/Services/JWiringServiceTests.cc b/src/programs/unit_tests/Services/JWiringServiceTests.cc index b27e3de5f..84c69494c 100644 --- a/src/programs/unit_tests/Services/JWiringServiceTests.cc +++ b/src/programs/unit_tests/Services/JWiringServiceTests.cc @@ -40,7 +40,9 @@ TEST_CASE("WiringTests") { jana::services::JWiringService sut; toml::table table = toml::parse(some_wiring); - auto wirings = sut.parse_table(table); + sut.AddWirings(table, "testcase"); + + const auto& wirings = sut.GetWirings(); REQUIRE(wirings.size() == 2); REQUIRE(wirings[0]->prefix == "myfac"); REQUIRE(wirings[1]->prefix == "myfac_modified"); @@ -74,7 +76,7 @@ TEST_CASE("WiringTests_DuplicatePrefixes") { jana::services::JWiringService sut; toml::table table = toml::parse(duplicate_prefixes); try { - auto wirings = sut.parse_table(table); + sut.AddWirings(table,"testcase"); REQUIRE(1 == 0); } catch (const JException& e) { @@ -101,11 +103,79 @@ TEST_CASE("WiringTests_Overlay") { below->configs["y"] = "42"; auto sut = jana::services::JWiringService(); - auto result = sut.overlay(std::move(above), std::move(below)); - REQUIRE(result->input_names[2] == "make"); - REQUIRE(result->input_levels[1] == JEventLevel::PhysicsEvent); - REQUIRE(result->configs["x"] == "6.18"); - REQUIRE(result->configs["y"] == "42"); + sut.Overlay(*above, *below); + REQUIRE(above->input_names[2] == "make"); + REQUIRE(above->input_levels[1] == JEventLevel::PhysicsEvent); + REQUIRE(above->configs["x"] == "6.18"); + REQUIRE(above->configs["y"] == "42"); } +static constexpr std::string_view fake_wiring_file = R"( + [[factory]] + plugin_name = "ECAL" + type_name = "ClusteringFac" + prefix = "myfac" + + [factory.configs] + x = "22" + y = "verbose" + [[factory]] + plugin_name = "ECAL" + type_name = "ClusteringFac" + prefix = "variantfac" + + [factory.configs] + x = "49" + y = "silent" + + [[factory]] + plugin_name = "BCAL" + type_name = "ClusteringFac" + prefix = "sillyfac" + + [factory.configs] + x = "618" + y = "mediocre" +)"; + + +TEST_CASE("WiringTests_FakeFacGen") { + jana::services::JWiringService sut; + toml::table table = toml::parse(fake_wiring_file); + sut.AddWirings(table, "testcase"); + + using Wiring = jana::services::JWiringService::Wiring; + std::vector> fake_facgen_wirings; + + // One gets overlaid with an existing wiring + auto a = std::make_unique(); + a->plugin_name = "ECAL"; + a->type_name = "ClusteringFac"; + a->prefix = "variantfac"; + a->configs["x"] = "42"; + fake_facgen_wirings.push_back(std::move(a)); + + // The other is brand new + auto b = std::make_unique(); + b->plugin_name = "ECAL"; + b->type_name = "ClusteringFac"; + b->prefix = "exuberantfac"; + b->configs["x"] = "27"; + fake_facgen_wirings.push_back(std::move(b)); + + // We should end up with three in total + sut.AddWirings(fake_facgen_wirings, "fake_facgen"); + auto final_wirings = sut.GetWirings("ECAL", "ClusteringFac"); + + REQUIRE(final_wirings.size() == 3); + + REQUIRE(final_wirings[0]->prefix == "myfac"); + REQUIRE(final_wirings[1]->prefix == "variantfac"); + REQUIRE(final_wirings[2]->prefix == "exuberantfac"); + + REQUIRE(final_wirings[0]->configs["x"] == "22"); // from file only + REQUIRE(final_wirings[1]->configs["x"] == "49"); // file overrides facgen + REQUIRE(final_wirings[2]->configs["x"] == "27"); // from facgen only + +} diff --git a/src/programs/unit_tests/Topology/ArrowTests.cc b/src/programs/unit_tests/Topology/ArrowTests.cc deleted file mode 100644 index 4a3b8877d..000000000 --- a/src/programs/unit_tests/Topology/ArrowTests.cc +++ /dev/null @@ -1,95 +0,0 @@ - -#include -#include - -namespace jana { -namespace arrowtests { - - -struct TestMapArrow : public JJunctionArrow { - - TestMapArrow(JMailbox* qi, - JPool* pi, - JPool* pd, - JMailbox* qd) - : JJunctionArrow("testmaparrow", false, false, true) { - - first_input = {this, qi, true, 1, 1}; - first_output = {this, pi, false, 1, 1}; - second_input = {this, pd, true, 1, 1}; - second_output = {this, qd, false, 1, 1}; - } - - Status process(Data& input_int, - Data& output_int, - Data& input_double, - Data& output_double) { - std::cout << "Hello from process" << std::endl; - - REQUIRE(input_int.item_count == 1); - REQUIRE(input_int.reserve_count == 1); - REQUIRE(output_int.item_count == 0); - REQUIRE(output_int.reserve_count == 0); - REQUIRE(input_double.item_count == 1); - REQUIRE(input_double.reserve_count == 0); - REQUIRE(output_double.item_count == 0); - REQUIRE(output_double.reserve_count == 1); - - int* x = input_int.items[0]; - input_int.items[0] = nullptr; - input_int.item_count = 0; - - // TODO: Maybe user shouldn't be allowed to modify reserve_count at all - // TODO Maybe user should only be allowed to push and pull from ... range...? - - double* y = input_double.items[0]; - input_double.items[0] = nullptr; - input_double.item_count = 0; - - // Do something useful here - *y = *x + 22.2; - - output_int.items[0] = x; - output_int.item_count = 1; - - output_double.items[0] = y; - output_double.item_count = 1; - return Status::KeepGoing; - } - -}; - - -TEST_CASE("ArrowTests_Basic") { - - JMailbox qi {2, 1, false}; - JPool pi {5, 1, true}; - JPool pd {5, 1, true}; - JMailbox qd {2, 1, false}; - - pi.init(); - pd.init(); - - TestMapArrow a {&qi, &pi, &pd, &qd}; - - int* x; - pi.pop(&x, 1, 1, 0); - *x = 100; - - qi.push_and_unreserve(&x, 1, 0, 0); - JArrowMetrics m; - a.execute(m, 0); - - double* y; - qd.pop_and_reserve(&y, 1, 1, 0); - REQUIRE(*y == 122.2); - -} - - -} // namespace arrowtests -} // namespace jana - - - - diff --git a/src/programs/unit_tests/Topology/JArrowTests.cc b/src/programs/unit_tests/Topology/JArrowTests.cc new file mode 100644 index 000000000..a433236ee --- /dev/null +++ b/src/programs/unit_tests/Topology/JArrowTests.cc @@ -0,0 +1,91 @@ + +#include +#include + +namespace jana::topology::jarrowtests { + +struct TestData { int x; }; + +struct BasicParallelArrow : public JArrow { + + BasicParallelArrow() { + create_ports(1, 1); + set_is_parallel(true); + } + + void fire(JEvent* input, OutputData& outputs, size_t& output_count, JArrow::FireResult& process_status) { + + input->Insert(new TestData {.x=22}); + + outputs[0] = {input, 1}; + output_count = 1; + process_status = JArrow::FireResult::KeepGoing; + + } +}; + + +TEST_CASE("BasicParallelArrow_Fire") { + + JApplication app; + app.Initialize(); + auto event = std::make_shared(&app); + + BasicParallelArrow sut; + JArrow::OutputData outputs; + size_t output_count; + JArrow::FireResult status; + + sut.fire(event.get(), outputs, output_count, status); + + REQUIRE(event->GetSingle()->x == 22); + REQUIRE(output_count == 1); + REQUIRE(outputs[0].first == event.get()); + REQUIRE(outputs[0].second == 1); + REQUIRE(status == JArrow::FireResult::KeepGoing); +} + + + +TEST_CASE("BasicParallelArrow_ExecuteSucceeds") { + + JApplication app; + app.Initialize(); + + BasicParallelArrow sut; + JEventPool input_pool(app.GetService(), 10, 1); + JEventQueue output_queue(10, 1); + + sut.attach(&input_pool, 0); + sut.attach(&output_queue, 1); + + auto result = sut.execute( 0); + + REQUIRE(result == JArrow::FireResult::KeepGoing); + REQUIRE(output_queue.GetSize(0) == 1); + JEvent* event = output_queue.Pop(0); + REQUIRE(event->GetSingle()->x == 22); +} + + +TEST_CASE("BasicParallelArrow_ExecuteFails") { + + JApplication app; + app.Initialize(); + + BasicParallelArrow sut; + JEventQueue input_queue(10, 1); + JEventQueue output_queue(10, 1); + + sut.attach(&input_queue, 0); + sut.attach(&output_queue, 1); + + auto result = sut.execute(0); + + REQUIRE(result == JArrow::FireResult::NotRunYet); + REQUIRE(output_queue.GetSize(0) == 0); +} + + + +} // namespace jana::topology::jarrowtests diff --git a/src/programs/unit_tests/Topology/JPoolTests.cc b/src/programs/unit_tests/Topology/JPoolTests.cc index 6aa48680a..d0a1afb30 100644 --- a/src/programs/unit_tests/Topology/JPoolTests.cc +++ b/src/programs/unit_tests/Topology/JPoolTests.cc @@ -1,95 +1,46 @@ +#include "JANA/JApplicationFwd.h" +#include "JANA/Services/JComponentManager.h" #include -#include +#include namespace jana { namespace jpooltests { -struct Event { - int x=3; - double y=7.8; - bool* was_dtor_called = nullptr; - - ~Event() { - if (was_dtor_called != nullptr) { - *was_dtor_called = true; - } - } -}; TEST_CASE("JPoolTests_SingleLocationLimitEvents") { + JApplication app; + app.Initialize(); + auto jcm = app.GetService(); - JPool pool(3, 1, true); - pool.init(); + JEventPool pool(jcm, 3, 1); - Event* e = pool.get(0); + auto* e = pool.Pop(0); REQUIRE(e != nullptr); - REQUIRE(e->x == 3); + REQUIRE(e->GetEventNumber() == 0); // Will segfault if not initialized - Event* f = pool.get(0); + auto* f = pool.Pop(0); REQUIRE(f != nullptr); - REQUIRE(f->x == 3); + REQUIRE(f->GetEventNumber() == 0); - Event* g = pool.get(0); + auto* g = pool.Pop(0); REQUIRE(g != nullptr); - REQUIRE(g->x == 3); + REQUIRE(g->GetEventNumber() == 0); - Event* h = pool.get(0); + auto* h = pool.Pop(0); REQUIRE(h == nullptr); - f->x = 5; - pool.put(f, 0); + f->SetEventNumber(5); + pool.Push(f, 0); - h = pool.get(0); + h = pool.Pop(0); REQUIRE(h != nullptr); - REQUIRE(h->x == 5); -} - -TEST_CASE("JPoolTests_SingleLocationUnlimitedEvents") { - - bool was_dtor_called = false; - JPool pool(3, 1, false); - pool.init(); - - Event* e = pool.get(0); - e->was_dtor_called = &was_dtor_called; - REQUIRE(e != nullptr); - REQUIRE(e->x == 3); - - Event* f = pool.get(0); - f->was_dtor_called = &was_dtor_called; - REQUIRE(f != nullptr); - REQUIRE(f->x == 3); - - Event* g = pool.get(0); - g->was_dtor_called = &was_dtor_called; - REQUIRE(g != nullptr); - REQUIRE(g->x == 3); - - Event* h = pool.get(0); - // Unlike the others, h is allocated on the heap - h->x = 9; - h->was_dtor_called = &was_dtor_called; - REQUIRE(h != nullptr); - REQUIRE(g->x == 3); - - f->x = 5; - pool.put(f, 0); - // f goes back into the pool, so dtor does not get called - REQUIRE(was_dtor_called == false); - - pool.put(h, 0); - // h's dtor DOES get called - REQUIRE(was_dtor_called == true); - - // When we retrieve another event, it comes from the pool - // So we get what used to be f - Event* i = pool.get(0); - REQUIRE(i != nullptr); - REQUIRE(i->x == 5); + REQUIRE(h->GetEventNumber() == 5); } } // namespace jana } // namespace jpooltests + + diff --git a/src/programs/unit_tests/Topology/MapArrow.h b/src/programs/unit_tests/Topology/MapArrow.h deleted file mode 100644 index e24670129..000000000 --- a/src/programs/unit_tests/Topology/MapArrow.h +++ /dev/null @@ -1,91 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#ifndef GREENFIELD_MAPARROW_H -#define GREENFIELD_MAPARROW_H - -#include - - -/// ParallelProcessor transforms S to T and it does so in a way which is thread-safe -/// and ideally stateless. It is conceptually equivalent to the first part -/// of JEventProcessor::Process, i.e. up until the lock is acquired. Alternatively, it could -/// become a JFactorySet, in which case process() would call all Factories present, thereby -/// making sure that everything which can be calculated in parallel has in fact been, before -/// proceeding to the (sequential) Sink. - -template -struct ParallelProcessor { - virtual T process(S s) = 0; -}; - - -/// MapArrow lifts a ParallelProcessor into a streaming async context -template -class MapArrow : public JArrow { - -private: - ParallelProcessor& _processor; - JMailbox *_input_queue; - JMailbox *_output_queue; - -public: - MapArrow(std::string name, ParallelProcessor& processor, JMailbox *input_queue, JMailbox *output_queue) - : JArrow(name, true, false, false) - , _processor(processor) - , _input_queue(input_queue) - , _output_queue(output_queue) { - }; - - void execute(JArrowMetrics& result, size_t /* location_id */) override { - - auto start_total_time = std::chrono::steady_clock::now(); - std::vector xs; - std::vector ys; - xs.reserve(get_chunksize()); - ys.reserve(get_chunksize()); - // TODO: These allocations are unnecessary and should be eliminated - - auto in_status = _input_queue->pop(xs, get_chunksize()); - - auto start_latency_time = std::chrono::steady_clock::now(); - for (S &x : xs) { - ys.push_back(_processor.process(x)); - } - auto message_count = xs.size(); - auto end_latency_time = std::chrono::steady_clock::now(); - - auto out_status = JMailbox::Status::Ready; - if (!ys.empty()) { - out_status = _output_queue->push(ys); - } - auto end_queue_time = std::chrono::steady_clock::now(); - - - auto latency = (end_latency_time - start_latency_time); - auto overhead = (end_queue_time - start_total_time) - latency; - - JArrowMetrics::Status status; - // if (in_status == JMailbox::Status::Finished) { - // set_status(JActivable::Status::Finished); - // status = JArrowMetrics::Status::Finished; - // } - if (in_status == JMailbox::Status::Ready && out_status == JMailbox::Status::Ready) { - status = JArrowMetrics::Status::KeepGoing; - } - else { - status = JArrowMetrics::Status::ComeBackLater; - } - result.update(status, message_count, 1, latency, overhead); - } - - size_t get_pending() final { return _input_queue->size(); } - - size_t get_threshold() final { return _input_queue->get_threshold(); } - - void set_threshold(size_t threshold) final { _input_queue->set_threshold(threshold); } -}; - - -#endif //GREENFIELD_MAPARROW_H diff --git a/src/programs/unit_tests/Topology/MultiLevelTopologyTests.cc b/src/programs/unit_tests/Topology/MultiLevelTopologyTests.cc index c9bf633c6..45088ee2d 100644 --- a/src/programs/unit_tests/Topology/MultiLevelTopologyTests.cc +++ b/src/programs/unit_tests/Topology/MultiLevelTopologyTests.cc @@ -1,5 +1,8 @@ #include "MultiLevelTopologyTests.h" +#include "JANA/Engine/JExecutionEngine.h" #include "JANA/JException.h" +#include "JANA/Topology/JArrow.h" +#include "JANA/Topology/JTopologyBuilder.h" #include #include @@ -7,436 +10,65 @@ namespace jana { namespace timeslice_tests { -enum class Level { None, Timeslice, Event, Subevent }; -enum class ComponentType { None, Source, Filter, Map, Split, Merge, Reduce }; -std::ostream& operator<<(std::ostream& os, Level l) { - switch (l) { - case Level::None: os << "None"; break; - case Level::Timeslice: os << "Timeslice"; break; - case Level::Event: os << "Event"; break; - case Level::Subevent: os << "Subevent"; break; - } - return os; -} - -std::ostream& operator<<(std::ostream& os, ComponentType ct) { - switch (ct) { - case ComponentType::None: os << "None"; break; - case ComponentType::Source: os << "Source"; break; - case ComponentType::Filter: os << "Filter"; break; - case ComponentType::Map: os << "Map"; break; - case ComponentType::Split: os << "Split"; break; - case ComponentType::Merge: os << "Merge"; break; - case ComponentType::Reduce: os << "Reduce"; break; - } - return os; -} - - -struct Arrow { - Level level = Level::None; - ComponentType componentType = ComponentType::None; - - Arrow* input = nullptr; - Arrow* output = nullptr; - -}; - -class Diagram { - std::vector grid; - size_t rows, cols, hspace, vspace; - - Diagram(size_t rows, size_t cols, size_t hspace=10, size_t vspace=4) - : rows(rows), cols(cols), hspace(hspace), vspace(vspace) { - - for (size_t i=0; i, Arrow*> arrows; - Arrow* source; - - void add(Level l, ComponentType ct) { - auto arrow = new Arrow; - arrow->level = l; - arrow->componentType = ct; - arrows[{l, ct}] = arrow; - } - - bool has_connection(Level l1, ComponentType c1, Level l2, ComponentType c2) { - - Arrow* a1 = arrows.at({l1, c1}); - Arrow* a2 = arrows.at({l2, c2}); - - bool success = true; - success &= (a1 != nullptr); - success &= (a2 != nullptr); - success &= (a1->output == a2); - success &= (a2->input == a1); - - return success; - } - - bool has_no_input(Level l, ComponentType c) { - return (arrows.at({l, c})->input == nullptr); - } - - bool has_no_output(Level l, ComponentType c) { - return (arrows.at({l, c})->output == nullptr); - } - - void print() { - Arrow* next = source; - while (next != nullptr) { - switch (next->level) { - case Level::Timeslice: std::cout << "Timeslice:"; break; - case Level::Event: std::cout << " Event:"; break; - case Level::Subevent: std::cout << " Subevent:"; break; - case Level::None: std::cout << "None:"; break; - } - std::cout << next->componentType << std::endl; - next = next->output; - } - } - - void print_diagram() { - std::cout << "====================================================================" << std::endl; - std::cout << "Level Source Filter Map Split Merge Reduce" << std::endl; - - Level levels[] = {Level::Timeslice, Level::Event, Level::Subevent}; - for (int i=0; i<3; ++i) { - // ----------- - // Print row - // ----------- - - Level l = levels[i]; - - switch (l) { - case Level::None: std::cout << "None "; break; - case Level::Timeslice: std::cout << "Timeslice "; break; - case Level::Event: std::cout << "Event "; break; - case Level::Subevent: std::cout << "Subevent "; break; - } - - bool pencil_active = false; - for (ComponentType ct : {ComponentType::Source, ComponentType::Filter, ComponentType::Map, ComponentType::Split, ComponentType::Merge, ComponentType::Reduce}) { - auto it = arrows.find({l, ct}); - - if (it == arrows.end()) { - if (pencil_active) { - std::cout << "----------"; - } - else { - std::cout << " "; - } - } - else { - std::cout << "[]"; - pencil_active = (it->second->output != nullptr && it->second->output->level == l); - if (pencil_active) { - std::cout << "--------"; - } - else { - std::cout << " "; - } - } - } - std::cout << std::endl; - - // -------- - // Print connectors to lower level - // -------- - // Skip lowest level - if (i < 2) { - // Vertical connector to split and merge - auto split_it = arrows.find({l, ComponentType::Split}); - if (split_it != arrows.end()) { - std::cout << " |"; - } - else { - std::cout << " "; - } - auto merge_it = arrows.find({l, ComponentType::Merge}); - if (merge_it != arrows.end()) { - std::cout << " |"; - } - else { - std::cout << " "; - } - std::cout << std::endl; - - // Horizontal connector - std::cout << " "; - bool pencil_active = false; - for (ComponentType ct : {ComponentType::Source, ComponentType::Filter, ComponentType::Map, ComponentType::Split, ComponentType::Merge, ComponentType::Reduce}) { - - bool found_split = (split_it != arrows.end() && ct == ComponentType::Split); - bool found_splitee = (split_it != arrows.end() && split_it->second->output->componentType == ct); - bool found_merge = (merge_it != arrows.end() && ct == ComponentType::Merge); - bool found_mergee = (merge_it != arrows.end() && merge_it->second->input->componentType == ct); - - if (found_splitee) { - pencil_active = !pencil_active; - } - if (found_split) { - pencil_active = !pencil_active; - } - if (found_merge) { - pencil_active = !pencil_active; - } - if (found_mergee) { - pencil_active = !pencil_active; - } - if (pencil_active) { - std::cout << "----------"; - } - else { - std::cout << " "; - } - } - std::cout << std::endl; - - - // Vertical connector to split and merge joinees - std::cout << " "; - for (ComponentType ct : {ComponentType::Source, ComponentType::Filter, ComponentType::Map, ComponentType::Split, ComponentType::Merge, ComponentType::Reduce}) { - if (split_it != arrows.end() && split_it->second->output->componentType == ct) { - std::cout << "| "; - } - else if (merge_it != arrows.end() && merge_it->second->input->componentType == ct) { - std::cout << "| "; - } - else { - std::cout << " "; - } - } - std::cout << std::endl; - } - } - std::cout << "====================================================================" << std::endl; - } - - - void wire_and_print() { - - source = nullptr; - for (Level l : {Level::Timeslice, Level::Event, Level::Subevent} ) { - Arrow* last = nullptr; +TEST_CASE("TimeslicesTests_FineGrained") { - for (ComponentType ct : {ComponentType::Source, ComponentType::Filter, ComponentType::Map, ComponentType::Split, ComponentType::Merge, ComponentType::Reduce}) { - auto it = arrows.find({l, ct}); - if (it != arrows.end()) { - - Arrow* next = it->second; - if (last != nullptr) { - next->input = last; - last->output = next; - } - last = next; - - // Handle sources: - if (ct == ComponentType::Source) { - if (source != nullptr) { - throw std::runtime_error("Too many sources!"); - } - source = next; - } - } - } - } - - Level levels[] = {Level::Timeslice, Level::Event, Level::Subevent}; - for (int i=0; i<2; ++i) { - Level upper = levels[i]; - Level lower = levels[i+1]; - - auto split_it = arrows.find({upper, ComponentType::Split}); - if (split_it != arrows.end()) { - // Have splitter, so we need to find the lower-level arrow it connects to - bool found_joinee = false; - for (ComponentType ct : {ComponentType::Filter, ComponentType::Map, ComponentType::Split, ComponentType::Reduce}) { - auto it = arrows.find({lower, ct}); - if (it != arrows.end()) { - Arrow* joinee = it->second; - split_it->second->output = joinee; - joinee->input = split_it->second; - found_joinee = true; - break; - } - } - if (!found_joinee) { - throw std::runtime_error("Invalid topology: Unable to find successor to split"); - } - } - - auto merge_it = arrows.find({upper, ComponentType::Merge}); - if (merge_it != arrows.end()) { - // Have merger, so we need to find the lower-level arrow it connects to - bool found_joinee = false; - for (ComponentType ct : {ComponentType::Reduce, ComponentType::Merge, ComponentType::Map, ComponentType::Filter}) { - auto it = arrows.find({lower, ct}); - if (it != arrows.end()) { - Arrow* joinee = it->second; - merge_it->second->input = joinee; - joinee->output = merge_it->second; - found_joinee = true; - break; - } - } - if (!found_joinee) { - throw std::runtime_error("Invalid topology: Unable to find predecessor to merge"); - } - } - } - - print(); - } -}; - - - -TEST_CASE("MultiLevelTopologyBuilderTests") { - MultiLevelTopologyBuilder b; - - SECTION("Default topology") { - - b.add(Level::Event, ComponentType::Source); - b.add(Level::Event, ComponentType::Map); - - b.wire_and_print(); - - REQUIRE(b.has_no_input( Level::Event, ComponentType::Source )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Source, Level::Event, ComponentType::Map )); - REQUIRE(b.has_no_output( Level::Event, ComponentType::Map )); - } - - SECTION("Default with separated map/reduce phases") { - - b.add(Level::Event, ComponentType::Source); - b.add(Level::Event, ComponentType::Map); - b.add(Level::Event, ComponentType::Reduce); - - b.wire_and_print(); - - REQUIRE(b.has_no_input( Level::Event, ComponentType::Source )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Source, Level::Event, ComponentType::Map )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Map, Level::Event, ComponentType::Reduce )); - REQUIRE(b.has_no_output( Level::Event, ComponentType::Reduce )); - } - - SECTION("Timeslices topology") { - - b.add(Level::Timeslice, ComponentType::Source); - b.add(Level::Timeslice, ComponentType::Map); - b.add(Level::Timeslice, ComponentType::Split); - b.add(Level::Timeslice, ComponentType::Merge); - b.add(Level::Event, ComponentType::Map); - b.add(Level::Event, ComponentType::Reduce); - - b.wire_and_print(); - - REQUIRE(b.has_no_input( Level::Timeslice, ComponentType::Source )); - REQUIRE(b.has_connection( Level::Timeslice, ComponentType::Source, Level::Timeslice, ComponentType::Map )); - REQUIRE(b.has_connection( Level::Timeslice, ComponentType::Map, Level::Timeslice, ComponentType::Split )); - REQUIRE(b.has_no_output( Level::Timeslice, ComponentType::Merge )); - - REQUIRE(b.has_connection( Level::Event, ComponentType::Map, Level::Event, ComponentType::Reduce )); - REQUIRE(b.has_connection( Level::Timeslice, ComponentType::Split, Level::Event, ComponentType::Map )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Reduce, Level::Timeslice, ComponentType::Merge )); - - } - - SECTION("Three-level topology") { - - b.add(Level::Timeslice, ComponentType::Source); - b.add(Level::Timeslice, ComponentType::Map); - b.add(Level::Timeslice, ComponentType::Split); - b.add(Level::Timeslice, ComponentType::Merge); - b.add(Level::Event, ComponentType::Map); - b.add(Level::Event, ComponentType::Split); - b.add(Level::Event, ComponentType::Merge); - b.add(Level::Event, ComponentType::Reduce); - b.add(Level::Subevent, ComponentType::Map); - - b.wire_and_print(); - - REQUIRE(b.has_no_input( Level::Timeslice, ComponentType::Source )); - REQUIRE(b.has_connection( Level::Timeslice, ComponentType::Source, Level::Timeslice, ComponentType::Map )); - REQUIRE(b.has_connection( Level::Timeslice, ComponentType::Map, Level::Timeslice, ComponentType::Split )); - REQUIRE(b.has_connection( Level::Timeslice, ComponentType::Split, Level::Event, ComponentType::Map )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Map, Level::Event, ComponentType::Split )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Split, Level::Subevent, ComponentType::Map )); - REQUIRE(b.has_connection( Level::Subevent, ComponentType::Map, Level::Event, ComponentType::Merge )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Merge, Level::Event, ComponentType::Reduce )); - REQUIRE(b.has_connection( Level::Event, ComponentType::Reduce, Level::Timeslice, ComponentType::Merge )); - REQUIRE(b.has_no_output( Level::Timeslice, ComponentType::Merge )); + JApplication app; + app.SetParameterValue("jana:loglevel", "trace"); + app.SetParameterValue("jana:nevents", "5"); + + app.Add(new MyTimesliceSource); + app.Add(new MyTimesliceUnfolder); + app.Add(new MyEventProcessor); + app.Add(new JFactoryGeneratorT); + app.Add(new JFactoryGeneratorT); + app.SetTicker(true); - } + app.Initialize(); + auto ee = app.GetService(); + auto top = app.GetService(); + enum ArrowId {TS_SRC=0, TS_MAP=1, TS_UNF=2, TS_FLD=3, PH_MAP=4, PH_TAP=5}; + JArrow::FireResult result = JArrow::FireResult::NotRunYet; - SECTION("Too many sources") { - b.add(Level::Timeslice, ComponentType::Source); - b.add(Level::Timeslice, ComponentType::Map); - b.add(Level::Timeslice, ComponentType::Split); - b.add(Level::Timeslice, ComponentType::Merge); - b.add(Level::Event, ComponentType::Map); - b.add(Level::Event, ComponentType::Reduce); - b.add(Level::Event, ComponentType::Source); + result = ee->Fire(TS_SRC, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); - REQUIRE_THROWS(b.wire_and_print()); - } + REQUIRE(top->arrows[TS_SRC]->get_port(1).queue == top->queues[0]); + REQUIRE(top->pools[0]->GetCapacity() == 4); + REQUIRE(top->pools[0]->GetSize(0) == 3); + REQUIRE(top->queues[0]->GetSize(0) == 1); - SECTION("Inactive components") { - b.add(Level::Timeslice, ComponentType::Map); - b.add(Level::Timeslice, ComponentType::Split); - b.add(Level::Timeslice, ComponentType::Merge); - b.add(Level::Event, ComponentType::Map); - b.add(Level::Event, ComponentType::Reduce); - b.add(Level::Event, ComponentType::Source); + result = ee->Fire(TS_MAP, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); + REQUIRE(top->queues[0]->GetSize(0) == 0); + REQUIRE(top->queues[1]->GetSize(0) == 1); + + // Parent + result = ee->Fire(TS_UNF, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); + + // Child + result = ee->Fire(TS_UNF, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); + + result = ee->Fire(PH_MAP, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); + + result = ee->Fire(PH_TAP, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); + + result = ee->Fire(TS_FLD, 0); + REQUIRE(result == JArrow::FireResult::KeepGoing); - b.wire_and_print(); - // Timeslice-level components are deactivated - } + REQUIRE(top->pools[0]->GetSize(0) == 3); // Unfolder still has parent + REQUIRE(top->pools[1]->GetSize(0) == 4); // Child returned to pool + } - - - TEST_CASE("TimeslicesTests") { JApplication app; - app.SetParameterValue("log:info", "JScheduler,JArrow,JArrowProcessingController"); + app.SetParameterValue("jana:loglevel", "trace"); app.SetParameterValue("jana:nevents", "5"); app.Add(new MyTimesliceSource); diff --git a/src/programs/unit_tests/Topology/MultiLevelTopologyTests.h b/src/programs/unit_tests/Topology/MultiLevelTopologyTests.h index 851dc6ef0..c4aaffb08 100644 --- a/src/programs/unit_tests/Topology/MultiLevelTopologyTests.h +++ b/src/programs/unit_tests/Topology/MultiLevelTopologyTests.h @@ -32,7 +32,7 @@ struct MyTimesliceSource : public JEventSource { void GetEvent(std::shared_ptr event) override { // TODO: Insert something - jout << "MyTimesliceSource: Emitting " << event->GetEventNumber() << jendl; + LOG_INFO(GetLogger()) << "MyTimesliceSource: Emitting " << event->GetEventNumber() << LOG_END; } }; @@ -66,7 +66,7 @@ struct MyTimesliceUnfolder : public JEventUnfolder { // TODO: if (item == 3) { - jout << "Unfold found item 3, finishing join" << jendl; + LOG_INFO(GetLogger()) << "Unfold found item 3, finishing join" << LOG_END; // TODO: Insert protocluster into child return Result::NextChildNextParent; } @@ -95,7 +95,7 @@ struct MyEventProcessor : public JEventProcessor { void Process(const JEvent& event) override { process_called_count++; - jout << "MyEventProcessor: Processing " << event.GetEventNumber() << jendl; + LOG_INFO(GetLogger()) << "MyEventProcessor: Processing " << event.GetEventNumber() << LOG_END; auto clusters = event.Get("evt"); // TODO: Validate that the clusters make sense REQUIRE(init_called_count == 1); diff --git a/src/programs/unit_tests/Topology/QueueTests.cc b/src/programs/unit_tests/Topology/QueueTests.cc index 810aeb95a..27222cbf8 100644 --- a/src/programs/unit_tests/Topology/QueueTests.cc +++ b/src/programs/unit_tests/Topology/QueueTests.cc @@ -2,38 +2,48 @@ // Copyright 2020, Jefferson Science Associates, LLC. // Subject to the terms in the LICENSE file found in the top-level directory. -#include +#include #include "catch.hpp" -TEST_CASE("QueueTests_Basic") { +TEST_CASE("JEventQueueTests_Basic") { - JMailbox q; - REQUIRE(q.size() == 0); + JEventQueue sut(2,1); - int* item = new int {22}; - bool result = q.try_push(&item, 1, 0); - REQUIRE(q.size() == 1); - REQUIRE(result == true); + JEvent event1; + event1.SetEventNumber(1); - int* items[10]; - auto count = q.pop(items, 1, 10, 0); - REQUIRE(count == 1); - REQUIRE(q.size() == 0); - REQUIRE(*(items[0]) == 22); + JEvent event2; + event2.SetEventNumber(2); - *(items[0]) = 33; - items[1] = new int {44}; - items[2] = new int {55}; + JEvent event3; + event3.SetEventNumber(3); - size_t reserve_count = q.reserve(3, 5, 0); - REQUIRE(reserve_count == 5); + REQUIRE(sut.GetCapacity() == 2); + REQUIRE(sut.GetSize(0) == 0); - q.push_and_unreserve(items, 3, reserve_count, 0); - REQUIRE(q.size() == 3); + sut.Push(&event1, 0); + REQUIRE(sut.GetSize(0) == 1); - count = q.pop_and_reserve(items, 2, 2, 0); - REQUIRE(count == 2); - REQUIRE(q.size() == 1); + JEvent* e = sut.Pop(0); + REQUIRE(sut.GetSize(0) == 0); + + e = sut.Pop(0); + REQUIRE(e == nullptr); + + sut.Push(&event3, 0); + sut.Push(&event2, 0); + REQUIRE(sut.GetSize(0) == 2); + + e = sut.Pop(0); + REQUIRE(e->GetEventNumber() == 3); + REQUIRE(sut.GetSize(0) == 1); + + sut.Push(&event1, 0); + REQUIRE_THROWS(sut.Push(&event1, 0)); } + + + + diff --git a/src/programs/unit_tests/Topology/SubeventTests.cc b/src/programs/unit_tests/Topology/SubeventTests.cc index b4b5c8b9a..6a654314b 100644 --- a/src/programs/unit_tests/Topology/SubeventTests.cc +++ b/src/programs/unit_tests/Topology/SubeventTests.cc @@ -7,8 +7,9 @@ #include #include +#include #include -#include +#include #include #include @@ -46,8 +47,8 @@ TEST_CASE("Create subevent processor") { TEST_CASE("Basic subevent arrow functionality") { MyProcessor processor; - JMailbox*> events_in; - JMailbox*> events_out; + JMailbox events_in; + JMailbox events_out; JMailbox> subevents_in; JMailbox> subevents_out; @@ -67,7 +68,7 @@ TEST_CASE("Basic subevent arrow functionality") { SetCallbackStyle(CallbackStyle::ExpertMode); } Result Emit(JEvent& event) override { - if (GetEventCount() == 10) return Result::FailureFinished; + if (GetEmittedEventCount() == 10) return Result::FailureFinished; std::vector inputs; inputs.push_back(new MyInput(22,3.6)); inputs.push_back(new MyInput(23,3.5)); @@ -95,18 +96,19 @@ TEST_CASE("Basic subevent arrow functionality") { SECTION("Execute subevent arrows end-to-end") { JApplication app; - app.SetParameterValue("log:info", "JWorker,JScheduler,JArrow,JArrowProcessingController,JEventProcessorArrow"); app.SetTimeoutEnabled(false); app.SetTicker(false); auto topology = app.GetService(); topology->set_configure_fn([&](JTopologyBuilder& topology) { - auto source_arrow = new JEventSourceArrow("simpleSource", - {new SimpleSource}, - &events_in, - topology.event_pool); - auto proc_arrow = new JEventProcessorArrow("simpleProcessor", &events_out, - nullptr, topology.event_pool); + + auto source_arrow = new JEventSourceArrow("simpleSource", {new SimpleSource}); + source_arrow->attach(topology.event_pool, JEventSourceArrow::EVENT_IN); + source_arrow->attach(&events_in, JEventSourceArrow::EVENT_OUT); + + auto proc_arrow = new JEventMapArrow("simpleProcessor"); + proc_arrow->attach(&events_out, 0); + proc_arrow->attach(topology.event_pool, 1); proc_arrow->add_processor(new SimpleProcessor); topology.arrows.push_back(source_arrow); @@ -114,6 +116,7 @@ TEST_CASE("Basic subevent arrow functionality") { topology.arrows.push_back(subprocess_arrow); topology.arrows.push_back(merge_arrow); topology.arrows.push_back(proc_arrow); + source_arrow->attach(split_arrow); split_arrow->attach(subprocess_arrow); subprocess_arrow->attach(merge_arrow); diff --git a/src/programs/unit_tests/Topology/TestTopologyComponents.h b/src/programs/unit_tests/Topology/TestTopologyComponents.h index e4eaec88c..c131d1d96 100644 --- a/src/programs/unit_tests/Topology/TestTopologyComponents.h +++ b/src/programs/unit_tests/Topology/TestTopologyComponents.h @@ -4,89 +4,121 @@ #pragma once -#include - -#include -#include -#include -#include "MapArrow.h" +#include +#include +struct EventData { + int x = 0; + double y = 0.0; + double z = 0.0; +}; -struct RandIntSource : public JPipelineArrow { +struct RandIntArrow : public JArrow { size_t emit_limit = 20; // How many to emit size_t emit_count = 0; // How many emitted so far int emit_sum = 0; // Sum of all ints emitted so far - JLogger logger; - RandIntSource(std::string name, JPool* pool, JMailbox* output_queue) - : JPipelineArrow(name, false, true, false, nullptr, output_queue, pool) {} + RandIntArrow(std::string name, JEventPool* pool, JEventQueue* output_queue) { + set_name(name); + set_is_source(true); + create_ports(1, 1); + attach(pool, 0); + attach(output_queue, 1); + } - void process(int* item, bool& success, JArrowMetrics::Status& status) { + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { if (emit_count >= emit_limit) { - success = false; - status = JArrowMetrics::Status::Finished; + outputs[0] = {event, 0}; // Send event back to the input pool + output_count = 1; + status = JArrow::FireResult::Finished; return; } - *item = 7; - emit_sum += *item; + + auto data = new EventData {7}; + event->Insert(data, "first"); + + emit_sum += data->x; emit_count += 1; - LOG_DEBUG(logger) << "RandIntSource emitted event " << emit_count << " with value " << *item << LOG_END; - success = true; - status = (emit_count == emit_limit) ? JArrowMetrics::Status::Finished : JArrowMetrics::Status::KeepGoing; - // This design lets us declare Finished immediately on the last event, instead of after - } - void initialize() override { - LOG_INFO(logger) << "RandIntSource.initialize() called!" << LOG_END; - }; + outputs[0] = {event, 1}; // Send event back to the input pool + output_count = 1; + status = (emit_count == emit_limit) ? JArrow::FireResult::Finished : JArrow::FireResult::KeepGoing; + // This design lets us declare Finished immediately on the last event, instead of after - void finalize() override { - LOG_INFO(logger) << "RandIntSource.finalize() called!" << LOG_END; + LOG_DEBUG(JArrow::m_logger) << "RandIntSource emitted event " << emit_count << " with value " << data->x << LOG_END; } }; -struct MultByTwoProcessor : public ParallelProcessor { +struct MultByTwoArrow : public JArrow { - double* process(int* x) override { - return new double(*x * 2.0); + MultByTwoArrow(std::string name, JEventQueue* input_queue, JEventQueue* output_queue) { + set_name(name); + set_is_parallel(true); + create_ports(1, 1); + attach(input_queue, 0); + attach(output_queue, 1); } -}; - -struct SubOneProcessor : public JPipelineArrow { + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { + auto prev = event->Get("first"); + auto x = prev.at(0)->x; + auto next = new EventData { .x=x, .y=x*2.0 }; + event->Insert(next, "second"); - SubOneProcessor(std::string name, JMailbox* input_queue, JMailbox* output_queue) - : JPipelineArrow(name, true, false, false, input_queue, output_queue, nullptr) {} - - void process(double* item, bool&, JArrowMetrics::Status&) { - *item -= 1; + outputs[0] = {event, 1}; + output_count = 1; + status = JArrow::FireResult::KeepGoing; } }; +struct SubOneArrow : public JArrow { + + SubOneArrow(std::string name, JEventQueue* input_queue, JEventQueue* output_queue) { + set_name(name); + set_is_parallel(true); + create_ports(1, 1); + attach(input_queue, 0); + attach(output_queue, 1); + } -template -struct SumSink : public JPipelineArrow, T> { + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { + auto prev = event->Get("second"); + auto x = prev.at(0)->x; + auto y = prev.at(0)->y; + auto z = y - 1; + auto next = new EventData { .x=x, .y=y, .z=z }; + event->Insert(next, "third"); + + outputs[0] = {event, 1}; + output_count = 1; + status = JArrow::FireResult::KeepGoing; + } +}; - T sum = 0; +struct SumArrow : public JArrow { - SumSink(std::string name, JMailbox* input_queue, JPool* pool) - : JPipelineArrow,T>(name, false, false, true, input_queue, nullptr, pool) {} + double sum = 0; - void process(T* item, bool&, JArrowMetrics::Status&) { - sum += *item; - LOG_DEBUG(JArrow::m_logger) << "SumSink.outprocess() called!" << LOG_END; + SumArrow(std::string name, JEventQueue* input_queue, JEventPool* pool) { + set_name(name); + set_is_sink(true); + create_ports(1, 1); + attach(input_queue, 0); + attach(pool, 1); } - void initialize() override { - LOG_INFO(JArrow::m_logger) << "SumSink.initialize() called!" << LOG_END; - }; + void fire(JEvent* event, OutputData& outputs, size_t& output_count, JArrow::FireResult& status) { + auto prev = event->Get("third"); + auto z = prev.at(0)->z; + sum += z; - void finalize() override { - LOG_INFO(JArrow::m_logger) << "SumSink.finalize() called!" << LOG_END; - }; + outputs[0] = {event, 1}; + output_count = 1; + status = JArrow::FireResult::KeepGoing; + } }; diff --git a/src/programs/unit_tests/Topology/TopologyTests.cc b/src/programs/unit_tests/Topology/TopologyTests.cc deleted file mode 100644 index d13e89c48..000000000 --- a/src/programs/unit_tests/Topology/TopologyTests.cc +++ /dev/null @@ -1,264 +0,0 @@ - -// Copyright 2020, Jefferson Science Associates, LLC. -// Subject to the terms in the LICENSE file found in the top-level directory. - -#include "catch.hpp" - -#include "JANA/Topology/JTopologyBuilder.h" -#include - -#include "TestTopologyComponents.h" -#include -#include - - -JArrowMetrics::Status step(JArrow* arrow) { - - JArrowMetrics metrics; - arrow->execute(metrics, 0); - auto status = metrics.get_last_status(); - return status; -} - - - - -TEST_CASE("JTopology: Basic functionality") { - - auto q1 = new JMailbox(); - auto q2 = new JMailbox(); - auto q3 = new JMailbox(); - - auto p1 = new JPool(0,1,false); - auto p2 = new JPool(0,1,false); - p1->init(); - p2->init(); - - MultByTwoProcessor processor; - - auto emit_rand_ints = new RandIntSource("emit_rand_ints", p1, q1); - auto multiply_by_two = new MapArrow("multiply_by_two", processor, q1, q2); - auto subtract_one = new SubOneProcessor("subtract_one", q2, q3); - auto sum_everything = new SumSink("sum_everything", q3, p2); - - auto topology = std::make_shared(); - - emit_rand_ints->attach(multiply_by_two); - multiply_by_two->attach(subtract_one); - subtract_one->attach(sum_everything); - - topology->arrows.push_back(emit_rand_ints); - topology->arrows.push_back(multiply_by_two); - topology->arrows.push_back(subtract_one); - topology->arrows.push_back(sum_everything); - - auto logger = JLogger(JLogger::Level::INFO); - topology->SetLogger(logger); - emit_rand_ints->set_logger(logger); - multiply_by_two->set_logger(logger); - subtract_one->set_logger(logger); - sum_everything->set_logger(logger); - - emit_rand_ints->set_chunksize(1); - - JScheduler scheduler(topology); - scheduler.logger = logger; - - SECTION("Before anything runs...") { - - // All queues are empty, none are finished - REQUIRE(multiply_by_two->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - REQUIRE(subtract_one->get_pending() == 0); - } - - SECTION("When nothing is in the input queue...") { - - LOG_INFO(logger) << "Nothing has run yet; should be empty" << LOG_END; - - //topology.log_queue_status(); - step(multiply_by_two); - step(subtract_one); - step(sum_everything); - step(multiply_by_two); - step(subtract_one); - step(sum_everything); - //topology.log_queue_status(); - - // All `execute` operations are no-ops - REQUIRE(multiply_by_two->get_pending() == 0); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - } - - SECTION("After emitting") { - LOG_INFO(logger) << "After emitting; should be something in q0" << LOG_END; - - scheduler.run_topology(1); - step(emit_rand_ints); - - REQUIRE(multiply_by_two->get_pending() == 1); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - - step(emit_rand_ints); - step(emit_rand_ints); - step(emit_rand_ints); - step(emit_rand_ints); - - REQUIRE(multiply_by_two->get_pending() == 5); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - - } - - SECTION("Running each stage sequentially yields the correct results") { - - //LOG_INFO(logger) << "Running each stage sequentially yields the correct results" << LOG_END; - scheduler.run_topology(1); - - for (int i = 0; i < 20; ++i) { - step(emit_rand_ints); - REQUIRE(multiply_by_two->get_pending() == 1); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - - step(multiply_by_two); - REQUIRE(multiply_by_two->get_pending() == 0); - REQUIRE(subtract_one->get_pending() == 1); - REQUIRE(sum_everything->get_pending() == 0); - - step(subtract_one); - REQUIRE(multiply_by_two->get_pending() == 0); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 1); - - step(sum_everything); - REQUIRE(multiply_by_two->get_pending() == 0); - REQUIRE(subtract_one->get_pending() == 0); - REQUIRE(sum_everything->get_pending() == 0); - } - - REQUIRE(sum_everything->sum == (7 * 2.0 - 1) * 20); - } - - SECTION("Running each stage in random order (sequentially) yields the correct results") { - //LOG_INFO(logger) << "Running each stage in arbitrary order yields the correct results" << LOG_END; - - JArrow* arrows[] = {emit_rand_ints, - multiply_by_two, - subtract_one, - sum_everything}; - - std::map results; - results["emit_rand_ints"] = JArrowMetrics::Status::KeepGoing; - results["multiply_by_two"] = JArrowMetrics::Status::KeepGoing; - results["subtract_one"] = JArrowMetrics::Status::KeepGoing; - results["sum_everything"] = JArrowMetrics::Status::KeepGoing; - - // Put something in the queue to get started - scheduler.run_topology(1); - step(emit_rand_ints); - - bool work_left = true; - JBenchUtils bench_utils = JBenchUtils(); - bench_utils.set_seed(5, "TopologyTests"); - while (work_left) { - // Pick a random arrow - JArrow* arrow = arrows[bench_utils.randint(0, 3)]; - - auto name = arrow->get_name(); - JArrowMetrics metrics; - arrow->execute(metrics, 0); - auto res = metrics.get_last_status(); - results[name] = res; - //LOG_TRACE(logger) << name << " => " - // << to_string(res) << LOG_END; - - size_t pending = multiply_by_two->get_pending() + - subtract_one->get_pending() + - sum_everything->get_pending(); - - work_left = (pending > 0); - - for (auto pair : results) { - if (pair.second == JArrowMetrics::Status::KeepGoing) { work_left = true; } - } - } - - //topology.log_queue_status(); - REQUIRE(sum_everything->sum == (7 * 2.0 - 1) * 20); - } - SECTION("Finished flag propagates") { - - scheduler.run_topology(1); - auto ts = scheduler.get_topology_state(); - JArrowMetrics::Status status; - - REQUIRE(ts.arrow_states[0].status == JScheduler::ArrowStatus::Active); - REQUIRE(ts.arrow_states[1].status == JScheduler::ArrowStatus::Active); - REQUIRE(ts.arrow_states[2].status == JScheduler::ArrowStatus::Active); - REQUIRE(ts.arrow_states[3].status == JScheduler::ArrowStatus::Active); - - for (int i = 0; i < 20; ++i) { - status = step(emit_rand_ints); - } - scheduler.next_assignment(0, emit_rand_ints, status); - ts = scheduler.get_topology_state(); - - REQUIRE(ts.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[1].status == JScheduler::ArrowStatus::Active); - REQUIRE(ts.arrow_states[2].status == JScheduler::ArrowStatus::Active); - REQUIRE(ts.arrow_states[3].status == JScheduler::ArrowStatus::Active); - - for (int i = 0; i < 20; ++i) { - step(multiply_by_two); - } - scheduler.next_assignment(0, multiply_by_two, status); - ts = scheduler.get_topology_state(); - - REQUIRE(ts.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[1].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[2].status == JScheduler::ArrowStatus::Active); - REQUIRE(ts.arrow_states[3].status == JScheduler::ArrowStatus::Active); - - for (int i = 0; i < 20; ++i) { - step(subtract_one); - } - scheduler.next_assignment(0, subtract_one, status); - ts = scheduler.get_topology_state(); - - REQUIRE(ts.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[1].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[2].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[3].status == JScheduler::ArrowStatus::Active); - - for (int i = 0; i < 20; ++i) { - step(sum_everything); - } - scheduler.next_assignment(0, sum_everything, status); - ts = scheduler.get_topology_state(); - - REQUIRE(ts.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[1].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[2].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(ts.arrow_states[3].status == JScheduler::ArrowStatus::Finalized); - - } - - SECTION("Running from inside JApplication returns the correct answer") { - JApplication app; - app.ProvideService(topology); // Override the builtin one - - REQUIRE(sum_everything->sum == 0); - - app.Run(true); - auto scheduler = app.GetService()->get_scheduler(); - - auto ts = scheduler->get_topology_state(); - REQUIRE(ts.current_topology_status == JScheduler::TopologyStatus::Finalized); - REQUIRE(ts.arrow_states[0].status == JScheduler::ArrowStatus::Finalized); - REQUIRE(sum_everything->sum == 20 * 13); - - } -} diff --git a/src/programs/unit_tests/Utils/JCallGraphRecorderTests.cc b/src/programs/unit_tests/Utils/JCallGraphRecorderTests.cc index 4196a5347..8bfa76c0c 100644 --- a/src/programs/unit_tests/Utils/JCallGraphRecorderTests.cc +++ b/src/programs/unit_tests/Utils/JCallGraphRecorderTests.cc @@ -5,6 +5,7 @@ #include #include #include "JANA/JEvent.h" +#include "JANA/JFactoryGenerator.h" TEST_CASE("Test topological sort algorithm in isolation") { @@ -68,13 +69,11 @@ struct FacD: public JFactoryT { TEST_CASE("Test topological sort algorithm using actual Factories") { JApplication app; - JFactorySet* factories = new JFactorySet; - factories->Add(new FacA()); - factories->Add(new FacB()); - factories->Add(new FacC()); - factories->Add(new FacD()); + app.Add(new JFactoryGeneratorT()); + app.Add(new JFactoryGeneratorT()); + app.Add(new JFactoryGeneratorT()); + app.Add(new JFactoryGeneratorT()); auto event = std::make_shared(&app); - event->SetFactorySet(factories); event->GetJCallGraphRecorder()->SetEnabled(); event->Get(); auto result = event->GetJCallGraphRecorder()->TopologicalSort(); diff --git a/src/python/common/janapy.h b/src/python/common/janapy.h index 233047bba..3b6154e4c 100644 --- a/src/python/common/janapy.h +++ b/src/python/common/janapy.h @@ -31,7 +31,6 @@ static JApplication *pyjapp = nullptr; inline void janapy_Start(void) { PY_INITIALIZED = true; } inline void janapy_Quit(bool skip_join=false) { pyjapp->Quit(skip_join); } inline void janapy_Stop(bool wait_until_idle=false) { pyjapp->Stop(wait_until_idle); } -inline void janapy_Resume(void) { pyjapp->Resume(); } inline bool janapy_IsInitialized(void) { return pyjapp->IsInitialized(); } inline bool janapy_IsQuitting(void) { return pyjapp->IsQuitting(); } @@ -135,7 +134,6 @@ m.def("Start", &janapy_Start, "Allow m.def("Run", &janapy_Run, "Begin processing events (use when running python as an extension)"); \ m.def("Quit", &janapy_Quit, "Tell JANA to quit gracefully", py::arg("skip_join")=false); \ m.def("Stop", &janapy_Stop, "Tell JANA to (temporarily) stop event processing. If optional agrument is True then block until all threads are stopped."); \ -m.def("Resume", &janapy_Resume, "Tell JANA to resume event processing."); \ \ m.def("IsInitialized", &janapy_IsInitialized, "Check if JApplication has already been initialized."); \ m.def("IsQuitting", &janapy_IsQuitting, "Check if JApplication is in the process of quitting."); \ diff --git a/src/python/modules/jana/jana_module.cc b/src/python/modules/jana/jana_module.cc index e2ec8e238..ac4d07842 100644 --- a/src/python/modules/jana/jana_module.cc +++ b/src/python/modules/jana/jana_module.cc @@ -5,6 +5,7 @@ #include #include #include +#include // Something to throw that makes a nicer error message class PYTHON_MODULE_STARTUP_FAILED{public: PYTHON_MODULE_STARTUP_FAILED(){}};