Add a new test producing histograms and improve comments in tests (#208)

test/k4FWCoreTest/CMakeLists.txt

@@ -129,9 +129,10 @@ add_test_with_env(FunctionalProducerRuntimeCollections options/ExampleFunctional
 add_test_with_env(FunctionalTransformerRuntimeCollections options/ExampleFunctionalTransformerRuntimeCollections.py)
 add_test_with_env(FunctionalTransformerRuntimeEmpty options/ExampleFunctionalTransformerRuntimeEmpty.py)
 add_test_with_env(FunctionalTransformerRuntimeCollectionsMultiple options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py)
+add_test_with_env(FunctionalTransformerHist options/ExampleFunctionalTransformerHist.py)
 
 add_test(NAME FunctionalCheckFiles COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/options/CheckOutputFiles.py)
-set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc")
+set_tests_properties(FunctionalCheckFiles PROPERTIES DEPENDS "FunctionalFile;FunctionalMTFile;FunctionalMultipleFile;FunctionalOutputCommands;FunctionalProducerAbsolutePath;FunctionalTransformerRuntimeEmpty;FunctionalMix;FunctionalMixIOSvc;FunctionalTransformerHist")
 # Do this after checking the files not to overwrite them
 add_test_with_env(FunctionalFile_toolong options/ExampleFunctionalFile.py -n 999 PROPERTIES DEPENDS FunctionalCheckFiles PASS_REGULAR_EXPRESSION
                   "Application Manager Terminated successfully with a user requested ScheduledStop")

test/k4FWCoreTest/options/CheckOutputFiles.py

@@ -23,6 +23,7 @@
 
     print(f'PYTHONPATH={os.environ["PYTHONPATH"]}')
     raise
+import ROOT
 
 
 def check_collections(filename, names):
@@ -62,7 +63,7 @@ def check_collections(filename, names):
     "functional_transformer_multiple_output_commands.root",
     ["VectorFloat", "MCParticles1", "MCParticles2", "SimTrackerHits", "TrackerHits"],
 )
-check_collections("/tmp/a/b/c/output_k4test_exampledata_producer.root", ["MCParticles"])
+check_collections("/tmp/a/b/c/functional_producer.root", ["MCParticles"])
 check_collections(
     "functional_transformer_runtime_empty.root",
     ["MCParticles0", "MCParticles1", "MCParticles2"],
@@ -110,11 +111,21 @@ def check_collections(filename, names):
 
 # Not working, collections produced by functional algorithms are not being written to the file
 # check_collections(
-#     "output_k4test_exampledata_functional_mix.root",
+#     "functional_mix.root",
 #     mix_collections,
 # )
 
 check_collections(
-    "output_k4test_exampledata_functional_mix_iosvc.root",
+    "functional_mix_iosvc.root",
     mix_collections,
 )
+
+f = ROOT.TFile.Open("functional_transformer_hist.root")
+for i in range(2):
+    if (
+        str(f.GetListOfKeys()[i])
+        != f"Name: ExampleFunctionalTransformerHist{i+1} Title: ExampleFunctionalTransformerHist{i+1}"
+    ):
+        raise RuntimeError(
+            "Directory structure does not match expected for functional_transformer_hist.root"
+        )

test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollections.py

@@ -17,8 +17,7 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a producer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example using a consumer that takes any number of collections
 
 from Gaudi.Configuration import INFO
 from Configurables import (
@@ -42,7 +41,6 @@
 )
 consumer = ExampleFunctionalConsumerRuntimeCollections(
     "Consumer",
-    # InputCollection="MCParticles0 MCParticles1 MCParticles2",
     InputCollection=["MCParticles0", "MCParticles1", "MCParticles2"],
     Offset=0,
 )

test/k4FWCoreTest/options/ExampleFunctionalConsumerRuntimeCollectionsMultiple.py

@@ -17,8 +17,9 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a producer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example using several producer of multiple collections and a
+# consumer with multiple inputs that can read any number of MCParticle
+# collections
 
 from Gaudi.Configuration import INFO
 from Configurables import (

test/k4FWCoreTest/options/ExampleFunctionalFile.py

@@ -17,16 +17,16 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example reading from a file and using a transformer to create new
+# data
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformer
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = "output_k4test_exampledata_producer.root"
+svc.input = "functional_producer.root"
 svc.output = "functional_transformer.root"
 
 transformer = ExampleFunctionalTransformer(

test/k4FWCoreTest/options/ExampleFunctionalFileMultiple.py

@@ -17,22 +17,20 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example reading from a file and transforming multiple collections
+# and then writing to a file
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformerMultiple
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = "output_k4test_exampledata_producer_multiple.root"
+svc.input = "functional_producer_multiple.root"
 svc.output = "functional_transformer_multiple.root"
 
 transformer = ExampleFunctionalTransformerMultiple(
     "Transformer",
-    # InputCollection="MCParticles",
-    # OutputCollection="NewMCParticles")
 )
 
 mgr = ApplicationMgr(

test/k4FWCoreTest/options/ExampleFunctionalMTFile.py

@@ -17,8 +17,8 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example of reading data from a file, transforming it and writing to
+# an output file in a multithreaded environment
 
 from Gaudi.Configuration import INFO, WARNING
 from Configurables import ExampleFunctionalTransformer, ExampleFunctionalConsumer
@@ -41,7 +41,7 @@
 scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, ShowDataFlow=True, OutputLevel=WARNING)
 
 svc = IOSvc("IOSvc")
-svc.input = "output_k4test_exampledata_producer_multiple.root"
+svc.input = "functional_producer_multiple.root"
 svc.output = "functional_transformerMT.root"
 
 consumer = ExampleFunctionalConsumer(

test/k4FWCoreTest/options/ExampleFunctionalMTMemory.py

@@ -17,8 +17,8 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example producing, transforming and consuming data in a
+# multithreaded environment
 
 from Gaudi.Configuration import INFO, WARNING
 from Configurables import (

test/k4FWCoreTest/options/ExampleFunctionalMemory.py

@@ -17,8 +17,7 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example producing, transforming and consuming data in memory
 
 from Gaudi.Configuration import INFO
 from Configurables import (

test/k4FWCoreTest/options/ExampleFunctionalOutputCommands.py

@@ -17,16 +17,15 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example using the outputCommands to drop collections from the output file
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalTransformerMultiple
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr, IOSvc
 
 svc = IOSvc("IOSvc")
-svc.input = "output_k4test_exampledata_producer_multiple.root"
+svc.input = "functional_producer_multiple.root"
 svc.output = "functional_transformer_multiple_output_commands.root"
 svc.outputCommands = [
     "drop Tracks",

test/k4FWCoreTest/options/ExampleFunctionalProducer.py

@@ -23,29 +23,22 @@
 from Configurables import ExampleFunctionalProducer
 from Configurables import EventDataSvc
 from k4FWCore import ApplicationMgr, IOSvc
-from Configurables import Writer
 from k4FWCore.parseArgs import parser
 
 parser.add_argument("--second", action="store_true")
 args = parser.parse_known_args()
 
 iosvc = IOSvc("IOSvc")
-name = (
-    "output_k4test_exampledata_producer.root"
-    if not args[0].second
-    else "output_k4test_exampledata_producer2.root"
-)
+name = "functional_producer.root" if not args[0].second else "functional_producer2.root"
 iosvc.output = name
 # Collections can be dropped
 # out.outputCommands = ["drop *"]
 
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
 
-writer = Writer("Writer")
-
 ApplicationMgr(
-    TopAlg=[producer, writer],
+    TopAlg=[producer],
     EvtSel="NONE",
     EvtMax=10 if not args[0].second else 20,
     ExtSvc=[EventDataSvc("EventDataSvc")],

test/k4FWCoreTest/options/ExampleFunctionalProducerAbsolutePath.py

@@ -17,15 +17,16 @@
 # limitations under the License.
 #
 
-# This is an example using a producer with a single output and saving that to a file
+# This is an example writing to a file with an absolute path to check that
+# the file is created in the expected location
 
 from Gaudi.Configuration import INFO
 from Configurables import EventDataSvc
 from Configurables import ExampleFunctionalProducer
 from k4FWCore import ApplicationMgr, IOSvc
 
 io = IOSvc("IOSvc")
-io.output = "/tmp/a/b/c/output_k4test_exampledata_producer.root"
+io.output = "/tmp/a/b/c/functional_producer.root"
 
 producer = ExampleFunctionalProducer("ExampleFunctionalProducer")
 

test/k4FWCoreTest/options/ExampleFunctionalProducerMultiple.py

@@ -17,7 +17,8 @@
 # limitations under the License.
 #
 
-# This is an example using a producer with a multiple outputs and saving that to a file
+# This is an example using a producer with multiple outputs and saving the new
+# collections to a file
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalProducerMultiple
@@ -26,7 +27,7 @@
 
 
 iosvc = IOSvc("IOSvc")
-iosvc.output = "output_k4test_exampledata_producer_multiple.root"
+iosvc.output = "functional_producer_multiple.root"
 # Collections can be dropped
 # out.outputCommands = ["drop *"]
 

test/k4FWCoreTest/options/ExampleFunctionalProducerRuntimeCollections.py

@@ -17,8 +17,8 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example using a producer that creates any number of collections and
+# then reading those with consumers
 
 from Gaudi.Configuration import INFO
 from Configurables import (

test/k4FWCoreTest/options/ExampleFunctionalSeveralInputFiles.py

@@ -17,8 +17,7 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example reading from multiple files, which for Gaudi will look like a single file
 
 from Gaudi.Configuration import INFO
 from Configurables import ExampleFunctionalConsumer
@@ -27,8 +26,8 @@
 
 svc = IOSvc("IOSvc")
 svc.input = [
-    "output_k4test_exampledata_producer.root",
-    "output_k4test_exampledata_producer2.root",
+    "functional_producer.root",
+    "functional_producer2.root",
 ]
 
 consumer = ExampleFunctionalConsumer("Consumer", InputCollection=["MCParticles"], Offset=0)

test/k4FWCoreTest/options/ExampleFunctionalTransformerHist.py

@@ -0,0 +1,82 @@
+#
+# Copyright (c) 2014-2024 Key4hep-Project.
+#
+# This file is part of Key4hep.
+# See https://key4hep.github.io/key4hep-doc/ for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is an example using two producers that create histograms and persist them to a ROOT file
+
+from Gaudi.Configuration import INFO
+from Configurables import ExampleFunctionalProducer, ExampleFunctionalTransformerHist
+from k4FWCore import ApplicationMgr
+from Configurables import RootHistSvc
+from Configurables import Gaudi__Histograming__Sink__Root as RootHistoSink
+from Configurables import HiveWhiteBoard, HiveSlimEventLoopMgr, AvalancheSchedulerSvc
+
+# Multithreaded also works fine with histograms
+multithreaded = False
+threads = 2
+slots = 3
+if multithreaded:
+    whiteboard = HiveWhiteBoard(
+        "EventDataSvc",
+        EventSlots=slots,
+        ForceLeaves=True,
+    )
+    slimeventloopmgr = HiveSlimEventLoopMgr(
+        "HiveSlimEventLoopMgr", SchedulerName="AvalancheSchedulerSvc", OutputLevel=INFO
+    )
+
+    scheduler = AvalancheSchedulerSvc(ThreadPoolSize=threads, OutputLevel=INFO)
+    scheduler.ShowDataDependencies = True
+    scheduler.ShowDataFlow = True
+    scheduler.ShowControlFlow = True
+
+
+producer1 = ExampleFunctionalProducer(
+    "ExampleFunctionalProducer1", OutputCollection=["MCParticles1"]
+)
+producer2 = ExampleFunctionalProducer(
+    "ExampleFunctionalProducer2", OutputCollection=["MCParticles2"]
+)
+
+transformer1 = ExampleFunctionalTransformerHist(
+    "ExampleFunctionalTransformerHist1",
+    InputCollection=["MCParticles1"],
+    OutputCollection=["dummy1"],
+)
+
+transformer2 = ExampleFunctionalTransformerHist(
+    "ExampleFunctionalTransformerHist2",
+    InputCollection=["MCParticles2"],
+    OutputCollection=["dummy2"],
+    FirstParticle=False,
+)
+
+hps = RootHistSvc("HistogramPersistencySvc")
+root_hist_svc = RootHistoSink("RootHistoSink")
+root_hist_svc.FileName = "functional_transformer_hist.root"
+
+
+app = ApplicationMgr(
+    TopAlg=[producer1, producer2, transformer1, transformer2],
+    EvtSel="NONE",
+    EvtMax=10,
+    ExtSvc=[root_hist_svc] + ([whiteboard] if multithreaded else []),
+    OutputLevel=INFO,
+    HistogramPersistency="ROOT",
+    EventLoop="EventLoopMgr" if not multithreaded else slimeventloopmgr,
+)

test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeCollectionsMultiple.py

@@ -17,8 +17,9 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example using multiple producers, a transformer that can read any
+# number of collections for its input and multiple consumers to check the
+# transformed collections
 
 from Gaudi.Configuration import INFO
 from Configurables import (

test/k4FWCoreTest/options/ExampleFunctionalTransformerRuntimeEmpty.py

@@ -17,8 +17,8 @@
 # limitations under the License.
 #
 
-# This is an example reading from a file and using a consumer with several inputs
-# to check that the contents of the file are the expected ones
+# This is an example using a transformer that takes and produces any number of
+# collection to produce no collections
 
 from Gaudi.Configuration import INFO
 from Configurables import (