From 2df6a42925218a52e89f8f811e4c77a400a28633 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 11:32:21 +0200 Subject: [PATCH 01/15] Avoid unnecessary copies of strings and vectors --- src/cpp/include/UTIL/CheckCollections.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index 03b3b1621..fe007ecae 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -56,15 +56,15 @@ namespace UTIL { /** Add a collection with (name,type) that should be added to events in patchEvent(). */ - void addPatchCollection(const std::string name, std::string type){ - _patchCols.push_back( {name, type} ) ; + void addPatchCollection(std::string name, std::string type){ + _patchCols.emplace_back( std::move(name), std::move(type) ) ; } /** Add a all collections as Vector(name,type), e.g. retrieved from getMissingCollections() that should be added to events in patchEvent(). */ void addPatchCollections(Vector cols){ - for(const auto& p : cols) - _patchCols.push_back( p ) ; + for(auto&& p : cols) + _patchCols.emplace_back( std::move(p) ) ; } /** Add and empty collection to the event for any collection that is in patchCollections and not in the Event From 8e696e480b398aeae7fe9562c83c9ee95748e2ae Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 11:32:56 +0200 Subject: [PATCH 02/15] Remove unused variable --- src/cpp/src/UTIL/CheckCollections.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index 2c58c7b1d..4f1800fda 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -57,7 +57,7 @@ namespace UTIL{ else { typeString = col->getTypeName(); } - const auto[ itx, inserted] = _map.emplace( name, std::make_pair( std::move(typeString) , 0 ) ) ; + const auto[ itx, _ ] = _map.emplace( name, std::make_pair( std::move(typeString) , 0 ) ) ; it = itx ; } From 4fb0e34eca11499d26026ac33719c6a7dd749fbe Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 11:33:10 +0200 Subject: [PATCH 03/15] Fix some trailing whitespaces --- src/cpp/src/UTIL/CheckCollections.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index 4f1800fda..fb7461698 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -17,15 +17,15 @@ namespace UTIL{ void CheckCollections::checkFile( const std::string& fileName, bool quiet){ - MT::LCReader lcReader(MT::LCReader::directAccess) ; + MT::LCReader lcReader(MT::LCReader::directAccess) ; lcReader.open( fileName ) ; //----------- the event loop ----------- while( const auto evt = lcReader.readNextEventHeader() ) { - + const auto* colNames = evt->getCollectionNames() ; - + for(const auto& name : *colNames){ - + auto it = _map.find( name ) ; if( it == _map.end() ){ @@ -38,7 +38,7 @@ namespace UTIL{ // be unset after reading in order for the reading of the headers to // function properly. std::string typeString; - if (col->getTypeName() == "LCRelation"){ + if (col->getTypeName() == "LCRelation"){ lcReader.setReadCollectionNames({name}); auto fullEvt = lcReader.readEvent(evt->getRunNumber(), evt->getEventNumber()); lcReader.setReadCollectionNames({}); @@ -89,7 +89,7 @@ namespace UTIL{ s.push_back( {e.first, e.second.first }) ; } return s ; - } + } // Obtain the from and to type from the encoded "LCRelation[From,To]" std::tuple getToFromType(const std::string_view fullType) { @@ -145,7 +145,7 @@ namespace UTIL{ } if (minimal == false){ for(auto e : _map ){ - + if( e.second.second != _nEvents ) os << " " << std::setw(width) << std::left << e.first << " " < Date: Fri, 21 Jun 2024 11:36:40 +0200 Subject: [PATCH 04/15] Fix docstring --- src/cpp/include/UTIL/CheckCollections.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index fe007ecae..f4022a4fc 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -60,7 +60,7 @@ namespace UTIL { _patchCols.emplace_back( std::move(name), std::move(type) ) ; } - /** Add a all collections as Vector(name,type), e.g. retrieved from getMissingCollections() that should be added to events in patchEvent(). + /** Add all collections as Vector(name,type), e.g. retrieved from getMissingCollections() that should be added to events in patchEvent(). */ void addPatchCollections(Vector cols){ for(auto&& p : cols) From 97c29a33e0e8fd4b5f54853526272fc13fbb83f8 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 13:21:23 +0200 Subject: [PATCH 05/15] Improve readability with structured bindings --- src/cpp/src/UTIL/CheckCollections.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index fb7461698..fff3a138c 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -102,32 +102,32 @@ namespace UTIL{ void CheckCollections::patchCollections(EVENT::LCEvent* evt ) const { - for(const auto& c : _patchCols ){ + for(const auto& [name, typeName] : _patchCols ){ try{ - auto* coll = evt->getCollection( c.first ) ; + auto* coll = evt->getCollection( name ) ; // For LCRelations we still have to check whether the FromType and // ToType are set and correct in case they are not if (coll->getTypeName() == "LCRelation") { auto& params = coll->parameters(); if (params.getStringVal("FromType").empty() || params.getStringVal("ToType").empty()) { - const auto [from, to] = getToFromType(c.second); + const auto [from, to] = getToFromType(typeName); params.setValue("FromType", std::string(from)); params.setValue("ToType", std::string(to)); } } } catch( EVENT::DataNotAvailableException& e) { //10 is the length of the String LCRelation after which the bracket is and the "ToType" and "FromType" start. - if (c.second.size() > 10 && c.second[10] == '[') { + if (typeName.size() > 10 && typeName[10] == '[') { auto relationColl = new IMPL::LCCollectionVec("LCRelation"); auto& params = relationColl->parameters(); - const auto [from, to] = getToFromType(c.second); + const auto [from, to] = getToFromType(typeName); params.setValue("FromType", std::string(from)); params.setValue("ToType", std::string(to)); - evt->addCollection( relationColl, c.first ) ; + evt->addCollection( relationColl, name ) ; } else { - evt->addCollection( new IMPL::LCCollectionVec(c.second), c.first ) ; + evt->addCollection( new IMPL::LCCollectionVec(typeName), name ) ; } } } From 4531dbcf9aac6caca70299ce31e8750835a64978 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 14:08:57 +0200 Subject: [PATCH 06/15] Format file with clang-format --- src/cpp/src/UTIL/CheckCollections.cc | 280 ++++++++++++++------------- 1 file changed, 148 insertions(+), 132 deletions(-) diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index fff3a138c..fdeb98200 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -2,176 +2,192 @@ #include "lcio.h" -#include "MT/LCReader.h" #include "IMPL/LCCollectionVec.h" +#include "MT/LCReader.h" #include -namespace UTIL{ +namespace UTIL { - void CheckCollections::checkFiles( const std::vector& fileNames, bool quiet){ +void CheckCollections::checkFiles(const std::vector &fileNames, + bool quiet) { - for( auto n : fileNames ) - checkFile( n ,quiet) ; - } + for (auto n : fileNames) + checkFile(n, quiet); +} - void CheckCollections::checkFile( const std::string& fileName, bool quiet){ - - MT::LCReader lcReader(MT::LCReader::directAccess) ; - lcReader.open( fileName ) ; - //----------- the event loop ----------- - while( const auto evt = lcReader.readNextEventHeader() ) { - - const auto* colNames = evt->getCollectionNames() ; - - for(const auto& name : *colNames){ - - auto it = _map.find( name ) ; - - if( it == _map.end() ){ - - auto col = evt->getCollection( name ) ; - // If the type of a collection is LCRelation we want to read the entire - // collections instead of just the header to get the 'ToType' and - // 'FromType'. setReadCollectionNames({name}) allows reading of only - // certain collections by name instead of an entire event. This flag has to - // be unset after reading in order for the reading of the headers to - // function properly. - std::string typeString; - if (col->getTypeName() == "LCRelation"){ - lcReader.setReadCollectionNames({name}); - auto fullEvt = lcReader.readEvent(evt->getRunNumber(), evt->getEventNumber()); - lcReader.setReadCollectionNames({}); - - auto fullcol = fullEvt->getCollection( name ) ; - const auto& params = fullcol->getParameters(); - const auto& fromType = params.getStringVal("FromType"); - const auto& toType = params.getStringVal("ToType"); - if (quiet == false){ - if (fromType == ""|| toType == ""){ - std::cout<< "WARNING! : Relation " << name <<" does not have the 'FromType' and 'ToType' set."<getCollectionNames(); + + for (const auto &name : *colNames) { + + auto it = _map.find(name); + + if (it == _map.end()) { + + auto col = evt->getCollection(name); + // If the type of a collection is LCRelation we want to read the entire + // collections instead of just the header to get the 'ToType' and + // 'FromType'. setReadCollectionNames({name}) allows reading of only + // certain collections by name instead of an entire event. This flag has + // to be unset after reading in order for the reading of the headers to + // function properly. + std::string typeString; + if (col->getTypeName() == "LCRelation") { + lcReader.setReadCollectionNames({name}); + auto fullEvt = + lcReader.readEvent(evt->getRunNumber(), evt->getEventNumber()); + lcReader.setReadCollectionNames({}); + + auto fullcol = fullEvt->getCollection(name); + const auto ¶ms = fullcol->getParameters(); + const auto &fromType = params.getStringVal("FromType"); + const auto &toType = params.getStringVal("ToType"); + if (quiet == false) { + if (fromType == "" || toType == "") { + std::cout << "WARNING! : Relation " << name + << " does not have the 'FromType' and 'ToType' set." + << std::endl; + } + } + typeString = "LCRelation[" + fromType + "," + toType + "]"; + } else { + typeString = col->getTypeName(); } - } - typeString = "LCRelation["+fromType+","+toType+"]"; - } - else { - typeString = col->getTypeName(); - } - const auto[ itx, _ ] = _map.emplace( name, std::make_pair( std::move(typeString) , 0 ) ) ; + const auto [itx, _] = + _map.emplace(name, std::make_pair(std::move(typeString), 0)); - it = itx ; - } - - it->second.second ++ ; + it = itx; } - _nEvents ++ ; + it->second.second++; } - lcReader.close() ; + _nEvents++; } + lcReader.close(); +} - CheckCollections::Vector CheckCollections::getMissingCollections() const { - Vector s ; - for(const auto& e : _map ){ - if( e.second.second != _nEvents ) - s.push_back( {e.first, e.second.first } ) ; - } - return s ; +CheckCollections::Vector CheckCollections::getMissingCollections() const { + Vector s; + for (const auto &e : _map) { + if (e.second.second != _nEvents) + s.push_back({e.first, e.second.first}); } - + return s; +} - CheckCollections::Vector CheckCollections::getConsistentCollections() const { - Vector s ; - for(auto e : _map ){ - if( e.second.second == _nEvents ) - s.push_back( {e.first, e.second.first }) ; - } - return s ; +CheckCollections::Vector CheckCollections::getConsistentCollections() const { + Vector s; + for (auto e : _map) { + if (e.second.second == _nEvents) + s.push_back({e.first, e.second.first}); } + return s; +} - // Obtain the from and to type from the encoded "LCRelation[From,To]" - std::tuple getToFromType(const std::string_view fullType) { - auto delim = fullType.find(','); - constexpr auto prefixLen = 11u; // length of "LCRelation[" - - return {fullType.substr(prefixLen, delim - prefixLen), - fullType.substr(delim + 1, fullType.size() - delim - 2)}; // need to strip final "]" as well - } +// Obtain the from and to type from the encoded "LCRelation[From,To]" +std::tuple +getToFromType(const std::string_view fullType) { + auto delim = fullType.find(','); + constexpr auto prefixLen = 11u; // length of "LCRelation[" - void CheckCollections::patchCollections(EVENT::LCEvent* evt ) const { + return {fullType.substr(prefixLen, delim - prefixLen), + fullType.substr(delim + 1, fullType.size() - delim - + 2)}; // need to strip final "]" as well +} - for(const auto& [name, typeName] : _patchCols ){ +void CheckCollections::patchCollections(EVENT::LCEvent *evt) const { - try{ - auto* coll = evt->getCollection( name ) ; - // For LCRelations we still have to check whether the FromType and - // ToType are set and correct in case they are not - if (coll->getTypeName() == "LCRelation") { - auto& params = coll->parameters(); - if (params.getStringVal("FromType").empty() || params.getStringVal("ToType").empty()) { - const auto [from, to] = getToFromType(typeName); - params.setValue("FromType", std::string(from)); - params.setValue("ToType", std::string(to)); - } - } - } catch( EVENT::DataNotAvailableException& e) { - //10 is the length of the String LCRelation after which the bracket is and the "ToType" and "FromType" start. - if (typeName.size() > 10 && typeName[10] == '[') { - auto relationColl = new IMPL::LCCollectionVec("LCRelation"); - auto& params = relationColl->parameters(); + for (const auto &[name, typeName] : _patchCols) { + try { + auto *coll = evt->getCollection(name); + // For LCRelations we still have to check whether the FromType and + // ToType are set and correct in case they are not + if (coll->getTypeName() == "LCRelation") { + auto ¶ms = coll->parameters(); + if (params.getStringVal("FromType").empty() || + params.getStringVal("ToType").empty()) { const auto [from, to] = getToFromType(typeName); params.setValue("FromType", std::string(from)); params.setValue("ToType", std::string(to)); - evt->addCollection( relationColl, name ) ; - } else { - evt->addCollection( new IMPL::LCCollectionVec(typeName), name ) ; } } + } catch (EVENT::DataNotAvailableException &e) { + // 10 is the length of the String LCRelation after which the bracket is + // and the "ToType" and "FromType" start. + if (typeName.size() > 10 && typeName[10] == '[') { + auto relationColl = new IMPL::LCCollectionVec("LCRelation"); + auto ¶ms = relationColl->parameters(); + + const auto [from, to] = getToFromType(typeName); + params.setValue("FromType", std::string(from)); + params.setValue("ToType", std::string(to)); + evt->addCollection(relationColl, name); + } else { + evt->addCollection(new IMPL::LCCollectionVec(typeName), name); + } } } +} - - void CheckCollections::print( std::ostream& os ,bool minimal) const { - - unsigned width = 50 ; - if (minimal == false){ - os << " ================================================================ " << std::endl ; - os << std::endl << " " << _nEvents << " events read " << std::endl ; - os << " collections that are not in all events : [# events where col is present]" << std::endl ; - os << " ================================================================ " << std::endl ; - } - if (minimal == false){ - for(auto e : _map ){ +void CheckCollections::print(std::ostream &os, bool minimal) const { + + unsigned width = 50; + if (minimal == false) { + os << " ================================================================ " + << std::endl; + os << std::endl << " " << _nEvents << " events read " << std::endl; + os << " collections that are not in all events : [# events where col " + "is present]" + << std::endl; + os << " ================================================================ " + << std::endl; + } + if (minimal == false) { + for (auto e : _map) { - if( e.second.second != _nEvents ) - os << " " << std::setw(width) << std::left << e.first << " " < Date: Fri, 21 Jun 2024 14:17:46 +0200 Subject: [PATCH 07/15] Use std::tie to get rid of unnecessary temporary --- src/cpp/src/UTIL/CheckCollections.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index fdeb98200..3f43a83b1 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -60,10 +60,8 @@ void CheckCollections::checkFile(const std::string &fileName, bool quiet) { } else { typeString = col->getTypeName(); } - const auto [itx, _] = + std::tie(it, std::ignore) = _map.emplace(name, std::make_pair(std::move(typeString), 0)); - - it = itx; } it->second.second++; From fc8d288756802227bf5e18c4d7a9365aacd9f2e6 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 14:19:26 +0200 Subject: [PATCH 08/15] Get rid of unnecessary branch --- src/cpp/src/UTIL/CheckCollections.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index 3f43a83b1..5c27084bf 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -32,13 +32,14 @@ void CheckCollections::checkFile(const std::string &fileName, bool quiet) { if (it == _map.end()) { auto col = evt->getCollection(name); + auto typeString = col->getTypeName(); + // If the type of a collection is LCRelation we want to read the entire // collections instead of just the header to get the 'ToType' and // 'FromType'. setReadCollectionNames({name}) allows reading of only // certain collections by name instead of an entire event. This flag has // to be unset after reading in order for the reading of the headers to // function properly. - std::string typeString; if (col->getTypeName() == "LCRelation") { lcReader.setReadCollectionNames({name}); auto fullEvt = @@ -57,8 +58,6 @@ void CheckCollections::checkFile(const std::string &fileName, bool quiet) { } } typeString = "LCRelation[" + fromType + "," + toType + "]"; - } else { - typeString = col->getTypeName(); } std::tie(it, std::ignore) = _map.emplace(name, std::make_pair(std::move(typeString), 0)); From 5de147ad591cf5db6181e2c647a3ae43e982a5b2 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 15:20:00 +0200 Subject: [PATCH 09/15] Keep track of ParticleID metadata while scanning --- src/cpp/include/UTIL/CheckCollections.h | 20 +++++- src/cpp/src/UTIL/CheckCollections.cc | 83 +++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 7 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index f4022a4fc..1a0500c9b 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -3,6 +3,8 @@ #include "lcio.h" +#include "UTIL/PIDHandler.h" + #include #include #include @@ -72,9 +74,25 @@ namespace UTIL { void patchCollections(EVENT::LCEvent* evt ) const ; private: + + /// Metadata for ParticleIDs that are handled via the PIDHandler. Necessary + /// for consistency with EDM4hep, where ParticleID no longer lives in + /// ReconstructedParticle and where the direction of the relation has been + /// reversed. + struct PIDMeta { + std::string name{}; ///< algorithm name + std::vector paramNames{}; ///< parameter names + uint32_t count{}; ///< How often this was found + }; + + void insertParticleIDMetas(const UTIL::PIDHandler& pidHandler, const std::string& recoName); + unsigned _nEvents =0 ; std::unordered_map< std::string, std::pair< std::string, unsigned > > _map{} ; - Vector _patchCols {} ; + /// Map from ReconstructedParticle collection names to attached ParticleID + /// meta information + std::unordered_map> _particleIDMetas{}; + Vector _patchCols{}; }; // class diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index 5c27084bf..8dbdf8f20 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -22,18 +22,22 @@ void CheckCollections::checkFile(const std::string &fileName, bool quiet) { lcReader.open(fileName); //----------- the event loop ----------- while (const auto evt = lcReader.readNextEventHeader()) { - const auto *colNames = evt->getCollectionNames(); + std::vector recoCollections{}; for (const auto &name : *colNames) { + const auto col = evt->getCollection(name); + auto typeString = col->getTypeName(); + + // For ReconstructedParticle we also have to check the ParticleIDs for + // consistency. We have to do this regardless of whether the + // ReconstructedParticle collection is already in the map or not + if (typeString == "ReconstructedParticle") { + recoCollections.emplace_back(name); + } auto it = _map.find(name); - if (it == _map.end()) { - - auto col = evt->getCollection(name); - auto typeString = col->getTypeName(); - // If the type of a collection is LCRelation we want to read the entire // collections instead of just the header to get the 'ToType' and // 'FromType'. setReadCollectionNames({name}) allows reading of only @@ -66,12 +70,46 @@ void CheckCollections::checkFile(const std::string &fileName, bool quiet) { it->second.second++; } + lcReader.setReadCollectionNames(recoCollections); + auto fullEvt = + lcReader.readEvent(evt->getRunNumber(), evt->getEventNumber()); + lcReader.setReadCollectionNames({}); + + for (const auto &name : recoCollections) { + auto handler = PIDHandler(fullEvt->getCollection(name)); + insertParticleIDMetas(handler, name); + } + _nEvents++; } lcReader.close(); } +void CheckCollections::insertParticleIDMetas(const UTIL::PIDHandler &pidHandler, + const std::string &recoName) { + const auto &algoIds = pidHandler.getAlgorithmIDs(); + auto mapIt = _particleIDMetas.find(recoName); + if (mapIt == _particleIDMetas.end()) { + std::tie(mapIt, std::ignore) = + _particleIDMetas.emplace(recoName, std::vector{}); + } + + auto &pidMetas = mapIt->second; + for (const auto id : algoIds) { + const auto &name = pidHandler.getAlgorithmName(id); + + if (auto it = std::find_if( + pidMetas.begin(), pidMetas.end(), + [&name](const auto &pidMeta) { return pidMeta.name == name; }); + it == pidMetas.end()) { + pidMetas.emplace_back(name, pidHandler.getParameterNames(id), 1); + } else { + it->count++; + } + } +} + CheckCollections::Vector CheckCollections::getMissingCollections() const { Vector s; for (const auto &e : _map) { @@ -185,6 +223,39 @@ void CheckCollections::print(std::ostream &os, bool minimal) const { os << " ================================================================ " << std::endl; } + + if (!minimal) { + os << " ================================================================\n" + << "Known ParticleID algorithms for ReconstructedParticle collections\n"; + for (const auto &[name, pidMetas] : _particleIDMetas) { + if (pidMetas.empty()) { + continue; + } + os << name << ":\n"; + for (const auto &meta : pidMetas) { + os << " " << std::setw(width) << std::left << meta.name << " [" + << meta.count << "]\n"; + } + os << " -----------------------------------------------------------------" + " \n"; + } + os << " ================================================================ " + << std::endl; + } else { + for (const auto &[name, pidMetas] : _particleIDMetas) { + for (const auto &meta : pidMetas) { + os << meta.name << std::setw(width) << " " << name << "|"; + if (!meta.paramNames.empty()) { + os << meta.paramNames[0]; + for (size_t i = 1; i < meta.paramNames.size(); ++i) { + os << "," << meta.paramNames[i]; + } + } + os << '\n'; + } + } + } + os << std::flush; } } // namespace UTIL From ae22278ab359060bf8df6ac570dd40cea70e51bb Mon Sep 17 00:00:00 2001 From: tmadlener Date: Mon, 24 Jun 2024 14:08:17 +0200 Subject: [PATCH 10/15] Make PIDMeta public to satisfy ROOT --- src/cpp/include/UTIL/CheckCollections.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index 1a0500c9b..45b5cc791 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -72,8 +72,6 @@ namespace UTIL { /** Add and empty collection to the event for any collection that is in patchCollections and not in the Event */ void patchCollections(EVENT::LCEvent* evt ) const ; - - private: /// Metadata for ParticleIDs that are handled via the PIDHandler. Necessary /// for consistency with EDM4hep, where ParticleID no longer lives in @@ -85,6 +83,8 @@ namespace UTIL { uint32_t count{}; ///< How often this was found }; + private: + void insertParticleIDMetas(const UTIL::PIDHandler& pidHandler, const std::string& recoName); unsigned _nEvents =0 ; From 17b2570794fdb666351c94217f52bf28ab051ece Mon Sep 17 00:00:00 2001 From: tmadlener Date: Fri, 21 Jun 2024 17:24:00 +0200 Subject: [PATCH 11/15] Patch ParticleID meta data on request --- src/cpp/include/UTIL/CheckCollections.h | 14 +++--- src/cpp/src/UTIL/CheckCollections.cc | 58 +++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index 45b5cc791..db822818b 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -58,16 +58,11 @@ namespace UTIL { /** Add a collection with (name,type) that should be added to events in patchEvent(). */ - void addPatchCollection(std::string name, std::string type){ - _patchCols.emplace_back( std::move(name), std::move(type) ) ; - } + void addPatchCollection(std::string name, std::string type); /** Add all collections as Vector(name,type), e.g. retrieved from getMissingCollections() that should be added to events in patchEvent(). */ - void addPatchCollections(Vector cols){ - for(auto&& p : cols) - _patchCols.emplace_back( std::move(p) ) ; - } + void addPatchCollections(Vector cols); /** Add and empty collection to the event for any collection that is in patchCollections and not in the Event */ @@ -87,6 +82,11 @@ namespace UTIL { void insertParticleIDMetas(const UTIL::PIDHandler& pidHandler, const std::string& recoName); + /// Sort the patch collections, such that all ParticleID patches are done at + /// the end **after** the ReconstructedParticle collections have been + /// patched in + void sortPatchCollections(); + unsigned _nEvents =0 ; std::unordered_map< std::string, std::pair< std::string, unsigned > > _map{} ; /// Map from ReconstructedParticle collection names to attached ParticleID diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index 8dbdf8f20..e1a2d2aea 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -5,6 +5,7 @@ #include "IMPL/LCCollectionVec.h" #include "MT/LCReader.h" +#include #include namespace UTIL { @@ -128,6 +129,27 @@ CheckCollections::Vector CheckCollections::getConsistentCollections() const { return s; } +void CheckCollections::addPatchCollection(std::string name, std::string type) { + _patchCols.emplace_back(std::move(name), std::move(type)); + sortPatchCollections(); +} + +void CheckCollections::addPatchCollections(Vector cols) { + for (auto &&p : cols) { + _patchCols.emplace_back(std::move(p)); + } + sortPatchCollections(); +} + +void CheckCollections::sortPatchCollections() { + std::sort(_patchCols.begin(), _patchCols.end(), + [](const auto &lhs, const auto &rhs) { + const int pidL = lhs.second.find('|') != std::string::npos; + const int pidR = rhs.second.find('|') != std::string::npos; + return pidL < pidR; + }); +} + // Obtain the from and to type from the encoded "LCRelation[From,To]" std::tuple getToFromType(const std::string_view fullType) { @@ -139,15 +161,33 @@ getToFromType(const std::string_view fullType) { 2)}; // need to strip final "]" as well } +// Obtain the name of the recontructed particle collection as well as the +// parameter names from an encoded "RecoColl|[name1[,names...]]" +std::tuple> +getRecoCollAndParamNames(const std::string_view fullType) { + auto delim = fullType.find('|'); + auto recoName = std::string(fullType.substr(0, delim)); + + std::vector paramNames{}; + while (delim != std::string_view::npos) { + auto oldDelim = delim + 1; + delim = fullType.find(',', oldDelim); + paramNames.emplace_back(fullType.substr(oldDelim, delim)); + } + + return {recoName, paramNames}; +} + void CheckCollections::patchCollections(EVENT::LCEvent *evt) const { for (const auto &[name, typeName] : _patchCols) { try { auto *coll = evt->getCollection(name); - // For LCRelations we still have to check whether the FromType and - // ToType are set and correct in case they are not - if (coll->getTypeName() == "LCRelation") { + const auto collType = coll->getTypeName(); + if (collType == "LCRelation") { + // For LCRelations we still have to check whether the FromType and + // ToType are set and correct in case they are not auto ¶ms = coll->parameters(); if (params.getStringVal("FromType").empty() || params.getStringVal("ToType").empty()) { @@ -156,6 +196,18 @@ void CheckCollections::patchCollections(EVENT::LCEvent *evt) const { params.setValue("ToType", std::string(to)); } } + if (collType == "Particle") { + // For ParticleID we need to make sure that this is attached to the + // required ReconstructedParticle collection + const auto &[recoName, paramNames] = getRecoCollAndParamNames(typeName); + auto pidHandler = UTIL::PIDHandler(evt->getCollection(recoName)); + // No real way of doing this without trying and catching the exception + try { + pidHandler.getAlgorithmID(name); + } catch (UnknownAlgorithm &) { + pidHandler.addAlgorithm(name, paramNames); + } + } } catch (EVENT::DataNotAvailableException &e) { // 10 is the length of the String LCRelation after which the bracket is // and the "ToType" and "FromType" start. From 0348ec62cda12488578bf201039d4343c9a8ff4c Mon Sep 17 00:00:00 2001 From: tmadlener Date: Mon, 24 Jun 2024 21:45:58 +0200 Subject: [PATCH 12/15] Set the PID meta data after all collections have been converted --- src/cpp/include/UTIL/CheckCollections.h | 5 -- src/cpp/src/UTIL/CheckCollections.cc | 91 +++++++++++++------------ 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index db822818b..3a467e996 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -82,11 +82,6 @@ namespace UTIL { void insertParticleIDMetas(const UTIL::PIDHandler& pidHandler, const std::string& recoName); - /// Sort the patch collections, such that all ParticleID patches are done at - /// the end **after** the ReconstructedParticle collections have been - /// patched in - void sortPatchCollections(); - unsigned _nEvents =0 ; std::unordered_map< std::string, std::pair< std::string, unsigned > > _map{} ; /// Map from ReconstructedParticle collection names to attached ParticleID diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index e1a2d2aea..18e8eeeee 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -1,5 +1,6 @@ #include "UTIL/CheckCollections.h" +#include "Exceptions.h" #include "lcio.h" #include "IMPL/LCCollectionVec.h" @@ -129,25 +130,41 @@ CheckCollections::Vector CheckCollections::getConsistentCollections() const { return s; } -void CheckCollections::addPatchCollection(std::string name, std::string type) { - _patchCols.emplace_back(std::move(name), std::move(type)); - sortPatchCollections(); +// Obtain the name of the recontructed particle collection as well as the +// parameter names from an encoded "RecoColl|[name1[,names...]]" +std::tuple> +getRecoCollAndParamNames(const std::string_view fullType) { + auto delim = fullType.find('|'); + auto recoName = std::string(fullType.substr(0, delim)); + + std::vector paramNames{}; + while (delim != std::string_view::npos) { + auto oldDelim = delim + 1; + delim = fullType.find(',', oldDelim); + paramNames.emplace_back(fullType.substr(oldDelim, delim)); + } + + return {recoName, paramNames}; } -void CheckCollections::addPatchCollections(Vector cols) { - for (auto &&p : cols) { - _patchCols.emplace_back(std::move(p)); +void CheckCollections::addPatchCollection(std::string name, std::string type) { + if (type.find('|') != std::string::npos) { + auto [recoName, paramNames] = getRecoCollAndParamNames(name); + _particleIDMetas[recoName].emplace_back(name, std::move(paramNames)); + } else { + _patchCols.emplace_back(std::move(name), std::move(type)); } - sortPatchCollections(); } -void CheckCollections::sortPatchCollections() { - std::sort(_patchCols.begin(), _patchCols.end(), - [](const auto &lhs, const auto &rhs) { - const int pidL = lhs.second.find('|') != std::string::npos; - const int pidR = rhs.second.find('|') != std::string::npos; - return pidL < pidR; - }); +void CheckCollections::addPatchCollections(Vector cols) { + for (auto &&[name, type] : cols) { + if (type.find('|') != std::string::npos) { + auto [recoName, paramNames] = getRecoCollAndParamNames(type); + _particleIDMetas[recoName].emplace_back(name, std::move(paramNames)); + } else { + _patchCols.emplace_back(std::move(name), std::move(type)); + } + } } // Obtain the from and to type from the encoded "LCRelation[From,To]" @@ -161,27 +178,22 @@ getToFromType(const std::string_view fullType) { 2)}; // need to strip final "]" as well } -// Obtain the name of the recontructed particle collection as well as the -// parameter names from an encoded "RecoColl|[name1[,names...]]" -std::tuple> -getRecoCollAndParamNames(const std::string_view fullType) { - auto delim = fullType.find('|'); - auto recoName = std::string(fullType.substr(0, delim)); - - std::vector paramNames{}; - while (delim != std::string_view::npos) { - auto oldDelim = delim + 1; - delim = fullType.find(',', oldDelim); - paramNames.emplace_back(fullType.substr(oldDelim, delim)); +// Add all algorithms that are specified in the pidMetas to the PIDHandler, such +// that the necessary metadata is present +void patchParticleIDs(UTIL::PIDHandler &pidHandler, + const std::vector &pidMetas) { + for (const auto &[name, paramNames, _] : pidMetas) { + try { + // simply assume that param names are OK if we find the algorithm + pidHandler.getAlgorithmID(name); + } catch (UnknownAlgorithm &) { + pidHandler.addAlgorithm(name, paramNames); + } } - - return {recoName, paramNames}; } void CheckCollections::patchCollections(EVENT::LCEvent *evt) const { - for (const auto &[name, typeName] : _patchCols) { - try { auto *coll = evt->getCollection(name); const auto collType = coll->getTypeName(); @@ -196,18 +208,6 @@ void CheckCollections::patchCollections(EVENT::LCEvent *evt) const { params.setValue("ToType", std::string(to)); } } - if (collType == "Particle") { - // For ParticleID we need to make sure that this is attached to the - // required ReconstructedParticle collection - const auto &[recoName, paramNames] = getRecoCollAndParamNames(typeName); - auto pidHandler = UTIL::PIDHandler(evt->getCollection(recoName)); - // No real way of doing this without trying and catching the exception - try { - pidHandler.getAlgorithmID(name); - } catch (UnknownAlgorithm &) { - pidHandler.addAlgorithm(name, paramNames); - } - } } catch (EVENT::DataNotAvailableException &e) { // 10 is the length of the String LCRelation after which the bracket is // and the "ToType" and "FromType" start. @@ -224,6 +224,13 @@ void CheckCollections::patchCollections(EVENT::LCEvent *evt) const { } } } + + for (const auto &[recoName, pidMeta] : _particleIDMetas) { + // Let the exception propagate. This is not something that we can easily + // handle in any meaningful way, so make users aware as early as possible + auto pidHandler = UTIL::PIDHandler(evt->getCollection(recoName)); + patchParticleIDs(pidHandler, pidMeta); + } } void CheckCollections::print(std::ostream &os, bool minimal) const { From cb698982a69b1e9875746ff2eedd0649e98ec084 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Tue, 25 Jun 2024 15:14:57 +0200 Subject: [PATCH 13/15] Make things work with c++17 --- src/cpp/include/UTIL/CheckCollections.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index 3a467e996..76c6bc16c 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -73,9 +73,24 @@ namespace UTIL { /// ReconstructedParticle and where the direction of the relation has been /// reversed. struct PIDMeta { - std::string name{}; ///< algorithm name + // c++17 doesn't yet have aggregate initialization in vectors, so we + // need this constructor + PIDMeta(const std::string &n, const std::vector &parN, + uint32_t c = 0) + : name(n), paramNames(parN), count(c) {} + + // Since we have one non-default constructor we need to default the rest + // explicitly + constexpr PIDMeta() = default; + PIDMeta(const PIDMeta &) = default; + PIDMeta &operator=(const PIDMeta &) = default; + PIDMeta(PIDMeta &&) = default; + PIDMeta &operator=(PIDMeta &&) = default; + ~PIDMeta() = default; + + std::string name{}; ///< algorithm name std::vector paramNames{}; ///< parameter names - uint32_t count{}; ///< How often this was found + uint32_t count{}; ///< How often this was found }; private: From b00af08ca5a739dc4779ee116b39aed91856ae35 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Wed, 4 Sep 2024 14:47:31 +0200 Subject: [PATCH 14/15] Add patching grammar to docstring --- src/cpp/include/UTIL/CheckCollections.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index 76c6bc16c..ac800439f 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -57,6 +57,23 @@ namespace UTIL { Vector getConsistentCollections() const ; /** Add a collection with (name,type) that should be added to events in patchEvent(). + * + * Depending on the contents of name and type one of the following things + * will happen: + * + * - if type is an LCIO type an empty collection of the given type will be + put into the event using the passed name + * - if type is `LCRelation[,]` an LCRelation collection + * will be created setting the and as FromType and + * ToType collection parameters. + * - if type contains a '|' the whole (name, type) pair will be considered + * to be ParticleID meta information and the name will be used as a PID + * algorithm name and the type information will be parsed as + * |[,], I.e. a ParticleID + * algorithm of name will be attached to the ReconstructedParticle + * collection with name . Additionally any parameter names + * that are in the comma-separated list after the '|' will be set via the + * PIDHandler for this ParticleID algorithm */ void addPatchCollection(std::string name, std::string type); From 0414a5821046ddef536a708cb7ed2c768f63960a Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 5 Sep 2024 10:32:23 +0200 Subject: [PATCH 15/15] Move inclusion of header into implementation file --- src/cpp/include/UTIL/CheckCollections.h | 5 ++--- src/cpp/src/UTIL/CheckCollections.cc | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cpp/include/UTIL/CheckCollections.h b/src/cpp/include/UTIL/CheckCollections.h index ac800439f..b326145a8 100644 --- a/src/cpp/include/UTIL/CheckCollections.h +++ b/src/cpp/include/UTIL/CheckCollections.h @@ -3,15 +3,14 @@ #include "lcio.h" -#include "UTIL/PIDHandler.h" - #include #include #include namespace UTIL { - +class PIDHandler; + /** Utility class for checking and patching events with respect to collections that are not present * in every event of a set of files. * diff --git a/src/cpp/src/UTIL/CheckCollections.cc b/src/cpp/src/UTIL/CheckCollections.cc index 18e8eeeee..c13778581 100644 --- a/src/cpp/src/UTIL/CheckCollections.cc +++ b/src/cpp/src/UTIL/CheckCollections.cc @@ -5,6 +5,7 @@ #include "IMPL/LCCollectionVec.h" #include "MT/LCReader.h" +#include "UTIL/PIDHandler.h" #include #include