From 85f67b079a7d981195848c1dc449d46b8df52c58 Mon Sep 17 00:00:00 2001
From: Peter Berta <peter.berta@cern.ch>
Date: Tue, 14 Sep 2021 13:03:19 +0200
Subject: [PATCH] Merge branch 'propagat_TE_data' into '22.0-mc20'
[ATR-24058][ATR-24141] Working to add R2 trigger TE data to R3 JSON structure to enable feature access from R22 AODs created from R2 data
See merge request atlas/athena!46419
(cherry picked from commit 8944255495157e07876c404f2f27a012e17505b8)
6a3c59d7 working to add R2 TE data to R3 JSON structure
b9133381 Add standalone signatures data
86dc0214 do not copy seq list, causes segfault on destruct
f09bf609 Change RAWtoALL/ESD/AOD slimming of R2 trigger navigation structures. Remove...
---
.../share/TrigDecTool_jobOptions.py | 23 ++-
.../TrigConfIO/src/JsonFileWriterHLT.cxx | 46 ++++++
.../src/test/Run2toRun3ConvertersHLT.cxx | 55 +++++++
.../TrigConfxAOD/Root/prepareTriggerMenu.cxx | 151 +++++++++++++-----
.../TrigNavTools/python/TrigNavToolsConfig.py | 2 +
.../python/HLTTriggerResultGetter.py | 5 +-
6 files changed, 228 insertions(+), 54 deletions(-)
diff --git a/Control/AthenaMonitoring/share/TrigDecTool_jobOptions.py b/Control/AthenaMonitoring/share/TrigDecTool_jobOptions.py
index c6abfcbd234a..c348dd10f752 100644
--- a/Control/AthenaMonitoring/share/TrigDecTool_jobOptions.py
+++ b/Control/AthenaMonitoring/share/TrigDecTool_jobOptions.py
@@ -20,20 +20,15 @@ if DQMonFlags.useTrigger():
from TriggerJobOpts.TriggerConfigGetter import TriggerConfigGetter
cfg = TriggerConfigGetter()
- if not hasattr(ToolSvc, DQMonFlags.nameTrigDecTool().split('/')[-1]):
- if rec.doTrigger():
- tdt_local_logger.error('DQ Monitoring is being asked to set up the TrigDecisionTool for some reason. THIS IS A TERRIBLE IDEA AND SHOULD BE CONSIDERED A BUG!')
- from AthenaMonitoring.TriggerInterface import getTrigDecisionTool
- from AthenaConfiguration.OldFlags2NewFlags import getNewConfigFlags
- # Translate all needed flags from old jobProperties to a new AthConfigFlag Container
- ConfigFlags = getNewConfigFlags()
-
- from AthenaConfiguration import ComponentAccumulator
- ComponentAccumulator.CAtoGlobalWrapper(getTrigDecisionTool, ConfigFlags)
-
- monTrigDecTool = getattr(ToolSvc, 'TrigDecisionTool')
- else:
- monTrigDecTool = getattr(ToolSvc, DQMonFlags.nameTrigDecTool().split('/')[-1])
+ from AthenaCommon.Configurable import Configurable
+ Configurable.configurableRun3Behavior+=1
+ from TrigDecisionTool.TrigDecisionToolConfig import getTrigDecisionTool
+ tdtAcc = getTrigDecisionTool(ConfigFlags)
+ Configurable.configurableRun3Behavior-=1
+
+ from AthenaConfiguration.ComponentAccumulator import conf2toConfigurable, appendCAtoAthena
+ monTrigDecTool = conf2toConfigurable(tdtAcc.getPrimary())
+ appendCAtoAthena( tdtAcc )
tdt_local_logger.info('Scheduled monitoring TDT %s', monTrigDecTool)
tdt_local_logger.info('Scheduling the trigger translator')
diff --git a/Trigger/TrigConfiguration/TrigConfIO/src/JsonFileWriterHLT.cxx b/Trigger/TrigConfiguration/TrigConfIO/src/JsonFileWriterHLT.cxx
index eb1a38ac4df2..afe0b97cd363 100644
--- a/Trigger/TrigConfiguration/TrigConfIO/src/JsonFileWriterHLT.cxx
+++ b/Trigger/TrigConfiguration/TrigConfIO/src/JsonFileWriterHLT.cxx
@@ -18,9 +18,36 @@ TrigConf::JsonFileWriterHLT::JsonFileWriterHLT() :
{}
+/// Helper function ptree key->[] to std::vector<T>
+template<typename T>
+std::vector<T> ToVector(const TrigConf::DataStructure& ds, const std::string& child){
+ using ptree = boost::property_tree::ptree;
+ std::vector<T> return_vector;
+ for( const ptree::value_type& entry : ds.data().get_child(child) ) {
+ return_vector.push_back( entry.second.get_value<T>() );
+ }
+ return return_vector;
+}
+
+/// Helper function ptree key->[[]] to std::vector<std::vector<T>>
+template<typename T>
+std::vector<std::vector<T>> ToVectorVector(const TrigConf::DataStructure& ds, const std::string& child){
+ using ptree = boost::property_tree::ptree;
+ std::vector<std::vector<T>> return_vector;
+ for( const ptree::value_type& outer : ds.data().get_child(child) ) {
+ return_vector.push_back(std::vector<T>());
+ for (const ptree::value_type& inner : outer.second) {
+ return_vector.back().push_back( inner.second.get_value<T>() );
+ }
+ }
+ return return_vector;
+}
+
+
bool
TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const HLTMenu & menu) const
{
+ using ptree = boost::property_tree::ptree;
json chains({});
for ( const auto & chain : menu ) {
json jChain({});
@@ -32,6 +59,17 @@ TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const H
jChain["groups"] = chain.groups();
jChain["streams"] = chain.streams();
jChain["seqeuncers"] = chain.sequencers();
+
+ // Optional Run2 payload
+ if (chain.hasChild("signature")) {
+ json jSig({});
+ jSig["counters"] = ToVector<uint32_t>(chain, "signature.counters");
+ jSig["logics"] = ToVector<int>(chain, "signature.logics");
+ jSig["labels"] = ToVector<std::string>(chain, "signature.labels");
+ jSig["outputTEs"] = ToVectorVector<std::string>(chain, "signature.outputTEs");
+ jChain["signature"] = jSig;
+ }
+
chains[chain.name()] = jChain;
}
@@ -58,6 +96,14 @@ TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const H
j["sequencers"] = sequencers;
j["streams"] = streams;
+ // Optional Run2 payload
+ if (menu.hasChild("sequence_run2")) {
+ json jSequence({});
+ jSequence["outputTEs"] = ToVector<std::string>(menu, "sequence_run2.outputTEs");
+ jSequence["inputTEs"] = ToVectorVector<std::string>(menu, "sequence_run2.inputTEs");
+ jSequence["algorithms"] = ToVectorVector<std::string>(menu, "sequence_run2.algorithms");
+ j["sequence_run2"] = jSequence;
+ }
std::ofstream outfile(filename);
outfile << std::setw(4) << j << std::endl;
diff --git a/Trigger/TrigConfiguration/TrigConfStorage/src/test/Run2toRun3ConvertersHLT.cxx b/Trigger/TrigConfiguration/TrigConfStorage/src/test/Run2toRun3ConvertersHLT.cxx
index 4fb708f85cd7..8ff5ce4f87ce 100644
--- a/Trigger/TrigConfiguration/TrigConfStorage/src/test/Run2toRun3ConvertersHLT.cxx
+++ b/Trigger/TrigConfiguration/TrigConfStorage/src/test/Run2toRun3ConvertersHLT.cxx
@@ -81,6 +81,35 @@ bool convertHLTMenu(const TrigConf::HLTFrame* frame, TrigConf::HLTMenu& menu) {
pChain.add_child("groups", asArray(cptr->groups()));
+ // Signature data
+ // Note: This is run-2 only.
+ // It is propagated here to allow legacy trigger feature access.
+ std::vector<uint32_t> counters;
+ std::vector<int> logics;
+ std::vector<std::string> labels;
+ ptree outputTEs_outerArray; // outputTEs is a std::vector<std::vector<std::string>>
+
+ for(auto& signature : cptr->signatureList() ){
+ uint32_t cntr = signature->signature_counter();
+ counters.push_back(cntr);
+ logics.push_back(signature->logic());
+ labels.push_back(signature->label());
+ ptree outputTEs_innerArray;
+ for(auto& outputTE : signature->outputTEs()){
+ outputTEs_innerArray.push_back( ptree::value_type("", outputTE->name()) );
+ }
+ outputTEs_outerArray.push_back( ptree::value_type("", outputTEs_innerArray) );
+ }
+
+ ptree pSig;
+ pSig.add_child("counters", asArray(counters));
+ pSig.add_child("logics", asArray(logics));
+ pSig.add_child("outputTEs", outputTEs_outerArray);
+ pSig.add_child("labels", asArray(labels));
+
+ pChain.add_child("signature", pSig);
+ // End of signature data
+
pChains.push_back(std::make_pair(cptr->chain_name(), pChain));
}
ptree pStreams;
@@ -100,6 +129,32 @@ bool convertHLTMenu(const TrigConf::HLTFrame* frame, TrigConf::HLTMenu& menu) {
pSequencers.add_child("missing", asArray(std::vector<std::string>({""})));
top.add_child("sequencers", pSequencers);
+ // Set run2 sequence information:
+ const TrigConf::HLTSequenceList& sequenceList = frame->getHLTSequenceList();
+ std::vector<std::string> outputTEs;
+ ptree inputTEs_outerArray; // sequenceInputTEs is a std::vector<std::vector<std::string>>
+ ptree algorithms_outerArray; // sequenceAlgorithms is a std::vector<std::vector<std::string>>
+ for(auto& seq : sequenceList){
+ outputTEs.push_back(seq->outputTE()->name());
+
+ ptree inputTEs_innerArray;
+ for(auto& input : seq->inputTEs()) {
+ inputTEs_innerArray.push_back( ptree::value_type("", input->name()) );
+ }
+ inputTEs_outerArray.push_back( ptree::value_type("", inputTEs_innerArray) );
+
+ ptree algorithms_innerArray;
+ for(const std::string& alg : seq->algorithms()) {
+ algorithms_innerArray.push_back( ptree::value_type("", alg) );
+ }
+ algorithms_outerArray.push_back( ptree::value_type("", algorithms_innerArray) );
+ }
+ ptree pSequence;
+ pSequence.add_child("outputTEs", asArray(outputTEs));
+ pSequence.add_child("inputTEs", inputTEs_outerArray);
+ pSequence.add_child("algorithms", algorithms_outerArray);
+ top.add_child("sequence_run2", pSequence);
+
menu.setData(std::move(top));
menu.setSMK(frame->smk());
return true;
diff --git a/Trigger/TrigConfiguration/TrigConfxAOD/Root/prepareTriggerMenu.cxx b/Trigger/TrigConfiguration/TrigConfxAOD/Root/prepareTriggerMenu.cxx
index 2653c3bc03ed..59346e632f5f 100644
--- a/Trigger/TrigConfiguration/TrigConfxAOD/Root/prepareTriggerMenu.cxx
+++ b/Trigger/TrigConfiguration/TrigConfxAOD/Root/prepareTriggerMenu.cxx
@@ -253,6 +253,32 @@ namespace TrigConf {
return StatusCode::SUCCESS;
}
+
+ /// Helper function ptree key->[] to std::vector<T>
+ template<typename T>
+ std::vector<T> ToVector(const TrigConf::DataStructure& ds, const std::string& child){
+ using ptree = boost::property_tree::ptree;
+ std::vector<T> return_vector;
+ for( const ptree::value_type& entry : ds.data().get_child(child) ) {
+ return_vector.push_back( entry.second.get_value<T>() );
+ }
+ return return_vector;
+ }
+
+ /// Helper function ptree key->[[]] to std::vector<std::vector<T>>
+ template<typename T>
+ std::vector<std::vector<T>> ToVectorVector(const TrigConf::DataStructure& ds, const std::string& child){
+ using ptree = boost::property_tree::ptree;
+ std::vector<std::vector<T>> return_vector;
+ for( const ptree::value_type& outer : ds.data().get_child(child) ) {
+ return_vector.push_back(std::vector<T>());
+ for (const ptree::value_type& inner : outer.second) {
+ return_vector.back().push_back( inner.second.get_value<T>() );
+ }
+ }
+ return return_vector;
+ }
+
/// Load JSON derived data into legacy structures to maintain
/// compatiblity with existing code.
///
@@ -315,46 +341,75 @@ namespace TrigConf {
// Fill the HLT configuration:
if (loadedHlt.isInitialized()) {
for (const Chain& loadedChain : loadedHlt) {
- // Figure out which level this chain is from:
- std::string level = "";
- if( loadedChain.name().find( "L2_" ) == 0 ) {
- level = "L2";
- } else if( loadedChain.name().find( "EF_" ) == 0 ) {
- level = "EF";
- } else if( loadedChain.name().find( "HLT_" ) == 0 ) {
- level = "HLT";
- } else {
- msg << MSG::WARNING << "prepareTriggerMenu(...): "
- << "Couldn't figure out 'level' for chain: "
- << loadedChain.name() << endmsg;
- }
- // An empty signature list for the chain:
- // This is not populated from JSON data
- std::vector< HLTSignature* > signatures;
+ // Figure out which level this chain is from:
+ std::string level = "";
+ if( loadedChain.name().find( "L2_" ) == 0 ) {
+ level = "L2";
+ } else if( loadedChain.name().find( "EF_" ) == 0 ) {
+ level = "EF";
+ } else if( loadedChain.name().find( "HLT_" ) == 0 ) {
+ level = "HLT";
+ } else {
+ msg << MSG::WARNING << "prepareTriggerMenu(...): "
+ << "Couldn't figure out 'level' for chain: "
+ << loadedChain.name() << endmsg;
+ }
+
+ // An empty signature list for the chain:
+ std::vector< HLTSignature* > signatures;
+
+ // Optional Run2 payload
+ std::vector<uint32_t> counters;
+ std::vector<int> logics;
+ std::vector<std::vector<std::string>> outputTEs;
+ if (loadedChain.hasChild("signature")) {
+ counters = ToVector<uint32_t>(loadedChain, "signature.counters");
+ logics = ToVector<int>(loadedChain, "signature.logics");
+ outputTEs = ToVectorVector<std::string>(loadedChain, "signature.outputTEs");
+ }
+
+ if( msg.level() <= MSG::VERBOSE ) {
+ msg << MSG::VERBOSE << "chain " << loadedChain.name()
+ << " has counter " << loadedChain.counter()
+ << " and " << counters.size() << " signatures (runs 1,2 only)" << endmsg;
+ }
+
+ for( size_t sig = 0; sig < counters.size(); ++sig ) {
+ std::vector< HLTTriggerElement* > outTEs;
+ for( size_t outTEcounter = 0; outTEcounter< outputTEs[ sig ].size(); ++outTEcounter ) {
+ HLTTriggerElement* element = new HLTTriggerElement( outputTEs[ sig ][ outTEcounter ] );
+ outTEs.push_back( element );
+ }
+ HLTSignature* signature = new HLTSignature( counters[ sig ], logics[ sig ], outTEs );
+ signatures.push_back( signature );
+ if( msg.level() <= MSG::VERBOSE ) {
+ msg << MSG::VERBOSE << "prepared signature: " << *( signatures.back() ) << endmsg;
+ }
+ }
- // Create the chain object:
- HLTChain* chain = new HLTChain( loadedChain.name(),
- loadedChain.counter(),
- 1, // Chain version not important
- level,
- loadedChain.l1item(), // L1 seeds (string)
- -1, // Lower chain ID not important
- signatures ); // Note: Empty
+ // Create the chain object:
+ HLTChain* chain = new HLTChain( loadedChain.name(),
+ loadedChain.counter(),
+ 1, // Chain version not important
+ level,
+ loadedChain.l1item(), // L1 seeds (string)
+ -1, // Lower chain ID not important
+ signatures ); // Empty for R3 JSONs
- chain->set_rerun_prescale( -1.0 ); // Not used in R3
- chain->set_pass_through( -1.0 ); // Not used in R3
- chain->set_leg_multiplicities( loadedChain.legMultiplicities() );
-
- for (const std::string& group : loadedChain.groups()){
- chain->addGroup(group);
- }
-
- if (loadedHltps.isInitialized()) {
- const HLTPrescalesSet::HLTPrescale& loadedPrescale = loadedHltps.prescale( loadedChain.name() );
- chain->set_prescale( loadedPrescale.prescale );
- } else {
- chain->set_prescale( 0 );
- }
+ chain->set_rerun_prescale( -1.0 ); // Not used in R3
+ chain->set_pass_through( -1.0 ); // Not used in R3
+ chain->set_leg_multiplicities( loadedChain.legMultiplicities() );
+
+ for (const std::string& group : loadedChain.groups()){
+ chain->addGroup(group);
+ }
+
+ if (loadedHltps.isInitialized()) {
+ const HLTPrescalesSet::HLTPrescale& loadedPrescale = loadedHltps.prescale( loadedChain.name() );
+ chain->set_prescale( loadedPrescale.prescale );
+ } else {
+ chain->set_prescale( 0 );
+ }
// Add it to the list of chains:
if( ! chainList.addHLTChain( chain ) ) {
@@ -367,7 +422,25 @@ namespace TrigConf {
}
}
- // Do not add sequence info to legacy structures (format is different)
+ // Add sequence information if it's available (R1 or R2 menu):
+ if( loadedHlt.hasChild("sequence_run2") ) {
+ std::vector<std::string> menu_outputTEs = ToVector<std::string>(loadedHlt, "sequence_run2.outputTEs");
+ std::vector<std::vector<std::string>> menu_inputTEs = ToVectorVector<std::string>(loadedHlt, "sequence_run2.inputTEs");
+ std::vector<std::vector<std::string>> menu_algorithms = ToVectorVector<std::string>(loadedHlt, "sequence_run2.algorithms");
+
+ for( size_t i = 0; i< menu_outputTEs.size(); ++i ) {
+ HLTTriggerElement* outputTE = new HLTTriggerElement( menu_outputTEs[ i ] );
+ std::vector< HLTTriggerElement* > inputTEs;
+ for( size_t j = 0; j < menu_inputTEs[ i ].size(); ++j ) {
+ HLTTriggerElement* te = new HLTTriggerElement( menu_inputTEs[ i ][ j ] );
+ inputTEs.push_back( te );
+ }
+ HLTSequence* sequence = new HLTSequence( inputTEs, outputTE, menu_algorithms[ i ] );
+ sequenceList.addHLTSequence( sequence );
+ // This throws a runtime_error if it fails, which we don't need to
+ // handle, since this is a FATAL error anyways.
+ }
+ }
// Bunchgroup data is TODO
// Create a new BunchGroupSet object, since an existing one can't be
diff --git a/Trigger/TrigEvent/TrigNavTools/python/TrigNavToolsConfig.py b/Trigger/TrigEvent/TrigNavTools/python/TrigNavToolsConfig.py
index 38a25cd47382..f889159b005c 100644
--- a/Trigger/TrigEvent/TrigNavTools/python/TrigNavToolsConfig.py
+++ b/Trigger/TrigEvent/TrigNavTools/python/TrigNavToolsConfig.py
@@ -21,6 +21,8 @@ def navigationThinningSvc (config):
svc.Actions = [ 'DropFeatures', 'Reload', 'SyncThinning', 'DropChains', 'Save', 'Restore']
if config['mode'] == 'cleanup':
svc.Actions = [ 'DropFeatures', 'Reload', 'SyncThinning', 'Save']
+ if config['mode'] == 'cleanup_noreload':
+ svc.Actions = [ 'DropFeatures', 'SyncThinning', 'Save']
if 'Print' in svc.Actions:
from AthenaCommon.Constants import DEBUG
diff --git a/Trigger/TriggerCommon/TriggerJobOpts/python/HLTTriggerResultGetter.py b/Trigger/TriggerCommon/TriggerJobOpts/python/HLTTriggerResultGetter.py
index 75464ade9623..d768b74bde85 100644
--- a/Trigger/TriggerCommon/TriggerJobOpts/python/HLTTriggerResultGetter.py
+++ b/Trigger/TriggerCommon/TriggerJobOpts/python/HLTTriggerResultGetter.py
@@ -381,7 +381,10 @@ class HLTTriggerResultGetter(Configured):
edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
- svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup',
+ # TimM Sep 2021: In MT the 'reload' slimming option in the R2 navigation thinning service was found to be creating
+ # AODs which would crash when trying to return features. We therefore remove this option by using the added 'cleanup_noreload'
+ # configuration, see ATR-24141 for details.
+ svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup_noreload',
'result':'HLTResult_HLT',
'features':edmlist})
--
GitLab