Merge branch 'propagat_TE_data' into '22.0-mc20'

[ATR-24058][ATR-24141] Working to add R2 trigger TE data to R3 JSON structure to enable feature access from R22 AODs created from R2 data

See merge request atlas/athena!46419

Control/AthenaMonitoring/share/TrigDecTool_jobOptions.py

@@ -20,20 +20,15 @@ if DQMonFlags.useTrigger():
          from TriggerJobOpts.TriggerConfigGetter import TriggerConfigGetter
          cfg = TriggerConfigGetter()
 
-   if not hasattr(ToolSvc, DQMonFlags.nameTrigDecTool().split('/')[-1]):
-      if rec.doTrigger():
-         tdt_local_logger.error('DQ Monitoring is being asked to set up the TrigDecisionTool for some reason.  THIS IS A TERRIBLE IDEA AND SHOULD BE CONSIDERED A BUG!')
-      from AthenaMonitoring.TriggerInterface import getTrigDecisionTool
-      from AthenaConfiguration.OldFlags2NewFlags import getNewConfigFlags
-      # Translate all needed flags from old jobProperties to a new AthConfigFlag Container
-      ConfigFlags = getNewConfigFlags()
-
-      from AthenaConfiguration import ComponentAccumulator
-      ComponentAccumulator.CAtoGlobalWrapper(getTrigDecisionTool, ConfigFlags)
-
-      monTrigDecTool = getattr(ToolSvc, 'TrigDecisionTool')
-   else:
-      monTrigDecTool = getattr(ToolSvc, DQMonFlags.nameTrigDecTool().split('/')[-1])
+   from AthenaCommon.Configurable import Configurable
+   Configurable.configurableRun3Behavior+=1
+   from TrigDecisionTool.TrigDecisionToolConfig import getTrigDecisionTool 
+   tdtAcc = getTrigDecisionTool(ConfigFlags)
+   Configurable.configurableRun3Behavior-=1
+
+   from AthenaConfiguration.ComponentAccumulator import conf2toConfigurable, appendCAtoAthena
+   monTrigDecTool = conf2toConfigurable(tdtAcc.getPrimary())
+   appendCAtoAthena( tdtAcc )
    tdt_local_logger.info('Scheduled monitoring TDT %s', monTrigDecTool)
 
    tdt_local_logger.info('Scheduling the trigger translator')

Trigger/TrigConfiguration/TrigConfIO/src/JsonFileWriterHLT.cxx

@@ -18,9 +18,36 @@ TrigConf::JsonFileWriterHLT::JsonFileWriterHLT() :
 {}
 
 
+/// Helper function ptree key->[] to std::vector<T>
+template<typename T>
+std::vector<T> ToVector(const TrigConf::DataStructure& ds, const std::string& child){
+   using ptree = boost::property_tree::ptree;
+   std::vector<T> return_vector;
+   for( const ptree::value_type& entry : ds.data().get_child(child) ) {
+      return_vector.push_back( entry.second.get_value<T>() );
+   }
+   return return_vector;
+}
+
+/// Helper function ptree key->[[]] to std::vector<std::vector<T>>
+template<typename T>
+std::vector<std::vector<T>> ToVectorVector(const TrigConf::DataStructure& ds, const std::string& child){
+   using ptree = boost::property_tree::ptree;
+   std::vector<std::vector<T>> return_vector;
+   for( const ptree::value_type& outer : ds.data().get_child(child) ) {
+      return_vector.push_back(std::vector<T>());
+      for (const ptree::value_type& inner : outer.second) {
+         return_vector.back().push_back( inner.second.get_value<T>() );
+      }
+   }
+   return return_vector;
+}
+
+
 bool
 TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const HLTMenu & menu) const
 {
+   using ptree = boost::property_tree::ptree;
    json chains({});
    for ( const auto & chain : menu ) {
       json jChain({});
@@ -32,6 +59,17 @@ TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const H
       jChain["groups"] = chain.groups();
       jChain["streams"] = chain.streams();
       jChain["seqeuncers"] = chain.sequencers();
+
+      // Optional Run2 payload
+      if (chain.hasChild("signature")) {
+         json jSig({});
+         jSig["counters"] = ToVector<uint32_t>(chain, "signature.counters");
+         jSig["logics"] = ToVector<int>(chain, "signature.logics");
+         jSig["labels"] = ToVector<std::string>(chain, "signature.labels");
+         jSig["outputTEs"] = ToVectorVector<std::string>(chain, "signature.outputTEs");
+         jChain["signature"] = jSig;
+      }
+
       chains[chain.name()] = jChain;
    }
 
@@ -58,6 +96,14 @@ TrigConf::JsonFileWriterHLT::writeJsonFile(const std::string & filename, const H
    j["sequencers"] = sequencers;
    j["streams"] = streams;
 
+   // Optional Run2 payload
+   if (menu.hasChild("sequence_run2")) {
+      json jSequence({});
+      jSequence["outputTEs"] = ToVector<std::string>(menu, "sequence_run2.outputTEs");
+      jSequence["inputTEs"] = ToVectorVector<std::string>(menu, "sequence_run2.inputTEs");
+      jSequence["algorithms"] = ToVectorVector<std::string>(menu, "sequence_run2.algorithms");
+      j["sequence_run2"] = jSequence;
+   }
 
    std::ofstream outfile(filename);
    outfile << std::setw(4) << j << std::endl;

Trigger/TrigConfiguration/TrigConfStorage/src/test/Run2toRun3ConvertersHLT.cxx

@@ -81,6 +81,35 @@ bool convertHLTMenu(const TrigConf::HLTFrame* frame, TrigConf::HLTMenu& menu) {
 
       pChain.add_child("groups", asArray(cptr->groups()));
 
+      // Signature data
+      // Note: This is run-2 only.
+      // It is propagated here to allow legacy trigger feature access.
+      std::vector<uint32_t> counters;
+      std::vector<int> logics;
+      std::vector<std::string> labels;
+      ptree outputTEs_outerArray; // outputTEs is a std::vector<std::vector<std::string>>
+
+      for(auto& signature : cptr->signatureList() ){
+         uint32_t cntr = signature->signature_counter();
+         counters.push_back(cntr);
+         logics.push_back(signature->logic());
+         labels.push_back(signature->label());
+         ptree outputTEs_innerArray;
+         for(auto& outputTE : signature->outputTEs()){
+            outputTEs_innerArray.push_back( ptree::value_type("", outputTE->name()) );
+         }
+         outputTEs_outerArray.push_back( ptree::value_type("", outputTEs_innerArray) );
+      }
+
+      ptree pSig; 
+      pSig.add_child("counters", asArray(counters));
+      pSig.add_child("logics", asArray(logics));
+      pSig.add_child("outputTEs", outputTEs_outerArray);
+      pSig.add_child("labels", asArray(labels));
+
+      pChain.add_child("signature", pSig);
+      // End of signature data
+
       pChains.push_back(std::make_pair(cptr->chain_name(), pChain));
    }
    ptree pStreams;
@@ -100,6 +129,32 @@ bool convertHLTMenu(const TrigConf::HLTFrame* frame, TrigConf::HLTMenu& menu) {
    pSequencers.add_child("missing", asArray(std::vector<std::string>({""})));
    top.add_child("sequencers", pSequencers);
 
+   // Set run2 sequence information:
+   const TrigConf::HLTSequenceList& sequenceList = frame->getHLTSequenceList();
+   std::vector<std::string> outputTEs;
+   ptree inputTEs_outerArray;  // sequenceInputTEs is a std::vector<std::vector<std::string>>
+   ptree algorithms_outerArray;  // sequenceAlgorithms is a std::vector<std::vector<std::string>>
+   for(auto& seq : sequenceList){
+      outputTEs.push_back(seq->outputTE()->name());
+
+      ptree inputTEs_innerArray;
+      for(auto& input : seq->inputTEs()) {
+         inputTEs_innerArray.push_back( ptree::value_type("", input->name()) );
+      }
+      inputTEs_outerArray.push_back( ptree::value_type("", inputTEs_innerArray) );
+
+      ptree algorithms_innerArray;
+      for(const std::string& alg : seq->algorithms()) {
+         algorithms_innerArray.push_back( ptree::value_type("", alg) );
+      }
+      algorithms_outerArray.push_back( ptree::value_type("", algorithms_innerArray) );
+   }
+   ptree pSequence;
+   pSequence.add_child("outputTEs", asArray(outputTEs));
+   pSequence.add_child("inputTEs", inputTEs_outerArray);
+   pSequence.add_child("algorithms", algorithms_outerArray);
+   top.add_child("sequence_run2", pSequence);
+
    menu.setData(std::move(top));
    menu.setSMK(frame->smk());
    return true;

Trigger/TrigConfiguration/TrigConfxAOD/Root/prepareTriggerMenu.cxx

@@ -253,6 +253,32 @@ namespace TrigConf {
       return StatusCode::SUCCESS;
    }
 
+
+   /// Helper function ptree key->[] to std::vector<T>
+   template<typename T>
+   std::vector<T> ToVector(const TrigConf::DataStructure& ds, const std::string& child){
+      using ptree = boost::property_tree::ptree;
+      std::vector<T> return_vector;
+      for( const ptree::value_type& entry : ds.data().get_child(child) ) {
+         return_vector.push_back( entry.second.get_value<T>() );
+      }
+      return return_vector;
+   }
+
+   /// Helper function ptree key->[[]] to std::vector<std::vector<T>>
+   template<typename T>
+   std::vector<std::vector<T>> ToVectorVector(const TrigConf::DataStructure& ds, const std::string& child){
+      using ptree = boost::property_tree::ptree;
+      std::vector<std::vector<T>> return_vector;
+      for( const ptree::value_type& outer : ds.data().get_child(child) ) {
+         return_vector.push_back(std::vector<T>());
+         for (const ptree::value_type& inner : outer.second) {
+            return_vector.back().push_back( inner.second.get_value<T>() );
+         }
+      }
+      return return_vector;
+   }
+
    /// Load JSON derived data into legacy structures to maintain 
    /// compatiblity with existing code.
    ///
@@ -315,46 +341,75 @@ namespace TrigConf {
       // Fill the HLT configuration:
       if (loadedHlt.isInitialized()) {
          for (const Chain& loadedChain : loadedHlt) {
-             // Figure out which level this chain is from:
-             std::string level = "";
-             if( loadedChain.name().find( "L2_" ) == 0 ) {
-                level = "L2";
-             } else if( loadedChain.name().find( "EF_" ) == 0 ) {
-                level = "EF";
-             } else if( loadedChain.name().find( "HLT_" ) == 0 ) {
-                level = "HLT";
-             } else {
-                msg << MSG::WARNING << "prepareTriggerMenu(...): "
-                    << "Couldn't figure out 'level' for chain: "
-                    << loadedChain.name() << endmsg;
-             }
-             // An empty signature list for the chain:
-             // This is not populated from JSON data
-             std::vector< HLTSignature* > signatures;
+            // Figure out which level this chain is from:
+            std::string level = "";
+            if( loadedChain.name().find( "L2_" ) == 0 ) {
+               level = "L2";
+            } else if( loadedChain.name().find( "EF_" ) == 0 ) {
+               level = "EF";
+            } else if( loadedChain.name().find( "HLT_" ) == 0 ) {
+               level = "HLT";
+            } else {
+               msg << MSG::WARNING << "prepareTriggerMenu(...): "
+                   << "Couldn't figure out 'level' for chain: "
+                   << loadedChain.name() << endmsg;
+            }
+
+            // An empty signature list for the chain:
+            std::vector< HLTSignature* > signatures;
+
+            // Optional Run2 payload
+            std::vector<uint32_t> counters;
+            std::vector<int> logics;
+            std::vector<std::vector<std::string>> outputTEs;
+            if (loadedChain.hasChild("signature")) {
+               counters = ToVector<uint32_t>(loadedChain, "signature.counters");
+               logics = ToVector<int>(loadedChain, "signature.logics");
+               outputTEs = ToVectorVector<std::string>(loadedChain, "signature.outputTEs");
+            }
+
+            if( msg.level() <= MSG::VERBOSE ) {
+               msg << MSG::VERBOSE << "chain " << loadedChain.name()
+                   << " has counter " << loadedChain.counter()
+                   << " and " << counters.size() << " signatures (runs 1,2 only)" << endmsg;
+            }
+
+            for( size_t sig = 0; sig < counters.size(); ++sig ) {
+               std::vector< HLTTriggerElement* > outTEs;
+               for( size_t outTEcounter = 0; outTEcounter< outputTEs[ sig ].size(); ++outTEcounter ) {
+                  HLTTriggerElement* element = new HLTTriggerElement( outputTEs[ sig ][ outTEcounter ] );
+                  outTEs.push_back( element );
+               }
+               HLTSignature* signature = new HLTSignature( counters[ sig ], logics[ sig ], outTEs );
+               signatures.push_back( signature );
+               if( msg.level() <= MSG::VERBOSE ) {
+                  msg << MSG::VERBOSE << "prepared signature: " << *( signatures.back() ) << endmsg;
+               }
+            }
  
-             // Create the chain object:
-             HLTChain* chain = new HLTChain( loadedChain.name(),
-                                             loadedChain.counter(),
-                                             1, // Chain version not important
-                                             level,
-                                             loadedChain.l1item(), // L1 seeds (string)
-                                             -1, // Lower chain ID not important
-                                             signatures ); // Note: Empty
+            // Create the chain object:
+            HLTChain* chain = new HLTChain( loadedChain.name(),
+                                            loadedChain.counter(),
+                                            1, // Chain version not important
+                                            level,
+                                            loadedChain.l1item(), // L1 seeds (string)
+                                            -1, // Lower chain ID not important
+                                            signatures ); // Empty for R3 JSONs
  
-             chain->set_rerun_prescale( -1.0 ); // Not used in R3
-             chain->set_pass_through( -1.0 );  // Not used in R3
-             chain->set_leg_multiplicities( loadedChain.legMultiplicities() );
-
-             for (const std::string& group : loadedChain.groups()){
-                chain->addGroup(group);
-             }
-
-             if (loadedHltps.isInitialized()) {
-                const HLTPrescalesSet::HLTPrescale& loadedPrescale = loadedHltps.prescale( loadedChain.name() );
-                chain->set_prescale( loadedPrescale.prescale );
-             } else {
-                chain->set_prescale( 0 );
-             }
+            chain->set_rerun_prescale( -1.0 ); // Not used in R3
+            chain->set_pass_through( -1.0 );  // Not used in R3
+            chain->set_leg_multiplicities( loadedChain.legMultiplicities() );
+
+            for (const std::string& group : loadedChain.groups()){
+               chain->addGroup(group);
+            }
+
+            if (loadedHltps.isInitialized()) {
+               const HLTPrescalesSet::HLTPrescale& loadedPrescale = loadedHltps.prescale( loadedChain.name() );
+               chain->set_prescale( loadedPrescale.prescale );
+            } else {
+               chain->set_prescale( 0 );
+            }
  
             // Add it to the list of chains:
             if( ! chainList.addHLTChain( chain ) ) {
@@ -367,7 +422,25 @@ namespace TrigConf {
          }
       }
 
-      // Do not add sequence info to legacy structures (format is different)
+      // Add sequence information if it's available (R1 or R2 menu):
+      if( loadedHlt.hasChild("sequence_run2") ) {
+         std::vector<std::string> menu_outputTEs = ToVector<std::string>(loadedHlt, "sequence_run2.outputTEs");
+         std::vector<std::vector<std::string>> menu_inputTEs = ToVectorVector<std::string>(loadedHlt, "sequence_run2.inputTEs");
+         std::vector<std::vector<std::string>> menu_algorithms = ToVectorVector<std::string>(loadedHlt, "sequence_run2.algorithms");
+
+         for( size_t i = 0; i< menu_outputTEs.size(); ++i ) {
+            HLTTriggerElement* outputTE = new HLTTriggerElement( menu_outputTEs[ i ] );
+            std::vector< HLTTriggerElement* > inputTEs;
+            for( size_t j = 0; j < menu_inputTEs[ i ].size(); ++j ) {
+               HLTTriggerElement* te = new HLTTriggerElement( menu_inputTEs[ i ][ j ] );
+               inputTEs.push_back( te );
+            }
+            HLTSequence* sequence = new HLTSequence( inputTEs, outputTE, menu_algorithms[ i ] );
+            sequenceList.addHLTSequence( sequence );
+            // This throws a runtime_error if it fails, which we don't need to
+            // handle, since this is a FATAL error anyways.
+         }
+      }
 
       // Bunchgroup data is TODO
       // Create a new BunchGroupSet object, since an existing one can't be

Trigger/TrigEvent/TrigNavTools/python/TrigNavToolsConfig.py

@@ -21,6 +21,8 @@ def navigationThinningSvc (config):
     svc.Actions = [ 'DropFeatures', 'Reload', 'SyncThinning', 'DropChains', 'Save', 'Restore']
   if config['mode'] == 'cleanup':    
     svc.Actions = [ 'DropFeatures', 'Reload', 'SyncThinning', 'Save']
+  if config['mode'] == 'cleanup_noreload':    
+    svc.Actions = [ 'DropFeatures', 'SyncThinning', 'Save']
 
   if 'Print' in svc.Actions:
     from AthenaCommon.Constants import DEBUG

Trigger/TriggerCommon/TriggerJobOpts/python/HLTTriggerResultGetter.py

@@ -381,7 +381,10 @@ class HLTTriggerResultGetter(Configured):
 
             edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
           
-            svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup', 
+            # TimM Sep 2021: In MT the 'reload' slimming option in the R2 navigation thinning service was found to be creating
+            # AODs which would crash when trying to return features. We therefore remove this option by using the added 'cleanup_noreload'
+            # configuration, see ATR-24141 for details. 
+            svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup_noreload', 
                                           'result':'HLTResult_HLT',
                                           'features':edmlist})