Merge branch 'hlt-data-scouting-online' into 'master'

HLT data scouting - online part

See merge request atlas/athena!21401

HLT/Trigger/TrigControl/TrigExamples/TrigExPartialEB/share/MTCalibPeb.py

@@ -45,7 +45,8 @@ ctpUnpacker = CTPUnpackingTool(ForceEnableAllChains = True)
 # Define the "menu" - L1 items do not matter if we set ForceEnableAllChains = True,
 # but they have to be defined in the L1 menu xml
 chainCTPMap = {"HLT_MTCalibPeb1": "L1_RD0_FILLED",
-               "HLT_MTCalibPeb2": "L1_RD0_FILLED"}
+               "HLT_MTCalibPeb2": "L1_RD0_FILLED",
+               "HLT_MTCalibPeb3": "L1_RD0_FILLED"}
 
 # Schedule the L1Decoder algo with the above tools
 from L1Decoder.L1DecoderConf import L1Decoder
@@ -63,18 +64,20 @@ hypo = MTCalibPebHypoAlg()
 hypo.HypoInputDecisions = "HLTChains"
 hypo.HypoOutputDecisions = "MTCalibPebDecisions"
 
+# Chain 1 - high rate, writes PEB info
 hypoTool1 = MTCalibPebHypoTool("HLT_MTCalibPeb1")
 hypoTool1.RandomAcceptRate = 0.75
-hypoTool1.BurnTimePerCycleMillisec = 100
+hypoTool1.BurnTimePerCycleMillisec = 10
 hypoTool1.NumBurnCycles = 3
 hypoTool1.PEBROBList = [0x42002a, 0x42002b] # example LAr EMBC ROBs
 hypoTool1.PEBSubDetList = [0x65, 0x66] # RPC A and C side
 
+# Chain 2 - lower rate, makes ROB requests
 hypoTool2 = MTCalibPebHypoTool("HLT_MTCalibPeb2")
 hypoTool2.RandomAcceptRate = 0.25
-hypoTool2.BurnTimePerCycleMillisec = 200
+hypoTool2.BurnTimePerCycleMillisec = 20
 hypoTool2.NumBurnCycles = 10
-hypoTool2.TimeBetweenROBReqMillisec = 100
+hypoTool2.TimeBetweenROBReqMillisec = 50
 hypoTool2.ROBAccessDict = {
  "01 :ADD: Preload  ": [ 0x42002a, 0x42002b ],    # robs for 1st preload
  "02 :ADD: Preload  ": [ 0x42002e, 0x42002f ],    # robs for 2nd preload
@@ -87,22 +90,46 @@ hypoTool2.ROBAccessDict = {
  "09 :COL: Ev.Build ": [ 0x0 ]                    # event building
 } # This is just an example with a few ROBs (LAr in this case) for testing the ROBDataProvider
 
-hypo.HypoTools = [hypoTool1, hypoTool2]
+# Chain 3 - medium rate, produces random data, writes PEB info for data scouting
+hypoTool3 = MTCalibPebHypoTool("HLT_MTCalibPeb3")
+hypoTool3.RandomAcceptRate = 0.50
+hypoTool3.BurnTimePerCycleMillisec = 20
+hypoTool3.NumBurnCycles = 5
+hypoTool3.CreateRandomData = {
+  "ExampleCollection1": 5,
+  "ExampleCollection2": 3,
+}
+hypoTool3.PEBROBList = [0x42002e, 0x42002f] # example LAr EMBC ROBs
+hypoTool3.PEBROBList.extend([0x7c0001]) # extra HLT result for data scouting
+
+# Add the hypo tools to the algorithm
+hypo.HypoTools = [hypoTool1, hypoTool2, hypoTool3]
 
 ################################################################################
 # HLT result maker configuration
 ################################################################################
-
-from TrigOutputHandling.TrigOutputHandlingConf import TriggerEDMSerialiserTool, StreamTagMakerTool, TriggerBitsMakerTool
+from TrigOutputHandling.TrigOutputHandlingConf import HLTResultMTMakerAlg, StreamTagMakerTool, TriggerBitsMakerTool
+from TrigOutputHandling.TrigOutputHandlingConfig import TriggerEDMSerialiserToolCfg, HLTResultMTMakerCfg
 
 # Tool serialising EDM objects to fill the HLT result
-serialiser = TriggerEDMSerialiserTool()
-serialiser.CollectionsToSerialize = ["xAOD::TrigCompositeContainer_v1#MTCalibPebDecisions",
-                                     "xAOD::TrigCompositeAuxContainer_v1#MTCalibPebDecisionsAux."]
+serialiser = TriggerEDMSerialiserToolCfg("Serialiser")
+serialiser.addCollectionListToMainResult([
+  "xAOD::TrigCompositeContainer_v1#"+hypo.HypoOutputDecisions,
+  "xAOD::TrigCompositeAuxContainer_v2#"+hypo.HypoOutputDecisions+"Aux.decisions",
+])
+# Data scouting example
+resultList = [serialiser.fullResultID(), 1]
+serialiser.addCollectionListToResults([
+  "xAOD::TrigCompositeContainer_v1#ExampleCollection1",
+  "xAOD::TrigCompositeAuxContainer_v2#ExampleCollection1Aux.floatVec_1.floatVec_2.floatVec_3.floatVec_4.floatVec_5",
+  "xAOD::TrigCompositeContainer_v1#ExampleCollection2",
+  "xAOD::TrigCompositeAuxContainer_v2#ExampleCollection2Aux.floatVec_1.floatVec_2.floatVec_3",
+], resultList)
 
 # StreamTag definitions
-streamExamplePEB = ['ExamplePEB', 'calibration', "True", "False"]
 streamPhysicsMain = ['Main', 'physics', "True", "True"]
+streamExamplePEB = ['ExamplePEB', 'calibration', "True", "False"]
+streamExampleDataScoutingPEB = ['ExampleDataScoutingPEB', 'physics', "True", "False"]
 
 # Tool adding stream tags to HLT result
 stmaker = StreamTagMakerTool()
@@ -111,6 +138,7 @@ stmaker.PEBDecisionKeys = [hypo.HypoOutputDecisions]
 stmaker.ChainToStream = {}
 stmaker.ChainToStream["HLT_MTCalibPeb1"] = streamExamplePEB
 stmaker.ChainToStream["HLT_MTCalibPeb2"] = streamPhysicsMain
+stmaker.ChainToStream["HLT_MTCalibPeb3"] = streamExampleDataScoutingPEB
 
 # Tool adding HLT bits to HLT result
 bitsmaker = TriggerBitsMakerTool()
@@ -118,6 +146,7 @@ bitsmaker.ChainDecisions = "HLTSummary"
 bitsmaker.ChainToBit = {}
 bitsmaker.ChainToBit["HLT_MTCalibPeb1"] = 3
 bitsmaker.ChainToBit["HLT_MTCalibPeb2"] = 50
+bitsmaker.ChainToBit["HLT_MTCalibPeb3"] = 11
 
 # Configure the HLT result maker to use the above tools
 hltResultMaker = svcMgr.HltEventLoopMgr.ResultMaker

HLT/Trigger/TrigControl/TrigExamples/TrigExPartialEB/src/MTCalibPebHypoTool.cxx

@@ -13,6 +13,7 @@
 #include <random>
 #include <thread>
 #include <sstream>
+#include <algorithm>
 
 // Local implementation-specific helper methods
 namespace {
@@ -80,6 +81,23 @@ namespace {
     }
     return StatusCode::SUCCESS;
   }
+  /// Fill a TrigCompositeContainer with random data
+  void fillRandomData(xAOD::TrigCompositeContainer& data, unsigned int maxElements) {
+    unsigned int nObjects = randomInteger<unsigned int>(0, maxElements);
+    for (unsigned int iObj=0; iObj<nObjects; ++iObj) {
+      xAOD::TrigComposite* object = new xAOD::TrigComposite;
+      data.push_back(object);
+      object->setName(std::string("object_")+std::to_string(iObj));
+      unsigned int nAuxDataVec = randomInteger<unsigned int>(0, maxElements);
+      for (unsigned int iAuxDataVec=0; iAuxDataVec<nAuxDataVec; ++iAuxDataVec) {
+        xAOD::TrigComposite::Accessor<std::vector<float>> floatVec(std::string("floatVec_")+std::to_string(iAuxDataVec));
+        unsigned int nValues = randomInteger<unsigned int>(0, maxElements);
+        std::vector<float> values;
+        for (unsigned int iValue=0; iValue<nValues; ++iValue) values.push_back( randomRealNumber<float>(0,1) );
+        floatVec(*object) = values;
+      }
+    }
+  }
 }
 
 // =============================================================================
@@ -101,6 +119,14 @@ MTCalibPebHypoTool::~MTCalibPebHypoTool() {}
 StatusCode MTCalibPebHypoTool::initialize() {
   ATH_MSG_INFO("Initialising " << name());
   ATH_CHECK(m_robDataProviderSvc.retrieve());
+
+  // Copy keys from map<string,uint> to WriteHandleKeyArray
+  std::transform(m_createRandomData.begin(),
+                 m_createRandomData.end(),
+                 std::back_inserter(m_randomDataWHK),
+                 [](const auto& p){return p.first;});
+  ATH_CHECK(m_randomDataWHK.initialize());
+
   return StatusCode::SUCCESS;
 }
 
@@ -110,6 +136,7 @@ StatusCode MTCalibPebHypoTool::initialize() {
 StatusCode MTCalibPebHypoTool::finalize() {
   ATH_MSG_INFO("Finalising " << name());
   ATH_CHECK(m_robDataProviderSvc.release());
+  ATH_CHECK(m_randomDataWHK.initialize());
   return StatusCode::SUCCESS;
 }
 
@@ -170,6 +197,27 @@ StatusCode MTCalibPebHypoTool::decide(const MTCalibPebHypoTool::Input& input) co
     std::this_thread::sleep_for(std::chrono::milliseconds(m_timeBetweenRobReqMillisec));
   }
 
+  // ---------------------------------------------------------------------------
+  // Produce random data
+  // ---------------------------------------------------------------------------
+  {
+    using DataCont = xAOD::TrigCompositeContainer;
+    using AuxCont = xAOD::TrigCompositeAuxContainer;
+    for (const SG::WriteHandleKey<DataCont>& handleKey : m_randomDataWHK) {
+      // Create data and aux container
+      std::unique_ptr<DataCont> data = std::make_unique<DataCont>();
+      std::unique_ptr<AuxCont> aux = std::make_unique<AuxCont>();
+      data->setStore(aux.get());
+      // Record the container in event store
+      SG::WriteHandle<DataCont> handle(handleKey,input.eventContext);
+      ATH_CHECK( handle.record(std::move(data),std::move(aux)) );
+      ATH_MSG_DEBUG("Recorded TrigCompositeContainer " << handleKey.key() << " in event store");
+      // Fill the container with random data
+      unsigned int maxElements = m_createRandomData.value().at(handleKey.key());
+      fillRandomData(*handle,maxElements);
+    }
+  }
+
   // ---------------------------------------------------------------------------
   // Random accept decision with PEB information
   // ---------------------------------------------------------------------------

HLT/Trigger/TrigControl/TrigExamples/TrigExPartialEB/src/MTCalibPebHypoTool.h

@@ -82,6 +82,12 @@ private:
     this, "PEBSubDetList", {},
     "The list of SubDets to add for partial event building in each passed event"
   };
+  Gaudi::Property<std::map<std::string,unsigned int> > m_createRandomData {
+    this, "CreateRandomData", {},
+    "Create random data which can be serialised and saved to output. The property value should be a dictionary in the "
+    "form {name: number} where name is the collection name and number is the maximum number of elements and "
+    "their decorations"
+  };
 
   // ------------------------- Service or tool handles -------------------------
   ServiceHandle<IROBDataProviderSvc> m_robDataProviderSvc;
@@ -89,6 +95,8 @@ private:
   // ------------------------- Other private members ---------------------------
   /// The decision id of the tool instance
   HLT::Identifier m_decisionId;
+  /// WriteHandleKey array for collections specified in the CreateRandomData property
+  SG::WriteHandleKeyArray<xAOD::TrigCompositeContainer> m_randomDataWHK;
 };
 
 #endif // TRIGEXPARTIALEB_MTCALIBPEBHYPOTOOL_H

Trigger/TrigSteer/TrigHLTResultByteStream/src/HLTResultMTByteStreamCnv.cxx

@@ -21,6 +21,15 @@
 #include "eformat/Issue.h"
 #include "eformat/SourceIdentifier.h"
 
+// Local definitions
+namespace {
+  /**
+   * ROBFragment module ID of the HLT result which contains all Trigger EDM collections. Can be considered to be made
+   * a property set from the EDM configuration to avoid multiple definitions, but the value should never change from 0.
+   */
+  constexpr uint16_t fullResultModuleId = 0;
+}
+
 // =============================================================================
 // Standard constructor
 // =============================================================================
@@ -117,26 +126,48 @@ StatusCode HLT::HLTResultMTByteStreamCnv::createRep(DataObject* pObj, IOpaqueAdd
   const std::vector<uint32_t>& hltBits = hltResult->getHltBitsAsWords();
   re->hlt_info(hltBits.size(), hltBits.data());
 
+  // Read the stream tags to decide which HLT ROBFragments to write out
+  std::set<eformat::helper::SourceIdentifier> resultIdsToWrite;
+  for (const eformat::helper::StreamTag& st : hltResult->getStreamTags()) {
+    // In case of full event building, add the full result ID
+    if (st.robs.empty() && st.dets.empty()) {
+      eformat::helper::SourceIdentifier sid(eformat::SubDetector::TDAQ_HLT, fullResultModuleId);
+      resultIdsToWrite.insert(sid);
+    }
+    // In case of partial event building, add the results explicitly requested in the stream tag
+    for (const uint32_t robid : st.robs) {
+      eformat::helper::SourceIdentifier sid(robid);
+      if (sid.subdetector_id() == eformat::SubDetector::TDAQ_HLT)
+        resultIdsToWrite.insert(sid);
+    }
+  }
+
   // Clear the FEA stack
   m_fullEventAssembler.clear();
 
-  // Loop over all module IDs and fill the ROBFragments
-  ATH_MSG_DEBUG("Iterating over modules to assemble output data");
-  for (const auto& p : hltResult->getSerialisedData()) {
-    const uint16_t moduleId = p.first;
-    const std::vector<uint32_t>& data = p.second;
+  // Loop over the module IDs and fill the ROBFragments
+  ATH_MSG_DEBUG("Iterating over " << resultIdsToWrite.size() << " HLT result IDs to assemble output data");
+  const std::unordered_map<uint16_t, std::vector<uint32_t>>& serialisedData = hltResult->getSerialisedData();
+  for (const eformat::helper::SourceIdentifier& resultId : resultIdsToWrite) {
+    // Find the serialised data for this module ID
+    const auto it = serialisedData.find(resultId.module_id());
+    if (it==serialisedData.end()) {
+      ATH_MSG_ERROR("HLT result with ID 0x" << MSG::hex << resultId.code() << MSG::dec
+                    << " requested by a stream tag, but missing in the serialised data");
+      return StatusCode::FAILURE;
+    }
+    const std::vector<uint32_t>& data = it->second;
 
     // Get a pointer to ROD data vector to be filled
-    eformat::helper::SourceIdentifier sid(eformat::TDAQ_HLT,moduleId);
-    std::vector<uint32_t>* rod = m_fullEventAssembler.getRodData(sid.code());
+    std::vector<uint32_t>* rod = m_fullEventAssembler.getRodData(resultId.code());
     if (!rod) {
-      ATH_MSG_ERROR("Failed to get RODDATA pointer for module " << sid.code());
+      ATH_MSG_ERROR("Failed to get RODDATA pointer for HLT result ID 0x" << MSG::hex << resultId.code() << MSG::dec);
       return StatusCode::FAILURE;
     }
 
     // Fill the ROD data vector
     rod->assign(data.cbegin(), data.cend());
-    ATH_MSG_DEBUG("Assembled data for module 0x" << MSG::hex << sid.code() << MSG::dec << " with "
+    ATH_MSG_DEBUG("Assembled data for HLT result ID 0x" << MSG::hex << resultId.code() << MSG::dec << " with "
                   << data.size() << " words of serialised payload");
   }