From b16916a26cf595fc9c364271f7e900d909ea622a Mon Sep 17 00:00:00 2001 From: Rafal Bielski Date: Tue, 5 Nov 2019 01:10:27 +0100 Subject: [PATCH] Implement HLT result truncation on total size --- .../TrigControl/TrigPSC/src/Config.cxx | 1 + HLT/Trigger/TrigControl/TrigPSC/src/Psc.cxx | 11 +++ .../src/TriggerEDMSerialiserTool.cxx | 79 +++++++++++++++++-- .../src/TriggerEDMSerialiserTool.h | 9 +++ 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/HLT/Trigger/TrigControl/TrigPSC/src/Config.cxx b/HLT/Trigger/TrigControl/TrigPSC/src/Config.cxx index ff1c3411b9c..0f689cc23df 100644 --- a/HLT/Trigger/TrigControl/TrigPSC/src/Config.cxx +++ b/HLT/Trigger/TrigControl/TrigPSC/src/Config.cxx @@ -328,6 +328,7 @@ void psc::Config::fillopt_common(const ptree& hlt) optmap["SOFTTIMEOUTFRACTION"] = hltmppu.get_child("softTimeoutFraction").data(); optmap["NEVENTSLOTS"] = hltmppu.get_child("numberOfEventSlots").data(); optmap["NTHREADS"] = hltmppu.get_child("numberOfAthenaMTThreads").data(); + optmap["MAXEVENTSIZEMB"] = hltmppu.get_child("maximumHltResultMb").data(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/HLT/Trigger/TrigControl/TrigPSC/src/Psc.cxx b/HLT/Trigger/TrigControl/TrigPSC/src/Psc.cxx index c597ae64787..17631d31f8f 100644 --- a/HLT/Trigger/TrigControl/TrigPSC/src/Psc.cxx +++ b/HLT/Trigger/TrigControl/TrigPSC/src/Psc.cxx @@ -381,6 +381,17 @@ bool psc::Psc::configure(const ptree& config) <<" number of Sub Det IDs read from OKS = " << m_config->enabled_SubDets.size()); } + // Write the maximum HLT output size into the JobOptions Catalogue + if (std::string opt = m_config->getOption("MAXEVENTSIZEMB"); !opt.empty()) { + StatusCode sc = p_jobOptionSvc->addPropertyToCatalogue("DataFlowConfig", + IntegerProperty("DF_MaxEventSizeMB", std::stoi(opt))); + if ( sc.isFailure() ) { + ERS_PSC_ERROR("psc::Psc::configure: Error could not write DF_MaxEventSizeMB in JobOptions Catalogue"); + return false; + } + ERS_DEBUG(1,"psc::Psc::configure: Wrote DF_MaxEventSizeMB=" << opt << " in JobOptions Catalogue"); + } + // Write configuration for HLT muon calibration infrastructure in JobOptions catalogue if ( (m_config->getOption("MUONCALBUFFERNAME") != "NONE") && (m_config->getOption("MUONCALBUFFERNAME") != "") ) { std::map muoncal_properties; diff --git a/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.cxx b/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.cxx index 52fb2043d5d..0163e87a8fc 100644 --- a/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.cxx +++ b/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.cxx @@ -5,6 +5,7 @@ #include #include #include +#include "GaudiKernel/IJobOptionsSvc.h" #include "GaudiKernel/IToolSvc.h" #include "GaudiKernel/System.h" #include "AthenaKernel/StorableConversions.h" @@ -17,6 +18,13 @@ #include "TriggerEDMSerialiserTool.h" +#include + +namespace { + // Special module ID used internally to store total result size limit in the truncation threshold map + constexpr uint16_t fullResultTruncationID = std::numeric_limits::max(); +} + TriggerEDMSerialiserTool::TriggerEDMSerialiserTool( const std::string& type, const std::string& name, const IInterface* parent ) @@ -34,6 +42,39 @@ StatusCode TriggerEDMSerialiserTool::initialize() { for ( const std::string& typeKeyAuxIDs : m_collectionsToSerialize.value() ) { ATH_CHECK(addCollectionToSerialise(typeKeyAuxIDs, m_toSerialise)); } + + // Retrieve the total result size limit from DataFlowConfig which is a special object + // used online to hold DF properties passed from TDAQ to HLT as run parameters + SmartIF jobOptionsSvc = service("JobOptionsSvc", /*createIf=*/ false); + if (!jobOptionsSvc.isValid()) { + ATH_MSG_WARNING("Could not retrieve JobOptionsSvc, will not update the EventSizeHardLimitMB property"); + } + else { + const Gaudi::Details::PropertyBase* prop = jobOptionsSvc->getClientProperty("DataFlowConfig", "DF_MaxEventSizeMB"); + if (prop && m_eventSizeHardLimitMB.assign(*prop)) { + ATH_MSG_DEBUG("Updated EventSizeHardLimitMB to " << m_eventSizeHardLimitMB.value() + << " from DataFlowConfig.DF_MaxEventSizeMB"); + } + else { + ATH_MSG_DEBUG("Could not retrieve DataFlowConfig.DF_MaxEventSizeMB from JobOptionsSvc. This is fine if running " + << "offline, but should not happen online. Leaving EventSizeHardLimitMB=" + << m_eventSizeHardLimitMB.value()); + } + } + + // Add the total result size limit to truncation threshold map + if (m_eventSizeHardLimitMB >= 0) { + if (m_fullResultTruncationFrac > 1.0) { + ATH_MSG_ERROR("Fraction cannot be > 1.0, but FullResultTruncationFrac is set to " << m_fullResultTruncationFrac); + return StatusCode::FAILURE; + } + float totalResultSizeLimitBytes = m_fullResultTruncationFrac * m_eventSizeHardLimitMB * 1024. * 1024.; + m_truncationThresholds[fullResultTruncationID] = static_cast(totalResultSizeLimitBytes); + } + else { + m_truncationThresholds[fullResultTruncationID] = std::numeric_limits::max(); + } + return StatusCode::SUCCESS; } @@ -374,19 +415,32 @@ StatusCode TriggerEDMSerialiserTool::tryAddData(HLT::HLTResultMT& hltResult, return StatusCode::FAILURE; } + // Size in this module const uint32_t currentSizeBytes = hltResult.getSerialisedData().count(id)==0 ? 0 : hltResult.getSerialisedData().at(id).size()*sizeof(uint32_t); + // Total size + size_t currentTotalSizeWords = 0; + for (const auto& [id, data] : hltResult.getSerialisedData()) currentTotalSizeWords += data.size(); + const uint32_t currentTotalSizeBytes = currentTotalSizeWords*sizeof(uint32_t); + // Size to be added const uint32_t extraSizeBytes = data.size()*sizeof(uint32_t); - if (currentSizeBytes+extraSizeBytes < m_truncationThresholds.value().at(id)) { - // The data fits, so add it to the result - ATH_MSG_DEBUG("Adding data to result with module ID " << id); - hltResult.addSerialisedData(id, data); + + if (currentTotalSizeBytes+extraSizeBytes > m_truncationThresholds.value().at(fullResultTruncationID)) { + // The data doesn't fit, flag the full result as truncated + ATH_MSG_DEBUG("Skipping adding data to result with module ID " << id << " because of full-result truncation"); + hltResult.addTruncatedModuleId(fullResultTruncationID); + hltResult.addTruncatedModuleId(id); } - else { - // The data doesn't fit, flag the result as truncated + else if (currentSizeBytes+extraSizeBytes > m_truncationThresholds.value().at(id)) { + // The data doesn't fit, flag this module's result as truncated ATH_MSG_DEBUG("Skipping adding data to truncated result with module ID " << id); hltResult.addTruncatedModuleId(id); } + else { + // The data fits, so add it to the result + ATH_MSG_DEBUG("Adding data to result with module ID " << id); + hltResult.addSerialisedData(id, data); + } return StatusCode::SUCCESS; } @@ -394,6 +448,15 @@ StatusCode TriggerEDMSerialiserTool::fillDebugInfo(const TruncationInfoMap& trun xAOD::TrigCompositeContainer& debugInfoCont, HLT::HLTResultMT& resultToFill, SGImplSvc* evtStore) const { + // If full result truncation happened, flag all results as truncated to produce debug info for all + if (resultToFill.getTruncatedModuleIds().count(fullResultTruncationID)>0) { + ATH_MSG_ERROR("HLT result truncation on total size! Limit of " + << m_truncationThresholds.value().at(fullResultTruncationID)/1024./1024. + << " MB exceeded. Flagging all module IDs as truncated."); + for (const auto& [id, data] : resultToFill.getSerialisedData()) { + resultToFill.addTruncatedModuleId(id); + } + } // Loop over truncation info and fill histograms and debug info objects for (const auto& [id, truncationInfoVec] : truncationInfoMap) { if (resultToFill.getTruncatedModuleIds().count(id)>0) { @@ -405,8 +468,8 @@ StatusCode TriggerEDMSerialiserTool::fillDebugInfo(const TruncationInfoMap& trun xAOD::TrigComposite::Accessor> typeNameVec("typeName"); xAOD::TrigComposite::Accessor> sizeVec("size"); xAOD::TrigComposite::Accessor> isRecordedVec("isRecorded"); - std::pair largestRecorded; - std::pair largestDropped; + std::pair largestRecorded{"None", 0}; + std::pair largestDropped{"None", 0}; moduleId(*debugInfoThisModule) = id; uint32_t sizeSum = 0; for (const TruncationInfo& truncationInfo : truncationInfoVec) { diff --git a/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.h b/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.h index d5d81e07d87..4b5ef8da79f 100644 --- a/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.h +++ b/Trigger/TrigSteer/TrigOutputHandling/src/TriggerEDMSerialiserTool.h @@ -63,6 +63,15 @@ class TriggerEDMSerialiserTool: public extends Gaudi::Property> m_truncationThresholds { this, "TruncationThresholds", {}, "HLT result truncation thresholds. Key is module ID, value is max size in bytes" }; + Gaudi::Property m_eventSizeHardLimitMB { + this, "EventSizeHardLimitMB", -1, + "Hard limit for output event size in megabytes. Set automatically in initialize from run parameters. " + "In partition, it corresponds to DCM sbaBlockSize_MiB. Value <0 means no limit." + }; + Gaudi::Property m_fullResultTruncationFrac { + this, "FullResultTruncationFrac", 0.8, + "Fraction of EventSizeHardLimitMB which defines the limit on the total size of HLT output (sum of all modules)" + }; /// StoreGate key for the truncation debug info object SG::WriteHandleKey m_debugInfoWHKey { this, "DebugInfoWHKey", "TruncationDebugInfo" -- GitLab