Commit 7233ca4e authored by Tim Martin's avatar Tim Martin Committed by Johannes Elmsheuser
Browse files

Offline R3 Trigger Slimming - further integration

parent a3ba6247
......@@ -145,17 +145,15 @@ std::vector< TrigCompositeUtils::LinkInfo<CONTAINER> > Trig::ChainGroup::feature
std::vector<TrigCompositeUtils::LinkInfo<CONTAINER>> returnVector =
TrigCompositeUtils::recursiveGetFeaturesOfType<CONTAINER>(navGraph, containerSGKey, lastFeatureOfTypeFlag, navElementLinkKey, allRequestedChainIDs);
// Check for missing navigation data if requesting the default "feature" links
if (navElementLinkKey == TrigCompositeUtils::featureString()) {
const auto& ctx = Gaudi::Hive::currentContext();
for (const TrigCompositeUtils::LinkInfo<CONTAINER>& linkInfo : returnVector) {
if (linkInfo.link.isValid()) continue; // We're looking for invalid links...
if (linkInfo.source->isRemapped()) continue; // ... which do not have remapping data.
ElementLink<TrigCompositeUtils::DecisionContainer> sourceEL = TrigCompositeUtils::decisionToElementLink( linkInfo.source, ctx );
ATH_MSG_WARNING("A link to a feature from " << sourceEL.dataID() << " is invalid, "
"this is due to this container not having its 'remap_linkColIndices' and 'remap_linkColKeys' decorations.");
break;
}
size_t invalid = 0;
for (const TrigCompositeUtils::LinkInfo<CONTAINER>& linkInfo : returnVector) {
if (not linkInfo.link.isValid()) {
++invalid;
}
}
if (invalid) {
ATH_MSG_WARNING(invalid << " of " << returnVector.size() << " returned features have invalid element links. Check the Trigger EDM. "
<< "Request was for features of type " << ClassID_traits<CONTAINER>::typeName());
}
return returnVector;
......
......@@ -16,16 +16,17 @@ When running in AnalysisBase, the tdt.TrigConfigTool="TrigConf::xAODConfigTool"
in place of the TrigConf::xAODConfigSvc
'''
def getRun3NavigationContainerFromInput(ConfigFlags):
# List of all possible keys of the Run 3 navigation summary collection
# in order of verbosity. Want to take the most verbose which is available.
possible_keys = []
possible_keys.append("HLTNav_Summary")
possible_keys.append("HLTNav_Summary_BSSlimmed")
possible_keys.append("HLTNav_Summary_ESDSlimmed")
possible_keys.append("HLTNav_Summary_AODSlimmed")
possible_keys.append("HLTNav_Summary_DAODSlimmed")
# List of all possible keys of the Run 3 navigation summary collection
# in order of verbosity. Want to take the most verbose which is available.
possible_keys = [
"HLTNav_Summary",
"HLTNav_Summary_OnlineSlimmed",
"HLTNav_Summary_ESDSlimmed",
"HLTNav_Summary_AODSlimmed",
"HLTNav_Summary_DAODSlimmed"
]
def getRun3NavigationContainerFromInput(ConfigFlags):
# What to return if we cannot look in the file
default_key = "HLTNav_Summary_OnlineSlimmed"
to_return = default_key
......@@ -41,17 +42,13 @@ def getRun3NavigationContainerFromInput(ConfigFlags):
return to_return
def getTrigDecisionTool(ConfigFlags):
from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
from AthenaConfiguration.ComponentFactory import CompFactory
from AthenaCommon.Logging import logging
msg = logging.getLogger('getTrigDecisionTool')
acc = ComponentAccumulator()
cfgsvc = CompFactory.TrigConf.xAODConfigSvc('xAODConfigSvc')
# We serve in-file metadata where possible. If it does not exist (e.g. RAWtoESD, RAWtoALL), then it is obtained from the Conditions and Detector stores
cfgsvc.UseInFileMetadata = ConfigFlags.Trigger.InputContainsConfigMetadata
acc.addService(cfgsvc)
from TrigConfxAOD.TrigConfxAODConfig import getxAODConfigSvc
acc = getxAODConfigSvc(ConfigFlags)
cfgsvc = acc.getPrimary()
tdt = CompFactory.Trig.TrigDecisionTool('TrigDecisionTool')
tdt.TrigConfigSvc = cfgsvc
......@@ -68,10 +65,6 @@ def getTrigDecisionTool(ConfigFlags):
acc.addPublicTool(nav)
acc.addPublicTool(tdt, primary=True)
if ConfigFlags.Trigger.InputContainsConfigMetadata:
from AthenaServices.MetaDataSvcConfig import MetaDataSvcCfg
acc.merge(MetaDataSvcCfg(ConfigFlags))
msg.info("Configuring the TrigDecisionTool and xAODConfigSvc to use ConfigSource:{}, Run3NavigationFormat:{}, Run3NavigationSummaryCollection:{}".format(
"InFileMetadata" if ConfigFlags.Trigger.InputContainsConfigMetadata else "ConditionsAndDetStore",
str(use_run3_format),
......
......@@ -20,7 +20,7 @@ def getTrigNavSlimmingMTConfig(ConfigFlags):
log = logging.getLogger("getTrigNavSlimmingMTConfig.py")
from TrigDecisionTool.TrigDecisionToolConfig import getTrigDecisionTool
from TrigDecisionTool.TrigDecisionToolConfig import getTrigDecisionTool, getRun3NavigationContainerFromInput, possible_keys
ca = getTrigDecisionTool(ConfigFlags)
tdt = ca.getPrimary()
......@@ -29,33 +29,40 @@ def getTrigNavSlimmingMTConfig(ConfigFlags):
# NOTE: This setup does not yet support derivations
inputCollection = getRun3NavigationContainerFromInput(ConfigFlags)
if ConfigFlags.Output.doWriteESD or rec.doWriteESD():
esdSlim = CompFactory.TrigNavSlimmingMTAlg('TrigNavSlimmingMTAlg_ESD')
esdSlim.TrigDecisionTool = tdt
esdSlim.OutputCollection = "HLTNav_Summary_ESDSlimmed"
esdSlim.PrimaryInputCollection = "HLTNav_Summary"
esdSlim.PrimaryInputCollection = inputCollection
esdSlim.AllOutputContainers = possible_keys
esdSlim.KeepFailedBranched = True
esdSlim.KeepOnlyFinalFeatures = False
ca.addEventAlgo(esdSlim)
log.info("Producing ESD Slimmed Trigger Navigation Collections")
log.info("Producing ESD Slimmed Trigger Navigation Collection. Reading {} and writing {}".format(esdSlim.PrimaryInputCollection, esdSlim.OutputCollection))
if esdSlim.OutputCollection not in possible_keys:
log.error("Producing a collection {} which is not listed in 'possible_keys'! Add this here too.".format(esdSlim.OutputCollection))
if ConfigFlags.Output.doWriteAOD or rec.doWriteAOD():
aodSlim = CompFactory.TrigNavSlimmingMTAlg('TrigNavSlimmingMTAlg_ESD')
aodSlim = CompFactory.TrigNavSlimmingMTAlg('TrigNavSlimmingMTAlg_AOD')
aodSlim.TrigDecisionTool = tdt
aodSlim.OutputCollection = "HLTNav_Summary_AODSlimmed"
if rec.readESD():
aodSlim.PrimaryInputCollection = "HLTNav_Summary_ESDSlimmed"
else:
aodSlim.PrimaryInputCollection = "HLTNav_Summary"
aodSlim.PrimaryInputCollection = inputCollection
aodSlim.AllOutputContainers = possible_keys
#
if ConfigFlags.Trigger.AODEDMSet == "AODFULL" or ConfigFlags.Trigger.AODEDMSet == "AODRun3_LARGE":
aodSlim.KeepFailedBranched = True
aodSlim.KeepOnlyFinalFeatures = False
log.info("Producing AOD LARGE Slimmed Trigger Navigation Collections")
log.info("Producing AOD LARGE Slimmed Trigger Navigation Collection. Reading {} and writing {}".format(aodSlim.PrimaryInputCollection, aodSlim.OutputCollection))
else:
aodSlim.KeepFailedBranched = False
aodSlim.KeepOnlyFinalFeatures = True
log.info("Producing AOD SMALL Slimmed Trigger Navigation Collections")
log.info("Producing AOD SMALL Trigger Navigation Collection. Reading {} and writing {}".format(aodSlim.PrimaryInputCollection, aodSlim.OutputCollection))
ca.addEventAlgo(aodSlim)
return ca
\ No newline at end of file
if aodSlim.OutputCollection not in possible_keys:
log.error("Producing a collection {} which is not listed in 'possible_keys'! Add this here too.".format(esdSlim.OutputCollection))
return ca
......@@ -22,6 +22,21 @@ StatusCode TrigNavSlimmingMTAlg::initialize() {
ATH_CHECK( m_trigDec.retrieve() );
m_trigDec->ExperimentalAndExpertMethods()->enable();
}
for (const std::string& output : m_allOutputContainers) {
if (output == m_primaryInputCollection.key()) {
continue; // We do want to search for failed nodes in the primary input (it may already be merged)
}
// We don't want to search for failed nodes in other possible summary keys, we might read in the
// summary collection from another running instance (e.g. an AODSlim alg reading in the output of
// ESDSlim in a RAWtoALL job).
m_allOutputContainersSet.insert(output);
}
m_allOutputContainersSet.insert(m_outputCollection.key());
msg() << MSG::INFO << "Initialized. Will *not* inspect the following SG Keys: ";
for (const std::string& key : m_allOutputContainersSet) {
msg() << key << " ";
}
msg() << endmsg;
return StatusCode::SUCCESS;
}
......@@ -57,7 +72,7 @@ StatusCode TrigNavSlimmingMTAlg::execute(const EventContext& ctx) const {
// Note: We use the "internal" version of this call such that we maintain our own cache,
// as we may need to call this more than once if keepFailedBranches is ture
TrigCompositeUtils::recursiveGetDecisionsInternal(terminusNode,
nullptr,
nullptr, // 'Coming from' is nullptr for the first call of the recursive function
transientNavGraph,
fullyExploredFrom,
chainIDs,
......@@ -70,15 +85,15 @@ StatusCode TrigNavSlimmingMTAlg::execute(const EventContext& ctx) const {
// These branches do not connect to the terminusNode, so we have to go hunting them explicitly.
// We need to pass in the evtStore as these nodes can be spread out over numerous collections.
// Like with the terminus node, we can restrict this search to only nodes which were rejected by certain chains.
// We also want to restrict the search to exclue the output collections of any other TrigNavSlimminMTAlg instances.\\s
if (m_keepFailedBranches) {
std::vector<const Decision*> rejectedNodes = getRejectedDecisionNodes(&*evtStore(), chainIDs);
std::vector<const Decision*> rejectedNodes = TrigCompositeUtils::getRejectedDecisionNodes(&*evtStore(), chainIDs, m_allOutputContainersSet);
for (const Decision* rejectedNode : rejectedNodes) {
// We do *not* enfoce that a member of chainIDs must be present in the starting node (rejectedNode)
// specifically because we know that at least one of chainIDs was _rejected_ here, but is active in the rejected
// node's seeds.
TrigCompositeUtils::recursiveGetDecisionsInternal(rejectedNode,
nullptr,
nullptr, // 'Coming from' is nullptr for the first call of the recursive function
transientNavGraph,
fullyExploredFrom,
chainIDs,
......
......@@ -61,15 +61,21 @@ private:
this, "NodesToDrop", {"F"},
"Any nodes within the set of names will be dropped as part of the thinning and the navigation re-wired around them"};
Gaudi::Property<std::vector<std::string>> m_allOutputContainers{
this, "AllOutputContainers", {},
"List of SG keys of all possible output containers at differing verbosity. Used to stop different instances of the alg interfering with each other."};
Gaudi::Property<std::vector<std::string>> m_chainsFilter{
this, "ChainsFilter", {},
"Optional list of HLT chains. If provided, only navigation data corresponding to these chains will be kept. "
"Matching multiple chains via regular expressions is supported."};
ToolHandle<Trig::TrigDecisionTool> m_trigDec{
PublicToolHandle<Trig::TrigDecisionTool> m_trigDec{
this, "TrigDecisionTool", "Trig::TrigDecisionTool/TrigDecisionTool",
"Trigger Decision Tool, used to apply the ChainsFilter"};
std::set<std::string> m_allOutputContainersSet; //!< Processed form of m_allOutputContainers
/**
* @brief Convert the ChainsFilter into the set of chain-IDd and chain-leg-IDs which comprises
* all of the DecisionIDs used by the members of the ChainsFilter.
......
......@@ -44,3 +44,4 @@ endif()
# Install files from the package:
atlas_install_joboptions( share/*.py )
atlas_install_scripts( scripts/*.py )
atlas_install_python_modules( python/*.py POST_BUILD_CMD ${ATLAS_FLAKE8} )
#
# Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
#
def getxAODConfigSvc(ConfigFlags):
from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
from AthenaConfiguration.ComponentFactory import CompFactory
acc = ComponentAccumulator()
cfgsvc = CompFactory.TrigConf.xAODConfigSvc('xAODConfigSvc')
# We serve in-file metadata where possible. If it does not exist (e.g. RAWtoESD, RAWtoALL), then it is obtained from the Conditions and Detector stores
cfgsvc.UseInFileMetadata = ConfigFlags.Trigger.InputContainsConfigMetadata
acc.addService(cfgsvc, primary=True)
if ConfigFlags.Trigger.InputContainsConfigMetadata:
from AthenaServices.MetaDataSvcConfig import MetaDataSvcCfg
acc.merge(MetaDataSvcCfg(ConfigFlags))
return acc
......@@ -236,7 +236,10 @@ namespace TrigCompositeUtils {
return nullptr;
}
std::vector<const Decision*> getRejectedDecisionNodes(asg::EventStoreType* eventStore, const DecisionIDContainer ids) {
std::vector<const Decision*> getRejectedDecisionNodes(asg::EventStoreType* eventStore,
const DecisionIDContainer ids,
const std::set<std::string>& keysToIgnore) {
std::vector<const Decision*> output;
// The list of containers we need to read can change on a file-by-file basis (it depends on the SMK)
// Hence we query SG for all collections rather than maintain a large and ever changing ReadHandleKeyArray
......@@ -264,6 +267,9 @@ namespace TrigCompositeUtils {
if ( key.find("HLTNav_") != 0 ) {
continue; // Only concerned about the decision containers which make up the navigation, they have name prefix of HLTNav
}
if (keysToIgnore.count(key) == 1) {
continue; // Have been asked to not explore this SG container
}
SG::ReadHandle<DecisionContainer> containerRH(key);
if (!containerRH.isValid()) {
throw std::runtime_error("Unable to retrieve " + key + " from event store.");
......
......@@ -287,9 +287,12 @@ namespace TrigCompositeUtils {
* @brief Query all DecisionCollections in the event store, locate all Decision nodes in the graph where an object failed selection for a given chain.
* @param[in] eventStore Pointer to event store within current event context
* @param[in] ids IDs of chain (if multi-leg chain, include all legs) to located failed decision nodes for. Passing an empty set returns all decision nodes which failed at least one chain.
* @param[in] keysToIgnore Set of SG keys of containers which should not be explored by getRejectedDecisionNodes.
* @return Vector of Decision nodes whose attached feature failed the trigger chain logic for chain with DecisionID id
**/
std::vector<const Decision*> getRejectedDecisionNodes(asg::EventStoreType* eventStore, const DecisionIDContainer ids = {});
std::vector<const Decision*> getRejectedDecisionNodes(asg::EventStoreType* eventStore,
const DecisionIDContainer ids = {},
const std::set<std::string>& keysToIgnore = std::set<std::string>());
/**
......
......@@ -152,8 +152,13 @@ TriggerHLTListRun3 = [
('TrigRoiDescriptorCollection#HLT_TAURoI', 'BS ESD AODFULL AODSLIM', 'Steer'),
('TrigRoiDescriptorCollection#HLT_FSRoI', 'BS ESD AODFULL AODSLIM', 'Steer'),
('xAOD::TrigCompositeContainer#HLTNav_Summary_OnlineSlimmed', 'BS ESD AODFULL AODSLIM', 'Steer'),
('xAOD::TrigCompositeAuxContainer#HLTNav_Summary_OnlineSlimmedAux.', 'BS ESD AODFULL AODSLIM', 'Steer'),
# xAOD::TrigCompositeContainer#HLTNav_Summary is now transient-only
('xAOD::TrigCompositeContainer#HLTNav_Summary_OnlineSlimmed', 'BS ESD AODFULL AODSLIM', 'Steer'), # TODO: set to BS-only after slimmed versions are validated
('xAOD::TrigCompositeAuxContainer#HLTNav_Summary_OnlineSlimmedAux.', 'BS ESD AODFULL AODSLIM', 'Steer'), # TODO: set to BS-only after slimmed versions are validated
('xAOD::TrigCompositeContainer#HLTNav_Summary_ESDSlimmed', 'ESD', 'Steer'),
('xAOD::TrigCompositeAuxContainer#HLTNav_Summary_ESDSlimmedAux.', 'ESD', 'Steer'),
('xAOD::TrigCompositeContainer#HLTNav_Summary_AODSlimmed', 'AODFULL AODSLIM', 'Steer'),
('xAOD::TrigCompositeAuxContainer#HLTNav_Summary_AODSlimmedAux.', 'AODFULL AODSLIM', 'Steer'),
('xAOD::TrigCompositeContainer#HLT_TrigCostContainer', 'CostMonDS ESD', 'Steer'),
('xAOD::TrigCompositeAuxContainer#HLT_TrigCostContainerAux.alg.store.view.thread.thash.slot.roi.start.stop', 'CostMonDS ESD', 'Steer'),
......
......@@ -390,7 +390,7 @@ class HLTTriggerResultGetter(Configured):
log.info("AOD list is subset of ESD list - good.")
def _addSlimming(stream, edm):
def _addSlimmingRun2(stream, edm):
from TrigNavTools.TrigNavToolsConfig import navigationThinningSvc
edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
......@@ -407,13 +407,30 @@ class HLTTriggerResultGetter(Configured):
del edmlist
if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteAOD():
_addSlimming('StreamAOD', _TriggerESDList ) #Use ESD item list also for AOD!
log.info("configured navigation slimming for AOD output")
if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteESD():
_addSlimming('StreamESD', _TriggerESDList )
log.info("configured navigation slimming for ESD output")
if ConfigFlags.Trigger.EDMVersion == 1 or ConfigFlags.Trigger.EDMVersion == 2:
# Run 1, 2 slimming
if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteAOD():
_addSlimmingRun2('StreamAOD', _TriggerESDList ) #Use ESD item list also for AOD!
log.info("configured navigation slimming for AOD output")
if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteESD():
_addSlimmingRun2('StreamESD', _TriggerESDList )
log.info("configured navigation slimming for ESD output")
if ConfigFlags.Trigger.EDMVersion >= 3:
# Change in the future to 'if EDMVersion >= 3 or doEDMVersionConversion:'
# Run 3 slimming
if ConfigFlags.Trigger.doNavigationSlimming:
from TrigNavSlimmingMT.TrigNavSlimmingMTConfig import getTrigNavSlimmingMTConfig
from AthenaCommon.Configurable import Configurable
Configurable.configurableRun3Behavior += 1
from AthenaConfiguration.ComponentAccumulator import appendCAtoAthena
appendCAtoAthena( getTrigNavSlimmingMTConfig(ConfigFlags) )
Configurable.configurableRun3Behavior -= 1
else:
log.info("doNavigationSlimming is False, won't schedule run 3 navigation slimming")
objKeyStore.addManyTypesStreamESD( _TriggerESDList )
objKeyStore.addManyTypesStreamAOD( _TriggerAODList )
......
......@@ -105,6 +105,10 @@ def createTriggerFlags():
flags.addFlag('Trigger.EDMVersion', lambda prevFlags: EDMVersion(prevFlags))
flags.addFlag('Trigger.doEDMVersionConversion', False)
flags.addFlag('Trigger.doConfigVersionConversion', True)
# Flag to control the scheduling of offline Run 3 trigger navigation slimming in RAWtoESD, RAWtoAOD, AODtoDAOD or RAWtoALL transforms.
flags.addFlag('Trigger.doNavigationSlimming', False) # Defaulting to False until validated (July 2021)
# enables additional algorithms colecting MC truth infrmation (this is only used by IDso maybe we need Trigger.ID.doTruth only?)
flags.addFlag('Trigger.doTruth', False)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment