From 4fc0a73d6cd9a5b06f840d786ca2b69eaea86071 Mon Sep 17 00:00:00 2001 From: James Richard Catmore <james.catmore@cern.ch> Date: Mon, 9 May 2022 14:10:31 +0200 Subject: [PATCH] Trying to fix squashing issues via force push. No idea what will happen.... --- .../DerivationFrameworkPhys/python/PHYS.py | 67 +++++++----- .../python/PhysCommonConfig.py | 22 +++- .../python/TriggerListsHelper.py | 87 +++++++++++++++ .../python/TriggerMatchingCommonConfig.py | 101 ++++++++++++++++++ .../python/TriggerMatchingToolConfig.py | 29 +++++ .../python/TrigNavSlimmingMTConfig.py | 13 +++ 6 files changed, 289 insertions(+), 30 deletions(-) create mode 100644 PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerListsHelper.py create mode 100644 PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerMatchingCommonConfig.py create mode 100644 PhysicsAnalysis/DerivationFramework/DerivationFrameworkTrigger/python/TriggerMatchingToolConfig.py diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYS.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYS.py index cbe99019fb7b..a81c5fab7f4c 100644 --- a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYS.py +++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PHYS.py @@ -18,7 +18,7 @@ def PHYSKernelCfg(ConfigFlags, name='PHYSKernel', **kwargs): # Common augmentations from DerivationFrameworkPhys.PhysCommonConfig import PhysCommonAugmentationsCfg - acc.merge(PhysCommonAugmentationsCfg(ConfigFlags)) + acc.merge(PhysCommonAugmentationsCfg(ConfigFlags, TriggerListsHelper = kwargs['TriggerListsHelper'])) # Thinning tools... from DerivationFrameworkInDet.InDetToolsConfig import TrackParticleThinningCfg, MuonTrackParticleThinningCfg, TauTrackParticleThinningCfg, DiTauTrackParticleThinningCfg @@ -88,8 +88,17 @@ def PHYSKernelCfg(ConfigFlags, name='PHYSKernel', **kwargs): def PHYSCfg(ConfigFlags): acc = ComponentAccumulator() - acc.merge(PHYSKernelCfg(ConfigFlags, name="PHYSKernel", StreamName = 'StreamDAOD_PHYS')) - + + # Get the lists of triggers needed for trigger matching. + # This is needed at this scope (for the slimming) and further down in the config chain + # for actually configuring the matching, so we create it here and pass it down + # TODO: this should ideally be called higher up to avoid it being run multiple times in a train + from DerivationFrameworkPhys.TriggerListsHelper import TriggerListsHelper + PHYSTriggerListsHelper = TriggerListsHelper() + + # Common augmentations + acc.merge(PHYSKernelCfg(ConfigFlags, name="PHYSKernel", StreamName = 'StreamDAOD_PHYS', TriggerListsHelper = PHYSTriggerListsHelper)) + # ============================ # Define contents of the format # ============================= @@ -127,18 +136,6 @@ def PHYSCfg(ConfigFlags): PHYSSlimmingHelper.StaticContent = StaticContent - # Trigger content - PHYSSlimmingHelper.IncludeTriggerNavigation = False - PHYSSlimmingHelper.IncludeJetTriggerContent = False - PHYSSlimmingHelper.IncludeMuonTriggerContent = False - PHYSSlimmingHelper.IncludeEGammaTriggerContent = False - PHYSSlimmingHelper.IncludeJetTauEtMissTriggerContent = False - PHYSSlimmingHelper.IncludeTauTriggerContent = False - PHYSSlimmingHelper.IncludeEtMissTriggerContent = False - PHYSSlimmingHelper.IncludeBJetTriggerContent = False - PHYSSlimmingHelper.IncludeBPhysTriggerContent = False - PHYSSlimmingHelper.IncludeMinBiasTriggerContent = False - # Truth containers if ConfigFlags.Input.isMC: PHYSSlimmingHelper.AppendToDictionary = {'TruthEvents':'xAOD::TruthEventContainer','TruthEventsAux':'xAOD::TruthEventAuxContainer', @@ -179,20 +176,38 @@ def PHYSCfg(ConfigFlags): "TauJets.dRmax.etOverPtLeadTrk", "HLT_xAOD__TrigMissingETContainer_TrigEFMissingET.ex.ey", "HLT_xAOD__TrigMissingETContainer_TrigEFMissingET_mht.ex.ey"] - + + # Trigger content + PHYSSlimmingHelper.IncludeTriggerNavigation = False + PHYSSlimmingHelper.IncludeJetTriggerContent = False + PHYSSlimmingHelper.IncludeMuonTriggerContent = False + PHYSSlimmingHelper.IncludeEGammaTriggerContent = False + PHYSSlimmingHelper.IncludeJetTauEtMissTriggerContent = False + PHYSSlimmingHelper.IncludeTauTriggerContent = False + PHYSSlimmingHelper.IncludeEtMissTriggerContent = False + PHYSSlimmingHelper.IncludeBJetTriggerContent = False + PHYSSlimmingHelper.IncludeBPhysTriggerContent = False + PHYSSlimmingHelper.IncludeMinBiasTriggerContent = False + + # Trigger matching + # Run 2 + if ConfigFlags.Trigger.EDMVersion == 2: + from DerivationFrameworkPhys.TriggerMatchingCommonConfig import AddRun2TriggerMatchingToSlimmingHelper + AddRun2TriggerMatchingToSlimmingHelper(SlimmingHelper = PHYSSlimmingHelper, + OutputContainerPrefix = "PhysCommonTau", + TriggerList = PHYSTriggerListsHelper.Run2TriggerNamesTau) + AddRun2TriggerMatchingToSlimmingHelper(SlimmingHelper = PHYSSlimmingHelper, + OutputContainerPrefix = "PhysCommonNoTau", + TriggerList = PHYSTriggerListsHelper.Run2TriggerNamesNoTau) + # Run 3 + if ConfigFlags.Trigger.EDMVersion == 3: + from TrigNavSlimmingMT.TrigNavSlimmingMTConfig import AddRun3TrigNavSlimmingCollectionsToSlimmingHelper + AddRun3TrigNavSlimmingCollectionsToSlimmingHelper(PHYSSlimmingHelper) + + # Output stream PHYSItemList = PHYSSlimmingHelper.GetItemList() acc.merge(OutputStreamCfg(ConfigFlags, "DAOD_PHYS", ItemList=PHYSItemList, AcceptAlgs=["PHYSKernel"])) return acc - -# Add trigger matching -# Run 2 -#PhysCommonTrigger.trigmatching_helper_notau.add_to_slimming(PHYSSlimmingHelper) -#PhysCommonTrigger.trigmatching_helper_tau.add_to_slimming(PHYSSlimmingHelper) -# Run 3 -#from TrigNavSlimmingMT.TrigNavSlimmingMTConfig import AddRun3TrigNavSlimmingCollectionsToEDM -#AddRun3TrigNavSlimmingCollectionsToEDM(PHYSStream) - - diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PhysCommonConfig.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PhysCommonConfig.py index 82b39ce6f0a1..e9c1113e5377 100644 --- a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PhysCommonConfig.py +++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/PhysCommonConfig.py @@ -54,7 +54,7 @@ def PhysCommonAugmentationsCfg(ConfigFlags,**kwargs): - # InDet, Muon and Egamma common augmentations + # InDet, Muon, Egamma and jet common augmentations from DerivationFrameworkInDet.InDetCommonConfig import InDetCommonCfg from DerivationFrameworkMuons.MuonsCommonConfig import MuonsCommonCfg from DerivationFrameworkEGamma.EGammaCommonConfig import EGammaCommonCfg @@ -71,12 +71,26 @@ def PhysCommonAugmentationsCfg(ConfigFlags,**kwargs): acc.merge(MuonsCommonCfg(ConfigFlags)) acc.merge(EGammaCommonCfg(ConfigFlags)) acc.merge(JetCommonCfg(ConfigFlags)) + # Trigger matching + from DerivationFrameworkPhys.TriggerMatchingCommonConfig import TriggerMatchingCommonRun2Cfg + from DerivationFrameworkPhys.TriggerMatchingCommonConfig import TriggerMatchingCommonRun3Cfg + # requires some wrangling due to the difference between run 2 and 3 + triggerListsHelper = kwargs['TriggerListsHelper'] + if ConfigFlags.Trigger.EDMVersion == 2: + acc.merge(TriggerMatchingCommonRun2Cfg(ConfigFlags, + name = "PhysCommonTrigMatchNoTau", + OutputContainerPrefix = "PhysCommonNoTau", + TriggerList = triggerListsHelper.Run2TriggerNamesNoTau)) + acc.merge(TriggerMatchingCommonRun2Cfg(ConfigFlags, + name = "PhysCommonTrigMatchTau", + OutputContainerPrefix = "PhysCommonTau", + TriggerList = triggerListsHelper.Run2TriggerNamesTau, + DRThreshold = 0.2)) + if ConfigFlags.Trigger.EDMVersion == 3: + acc.merge(TriggerMatchingCommonRun3Cfg(ConfigFlags, TriggerList = triggerListsHelper.Run3TriggerNames)) - # Jets... # Tau... # Flavour tagging... - # Truth... - # Trigger return acc diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerListsHelper.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerListsHelper.py new file mode 100644 index 000000000000..a0c6676bf744 --- /dev/null +++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerListsHelper.py @@ -0,0 +1,87 @@ +# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration +# TriggerListsHelper: helper class which retrieves the full set of triggers needed for +# trigger matching in the DAODs and can then return them when needed. + +from AthenaConfiguration.AllConfigFlags import ConfigFlags +from TriggerMenuMT.TriggerAPI.TriggerAPI import TriggerAPI +from TriggerMenuMT.TriggerAPI.TriggerEnums import TriggerPeriod, TriggerType +from PathResolver import PathResolver +import re + +def read_trig_list_file(fname): + """Read a text file containing a list of triggers + Returns the list of triggers held in the file + """ + triggers = [] + with open(PathResolver.FindCalibFile(fname)) as fp: + for line in fp: + line = line.strip() + if line == "" or line.startswith("#"): + continue + triggers.append(line) + return triggers + + +class TriggerListsHelper: + def __init__(self): + self.Run2TriggerNamesTau = [] + self.Run2TriggerNamesNoTau = [] + self.Run3TriggerNames = [] + self.GetTriggerLists() + + def GetTriggerLists(self): + #==================================================================== + # TRIGGER CONTENT + #==================================================================== + ## See https://twiki.cern.ch/twiki/bin/view/Atlas/TriggerAPI + ## Get single and multi mu, e, photon triggers + ## Jet, tau, multi-object triggers not available in the matching code + allperiods = TriggerPeriod.y2015 | TriggerPeriod.y2016 | TriggerPeriod.y2017 | TriggerPeriod.y2018 | TriggerPeriod.future2e34 + trig_el = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.el, livefraction=0.8) + trig_mu = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.mu, livefraction=0.8) + trig_g = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.g, livefraction=0.8) + trig_tau = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.tau, livefraction=0.8) + ## Add cross-triggers for some sets + trig_em = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.el, additionalTriggerType=TriggerType.mu, livefraction=0.8) + trig_et = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.el, additionalTriggerType=TriggerType.tau, livefraction=0.8) + trig_mt = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.mu, additionalTriggerType=TriggerType.tau, livefraction=0.8) + # Note that this seems to pick up both isolated and non-isolated triggers already, so no need for extra grabs + trig_txe = TriggerAPI.getLowestUnprescaledAnyPeriod(allperiods, triggerType=TriggerType.tau, additionalTriggerType=TriggerType.xe, livefraction=0.8) + + extra_notau = read_trig_list_file("DerivationFrameworkPhys/run2ExtraMatchingTriggers.txt") + extra_tau = read_trig_list_file("DerivationFrameworkPhys/run2ExtraMatchingTauTriggers.txt") + + ## Merge and remove duplicates + trigger_names_full_notau = list(set(trig_el+trig_mu+trig_g+trig_em+trig_et+trig_mt+extra_notau)) + trigger_names_full_tau = list(set(trig_tau+trig_txe+extra_tau)) + # + ## Now reduce the list... + trigger_names_notau = [] + trigger_names_tau = [] + from AthenaConfiguration.AutoConfigFlags import GetFileMD + + if ConfigFlags.Trigger.EDMVersion == 3: + r_tau = re.compile("HLT_.*tau.*") + r_notau = re.compile("HLT_[1-9]*(e|mu|g).*") + for chain_name in GetFileMD(ConfigFlags.Input.Files)['TriggerMenu']['HLTChains']: + result_tau = r_tau.match(chain_name) + result_notau = r_notau.match(chain_name) + if result_tau is not None: trigger_names_tau.append(chain_name) + if result_notau is not None: trigger_names_notau.append(chain_name) + trigger_names_all = set.union(set(trigger_names_notau), set(trigger_names_tau)) + trigger_names_all = list(trigger_names_all) + trigger_names_notau = set(trigger_names_notau) - set(trigger_names_tau) + trigger_names_notau = list(trigger_names_notau) + self.Run3TriggerNames = trigger_names_all + else: + # Note: ['TriggerMenu']['HLTChains'] python access is maintained for compatibility with Run 2 MC + # POOL inputs (containing xAOD::TriggerMenu). + for chain_name in GetFileMD(ConfigFlags.Input.Files)['TriggerMenu']['HLTChains']: + if chain_name in trigger_names_full_notau: trigger_names_notau.append(chain_name) + if chain_name in trigger_names_full_tau: trigger_names_tau.append(chain_name) + self.Run2TriggerNamesNoTau = trigger_names_notau + self.Run2TriggerNamesTau = trigger_names_tau + + return + + diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerMatchingCommonConfig.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerMatchingCommonConfig.py new file mode 100644 index 000000000000..9838fbcf37d4 --- /dev/null +++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkPhys/python/TriggerMatchingCommonConfig.py @@ -0,0 +1,101 @@ +# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration + +#==================================================================== +# TriggerMatchingCommonConfig.py +# This defines the common trigger matching shared by PHYS and PHYSLITE +# Using it avoids name clashes when running in trains +# In principle it can also be used by other formats who want to take +# advantage of PHYS/PHYSLITE containers +# Component accumulator version +#==================================================================== +from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator +from AthenaConfiguration.ComponentFactory import CompFactory +from PathResolver import PathResolver + +def read_trig_list_file(fname): + """Read a text file containing a list of triggers + + Returns the list of triggers held in the file + """ + triggers = [] + with open(PathResolver.FindCalibFile(fname)) as fp: + for line in fp: + line = line.strip() + if line == "" or line.startswith("#"): + continue + triggers.append(line) + return triggers + + +def AddRun2TriggerMatchingToSlimmingHelper(**kwargs): + """Adds the trigger matching info to the slimming helper""" + + slimmingHelper = kwargs['SlimmingHelper'] + triggerList = kwargs['TriggerList'] + containerPrefix = kwargs['OutputContainerPrefix'] + + outputContainers = [ + "{0}{1}".format(containerPrefix, chain).replace(".", "_") + for chain in triggerList + ] + + slimmingHelper.AllVariables += outputContainers + for cont in outputContainers: + slimmingHelper.AppendToDictionary.update( + { + cont: "xAOD::TrigCompositeContainer", + cont + "Aux": "AOD::AuxContainerBase!", + } + ) + + +def TriggerMatchingCommonRun2Cfg(ConfigFlags, name, **kwargs): + """Configure the common trigger matching for run 2 DAODs""" + + if ConfigFlags.Trigger.EDMVersion != 2: + raise ValueError('This configuration can only be used for Run 2 trigger navigation') + + triggerList = kwargs['TriggerList'] + outputContainerPrefix = kwargs['OutputContainerPrefix'] + kwargs.setdefault('DRThreshold', None) + + acc = ComponentAccumulator() + + # Create trigger matching decorations + from DerivationFrameworkTrigger.TriggerMatchingToolConfig import TriggerMatchingToolCfg + if kwargs['DRThreshold'] is None: + PhysCommonTriggerMatchingTool = acc.getPrimaryAndMerge(TriggerMatchingToolCfg( + ConfigFlags, + name=name, + ChainNames = triggerList, + OutputContainerPrefix = outputContainerPrefix)) + else: + PhysCommonTriggerMatchingTool = acc.getPrimaryAndMerge(TriggerMatchingToolCfg( + ConfigFlags, + name=name, + ChainNames = triggerList, + OutputContainerPrefix = outputContainerPrefix, + DRThreshold = kwargs['DRThreshold'])) + CommonAugmentation = CompFactory.DerivationFramework.CommonAugmentation + acc.addEventAlgo(CommonAugmentation(f"{outputContainerPrefix}TriggerMatchingKernel", + AugmentationTools=[PhysCommonTriggerMatchingTool])) + return(acc) + + +def TriggerMatchingCommonRun3Cfg(ConfigFlags, **kwargs): + """Configure the common trigger matching for run 3 DAODs""" + + if ConfigFlags.Trigger.EDMVersion != 3: + raise ValueError('This configuration can only be used for Run 3 trigger navigation') + + triggerList = kwargs['TriggerList'] + + acc = ComponentAccumulator() + + # Run 3 trigger navigation slimming proposal for in-DAOD trigger matching. + from TrigNavSlimmingMT.TrigNavSlimmingMTConfig import TrigNavSlimmingMTDerivationCfg + acc.merge(TrigNavSlimmingMTDerivationCfg(ConfigFlags,chainsFilter=triggerList)) + + return(acc) + + diff --git a/PhysicsAnalysis/DerivationFramework/DerivationFrameworkTrigger/python/TriggerMatchingToolConfig.py b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkTrigger/python/TriggerMatchingToolConfig.py new file mode 100644 index 000000000000..280c21e7457c --- /dev/null +++ b/PhysicsAnalysis/DerivationFramework/DerivationFrameworkTrigger/python/TriggerMatchingToolConfig.py @@ -0,0 +1,29 @@ +# Copyright (C) 2002-2022 CERN for the benefit of the ATLAS collaboration + +from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator +from AthenaConfiguration.ComponentFactory import CompFactory + +def R3IParticleRetrievalToolCfg(ConfigFlags): + """Configure the IParticle retrieval tool, depends on R1/R2 or R3 trigger""" + acc = ComponentAccumulator() + from TrigDecisionTool.TrigDecisionToolConfig import TrigDecisionToolCfg + tdt = acc.getPrimaryAndMerge(TrigDecisionToolCfg(ConfigFlags)) + if ConfigFlags.Trigger.EDMVersion == 3: + IParticleRetrievalTool = CompFactory.Trig.R3IParticleRetrievalTool + else: + IParticleRetrievalTool = CompFactory.Trig.IParticleRetrievalTool + acc.addPublicTool(IParticleRetrievalTool("OnlineParticleTool", TrigDecisionTool = tdt), + primary = True) + return(acc) + +def TriggerMatchingToolCfg(ConfigFlags, name, **kwargs): + """Config fragment for the trigger matching tool used in DAOD production""" + acc = ComponentAccumulator() + OnlineParticleTool = acc.getPrimaryAndMerge(R3IParticleRetrievalToolCfg(ConfigFlags)) + kwargs['OnlineParticleTool'] = OnlineParticleTool + TriggerMatchingTool = CompFactory.DerivationFramework.TriggerMatchingTool + acc.addPublicTool(TriggerMatchingTool(name, **kwargs), + primary = True) + return(acc) + + diff --git a/Trigger/TrigAnalysis/TrigNavSlimmingMT/python/TrigNavSlimmingMTConfig.py b/Trigger/TrigAnalysis/TrigNavSlimmingMT/python/TrigNavSlimmingMTConfig.py index 530b25d50021..c6ede36716a5 100644 --- a/Trigger/TrigAnalysis/TrigNavSlimmingMT/python/TrigNavSlimmingMTConfig.py +++ b/Trigger/TrigAnalysis/TrigNavSlimmingMT/python/TrigNavSlimmingMTConfig.py @@ -93,6 +93,19 @@ def AddRun3TrigNavSlimmingCollectionsToEDM(stream): # stream.AddItem("TrigRoiDescriptorCollection#HLTNav_RepackedROIs") +# Same as the above but adds the branches to the slimming helper. +# This is the component accumulator version +def AddRun3TrigNavSlimmingCollectionsToSlimmingHelper(slimmingHelper): + slimmingHelper.AppendToDictionary = {'HLTNav_Summary_DAODSlimmed':'xAOD::TrigCompositeContainer','HLTNav_Summary_DAODSlimmedAux':'xAOD::TrigCompositeAuxContainer', + 'HLTNav_RepackedFeatures_Particle':'xAOD::ParticleContainer','HLTNav_RepackedFeatures_ParticleAux':'xAOD::ParticleAuxContainer', + 'HLTNav_RepackedFeatures_MET':'xAOD::TrigMissingETContainer','HLTNav_RepackedFeatures_METAux':'xAOD::TrigMissingETAuxContainer', + 'HLTNav_RepackedROIs':'TrigRoiDescriptorCollection'} + + slimmingHelper.AllVariables += ['HLTNav_Summary_DAODSlimmed', + 'HLTNav_RepackedFeatures_Particle', + 'HLTNav_RepackedFeatures_MET', + 'HLTNav_RepackedROIs'] + # # Return an ComponentAccumulator which configures trigger navigation slimming during # RAW->ALL, RAW->ESD or ESD->AOD (and MC equivalents) -- GitLab