Commit 35471352 authored by Rafal Bielski's avatar Rafal Bielski Committed by Frank Winklmeier
Browse files

Move HLT output configuration from full_menu.py to TriggerConfig

parent 03c910fb
......@@ -63,6 +63,10 @@ def _createCfgFlags():
acf.addFlag('Output.doESD', False) # produce ESD containers
acf.addFlag('Output.doWriteRDO', False) # write out RDO file
acf.addFlag('Output.doWriteESD', False) # write out ESD file
acf.addFlag('Output.doWriteAOD', False) # write out AOD file
acf.addFlag('Output.doWriteBS', False) # write out RDO ByteStream file
acf.addFlag('Output.EVNTFileName','myEVNT.pool.root')
acf.addFlag('Output.HITSFileName','myHITS.pool.root')
......
......@@ -81,6 +81,9 @@ if hasattr(runArgs,"inputRDO_TRIGFile"):
DQMonFlags.doHLTMon = False
DQMonFlags.useTrigger = False
DQMonFlags.doLVL1CaloMon = False
# Auto-configure EDM decoding version
from TriggerJobOpts.HLTTriggerResultGetter import EDMDecodingVersion
EDMDecodingVersion()
from AthenaCommon.KeyStore import CfgItemList, CfgKeyStore
from RecExConfig.ObjKeyStore import objKeyStore
if TriggerFlags.doMT():
......
......@@ -42,14 +42,16 @@ TrigDecisionMakerMT::~TrigDecisionMakerMT() {}
StatusCode TrigDecisionMakerMT::initialize()
{
bool resultObjectIsUsed = true;
if (!m_bitsMakerTool.empty()) {
ATH_MSG_INFO("TrigDecisionMakerMT is setting up for MC to use the TriggerBitsMakerTool");
ATH_CHECK( m_bitsMakerTool.retrieve() );
resultObjectIsUsed = false;
} else {
ATH_MSG_INFO("TrigDecisionMakerMT is setting up for Data to use the HLTResultMT. "
"If this job is for MC, make sure that the BitsMakerTool property is set instead.");
ATH_CHECK( m_hltResultKeyIn.initialize() );
}
ATH_CHECK( m_hltResultKeyIn.initialize(resultObjectIsUsed) ); // If false, this removes the ReadHandle
ATH_CHECK( m_ROIBResultKeyIn.initialize() );
ATH_CHECK( m_EventInfoKeyIn.initialize() );
......
......@@ -46,7 +46,7 @@ def HLTResultMTMakerCfg(name="HLTResultMTMaker"):
return m
def TriggerEDMSerialiserToolCfg(name):
def TriggerEDMSerialiserToolCfg(name="TriggerEDMSerialiserTool"):
from TriggerMenuMT.HLTMenuConfig.Menu.EventBuildingInfo import getFullHLTResultID
# Configuration helper methods
......@@ -93,3 +93,21 @@ def TriggerEDMSerialiserToolCfg(name):
serialiser.TPTool = tpTool
return serialiser
def StreamTagMakerToolCfg(name="StreamTagMakerTool"):
from TrigOutputHandlingConf import StreamTagMakerTool
stmaker = StreamTagMakerTool(name)
# Extra configuration may come here
return stmaker
def TriggerBitsMakerToolCfg(name="TriggerBitsMakerTool"):
from TrigOutputHandlingConf import TriggerBitsMakerTool
from TriggerJobOpts.TriggerFlags import TriggerFlags
bitsmaker = TriggerBitsMakerTool(name)
# Extra configuration may come here
return bitsmaker
......@@ -10,7 +10,7 @@ ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
ex.job_options = 'TrigUpgradeTest/full_menu.py'
ex.input = 'data'
ex.args = '-c "doEmptyMenu=True;BFieldAutoConfig=False;doWriteESD=False"'
ex.args = '-c "doEmptyMenu=True;BFieldAutoConfig=False;"'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
test = Test.Test()
......
......@@ -13,7 +13,7 @@ ex.input = 'data'
ex.forks = 2
ex.threads = 2
ex.concurrent_events = 2
ex.args = '-c "doEmptyMenu=True;BFieldAutoConfig=False;doWriteESD=False"'
ex.args = '-c "doEmptyMenu=True;BFieldAutoConfig=False;"'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
test = Test.Test()
......
......@@ -10,7 +10,7 @@ ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
ex.job_options = 'TrigUpgradeTest/full_menu.py'
ex.input = 'data'
ex.args = '-c "doWriteESD=False" --ros2rob /cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/ATLASros2rob2018-r22format.py'
ex.args = '--ros2rob /cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/ATLASros2rob2018-r22format.py'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
ros2json = CheckSteps.InputDependentStep("RosRateToJson")
......
......@@ -10,7 +10,6 @@ ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
ex.job_options = 'TrigUpgradeTest/full_menu.py'
ex.input = 'data'
ex.args = '-c "doWriteESD=False"'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
test = Test.Test()
......
......@@ -10,7 +10,7 @@ writeBS = ExecStep.ExecStep("WriteBS")
writeBS.type = 'athenaHLT'
writeBS.job_options = 'TrigUpgradeTest/full_menu.py'
writeBS.input = 'data'
writeBS.args = '-c "doWriteESD=False" -o output'
writeBS.args = '-o output'
writeBS.perfmon = False # perfmon with athenaHLT doesn't work at the moment
decodeBS = ExecStep.ExecStep("DecodeBS")
......
......@@ -19,7 +19,7 @@ writeBS = ExecStep.ExecStep("WriteBS")
writeBS.type = 'athenaHLT'
writeBS.job_options = 'TrigUpgradeTest/full_menu.py'
writeBS.input = 'data'
writeBS.args = '-c "doWriteESD=False" -o output'
writeBS.args = '-o output'
writeBS.perfmon = False # perfmon with athenaHLT doesn't work at the moment
decodeBS = ExecStep.ExecStep("DecodeBS")
......
......@@ -13,7 +13,6 @@ ex.input = 'data'
ex.forks = 8
ex.threads = 1
ex.concurrent_events = 1
ex.args = '-c "doWriteESD=False"'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
test = Test.Test()
......
......@@ -19,7 +19,6 @@ ex = ExecStep.ExecStep()
ex.type = 'athenaHLT'
ex.job_options = 'TrigUpgradeTest/full_menu.py'
ex.input = 'data'
ex.args = '-c "doWriteESD=False"'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
test = Test.Test()
......
......@@ -13,7 +13,7 @@ ex.input = 'data'
ex.forks = 2
ex.threads = 2
ex.concurrent_events = 2
ex.args = '--stdcmalloc -c "doEmptyMenu=True;doJetSlice=True;doWriteESD=False"'
ex.args = '--stdcmalloc -c "doEmptyMenu=True;doJetSlice=True;"'
ex.perfmon = False # perfmon with athenaHLT doesn't work at the moment
test = Test.Test()
......
......@@ -2,69 +2,16 @@
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
#
##################################################################
# Pre-setup
##################################################################
from AthenaCommon.Logging import logging
__log = logging.getLogger('full_menu')
# Set the menu name
from TriggerJobOpts.TriggerFlags import TriggerFlags
TriggerFlags.triggerMenuSetup = "LS2_v1"
##################################################################
# Functions creating EDM list for output writing
# TODO: Move to TriggerJobOpts
##################################################################
def getEDMListForPOOL(key, decObj):
# Get the list from TriggerEDM
from TrigEDMConfig.TriggerEDM import getTriggerEDMList
from TrigEDMConfig.TriggerEDMRun3 import persistent
TriggerFlags.EDMDecodingVersion = 3
TriggerFlags.ESDEDMSet = key
edmList = getTriggerEDMList(TriggerFlags.ESDEDMSet(), TriggerFlags.EDMDecodingVersion())
# Build the output ItemList
ItemList = []
for edmType, edmKeys in edmList.iteritems():
ItemList.extend([edmType+'#'+collKey for collKey in edmKeys])
# Add decision containers (navigation)
for item in decObj:
ItemList.append( 'xAOD::TrigCompositeContainer#{:s}'.format(item) )
ItemList.append( 'xAOD::TrigCompositeAuxContainer#{:s}Aux.-'.format(item) )
return list(set(ItemList))
def getEDMDictForBS(decObj):
from TriggerMenuMT.HLTMenuConfig.Menu import EventBuildingInfo
from TrigEDMConfig.TriggerEDM import getTriggerEDMList
from TrigEDMConfig.TriggerEDMRun3 import persistent
TriggerFlags.EDMDecodingVersion = 3
TriggerFlags.ESDEDMSet = 'BS'
ItemListDict = {}
ItemModuleDict = {}
for key in ['BS'] + EventBuildingInfo.getAllDataScoutingIdentifiers():
edmList = getTriggerEDMList(key, TriggerFlags.EDMDecodingVersion())
moduleId = EventBuildingInfo.getFullHLTResultID() if key=='BS' else EventBuildingInfo.getDataScoutingResultID(key)
for edmType, edmKeys in edmList.iteritems():
for collKey in edmKeys:
item = persistent(edmType)+'#'+collKey
if item not in ItemModuleDict.keys():
ItemModuleDict[item] = [moduleId]
else:
ItemModuleDict[item].append(moduleId)
# Add decision containers (navigation)
for item in decObj:
typeName = 'xAOD::TrigCompositeContainer#{:s}'.format(item)
typeNameAux = 'xAOD::TrigCompositeAuxContainer#{:s}Aux.-'.format(item)
if typeName not in ItemModuleDict.keys():
ItemModuleDict[typeName] = [EventBuildingInfo.getFullHLTResultID()]
if typeNameAux not in ItemModuleDict.keys():
ItemModuleDict[typeNameAux] = [EventBuildingInfo.getFullHLTResultID()]
return ItemModuleDict
##################################################################
# Include the common configuration
##################################################################
......@@ -73,12 +20,9 @@ include("TrigUpgradeTest/testHLT_MT.py")
##################################################################
# Generate the menu
##################################################################
from TriggerMenuMT.HLTMenuConfig.Menu.GenerateMenuMT import GenerateMenuMT
menu = GenerateMenuMT()
def signaturesToGenerate():
TriggerFlags.Slices_all_setOff()
for sig in opt.enabledSignatures:
......@@ -87,44 +31,21 @@ def signaturesToGenerate():
menu.overwriteSignaturesWith(signaturesToGenerate)
allChainConfigs = menu.generateMT()
##########################################
# Some debug
# Debugging for view cross-dependencies
##########################################
from AthenaCommon.AlgSequence import dumpSequence, AthSequencer
dumpSequence(topSequence)
if opt.reverseViews:
from TriggerJobOpts.TriggerConfig import collectViewMakers
viewMakers = collectViewMakers( topSequence )
for alg in viewMakers:
alg.ReverseViewsDebug = True
##########################################
# Output configuration
# Configure trigger output using parts of the NewJO configuration
# in a somewhat hacky way
##########################################
# Configure BS result only in the following three cases
# 1) Running in partition
# 2) Running with athenaHLT
# 3) Running with athena and saving BS output
import os
onlineWriteBS = os.getenv('TDAQ_PARTITION') is not None # athenaHLT also sets this
offlineWriteBS = False # TODO: cover configuration of BS writing in athena
configureBSResult = onlineWriteBS or offlineWriteBS
if onlineWriteBS and opt.doWriteESD:
__log.error("ESD writing configured in online mode")
from TriggerJobOpts.TriggerConfig import collectHypos, collectFilters, collectDecisionObjects, triggerOutputCfg
from AthenaCommon.CFElements import findAlgorithm,findSubSequence
if opt.doWriteESD:
from RecExConfig.RecFlags import rec
rec.doESD=True
rec.doWriteESD=True
import DecisionHandling
for a in findSubSequence(topSequence, "HLTAllSteps").getChildren():
if isinstance(a, DecisionHandling.DecisionHandlingConf.TriggerSummaryAlg):
a.OutputLevel = DEBUG
# this part uses parts from the NewJO configuration, it is very hacky for the moment
from TriggerJobOpts.TriggerConfig import collectHypos, collectFilters, collectDecisionObjects, triggerOutputStreamCfg
hypos = collectHypos(findSubSequence(topSequence, "HLTAllSteps"))
filters = collectFilters(findSubSequence(topSequence, "HLTAllSteps"))
......@@ -140,52 +61,20 @@ else:
__log.warning("Failed to find L1Decoder, cannot determine Decision names for output configuration")
decObj = []
# Configure ESD writing
if opt.doWriteESD:
import AthenaPoolCnvSvc.WriteAthenaPool
from OutputStreamAthenaPool.OutputStreamAthenaPool import createOutputStream
StreamESD=createOutputStream("StreamESD","myESD.pool.root",True)
StreamESD.ItemList = getEDMListForPOOL('ESD', decObj)
# find DecisionSummaryMakerAlg
summaryMakerAlg = findAlgorithm(topSequence, "DecisionSummaryMakerAlg")
if not summaryMakerAlg:
__log.warning("Failed to find DecisionSummaryMakerAlg")
from AthenaConfiguration.AllConfigFlags import ConfigFlags
from AthenaCommon.Configurable import Configurable
Configurable.configurableRun3Behavior+=1
acc, edmSet = triggerOutputCfg(ConfigFlags, decObj, summaryMakerAlg)
Configurable.configurableRun3Behavior-=1
acc.appendToGlobals()
##########################################
# HLT result configuration
# TODO: Move this to TriggerJobOpts and call from here
# Print top sequence for debugging
##########################################
if configureBSResult:
from TrigOutputHandling.TrigOutputHandlingConfig import TriggerEDMSerialiserToolCfg
from TrigOutputHandling.TrigOutputHandlingConf import StreamTagMakerTool, TriggerBitsMakerTool
# Tool serialising EDM objects to fill the HLT result
serialiser = TriggerEDMSerialiserToolCfg('Serialiser')
ItemDict = getEDMDictForBS(decObj)
for item, modules in ItemDict.iteritems():
__log.debug('adding to serialiser list: %s, modules: %s', item, modules)
serialiser.addCollection(item, modules)
# Tool adding stream tags to HLT result
stmaker = StreamTagMakerTool()
# Map decisions producing PEBInfo from DecisionSummaryMakerAlg.FinalStepDecisions to StreamTagMakerTool.PEBDecisionKeys
import AthenaCommon.AlgSequence as acas
summaryMakerAlg = [s for s in acas.iter_algseq(topSequence) if s.getName() == "DecisionSummaryMakerAlg"][0]
chainToDecisionKeyDict = summaryMakerAlg.getProperties()['FinalStepDecisions']
stmaker.PEBDecisionKeys = []
for chain, decisionKey in chainToDecisionKeyDict.iteritems():
if 'PEBInfoWriter' in decisionKey:
__log.debug('Chain %s produces decision %s with PEBInfo', chain, decisionKey)
stmaker.PEBDecisionKeys.append(decisionKey)
bitsmaker = TriggerBitsMakerTool()
# Configure the HLT result maker to use the above tools
from AthenaCommon.AppMgr import ServiceMgr as svcMgr
hltResultMaker = svcMgr.HltEventLoopMgr.ResultMaker
hltResultMaker.MakerTools = [stmaker, serialiser, bitsmaker]
# Debugging for view cross-dependencies
if opt.reverseViews:
from TriggerJobOpts.TriggerConfig import collectViewMakers
viewMakers = collectViewMakers( topSequence )
for alg in viewMakers:
alg.ReverseViewsDebug = True
from AthenaCommon.AlgSequence import dumpSequence, AthSequencer
dumpSequence(topSequence)
......@@ -5,11 +5,6 @@
from AthenaCommon.Logging import logging
__log = logging.getLogger('full_menu')
# import flags
from RecExConfig.RecFlags import rec
rec.doESD=True
rec.doWriteESD=True
createHLTMenuExternally=True # menu will be build up explicitly here
include("TrigUpgradeTest/testHLT_MT.py")
......@@ -289,63 +284,48 @@ from TriggerMenuMT.HLTMenuConfig.Menu.HLTCFConfig import makeHLTTree
from TriggerMenuMT.HLTMenuConfig.Menu.TriggerConfigHLT import TriggerConfigHLT
makeHLTTree( triggerConfigHLT=TriggerConfigHLT )
##########################################
# Some debug
# Configure trigger output using parts of the NewJO configuration
# in a somewhat hacky way - copy-pasted from full_menu.py
##########################################
from AthenaCommon.AlgSequence import dumpSequence, AthSequencer
dumpSequence(topSequence)
import DecisionHandling
from TriggerJobOpts.TriggerConfig import collectHypos, collectFilters, collectDecisionObjects, triggerOutputCfg
from AthenaCommon.CFElements import findAlgorithm,findSubSequence
for a in findSubSequence(topSequence, "HLTAllSteps").getChildren():
if isinstance(a, DecisionHandling.DecisionHandlingConf.TriggerSummaryAlg):
a.OutputLevel = DEBUG
# this part uses parts from the NewJO configuration, it is very hacky for the moment
from TriggerJobOpts.TriggerConfig import collectHypos, collectFilters, collectDecisionObjects, triggerOutputStreamCfg
hypos = collectHypos(findSubSequence(topSequence, "HLTAllSteps"))
filters = collectFilters(findSubSequence(topSequence, "HLTAllSteps"))
# try to find L1Decoder
l1decoder = findAlgorithm(topSequence,'L1Decoder')
l1decoder.OutputLevel=DEBUG
if not l1decoder:
l1decoder = findAlgorithm(topSequence,'L1EmulationTest')
if l1decoder:
decObj = collectDecisionObjects( hypos, filters, l1decoder )
__log.debug("Decision Objects to export to ESD [hack method - should be replaced with triggerRunCfg()]")
__log.debug("Decision Objects to write to output [hack method - should be replaced with triggerRunCfg()]")
__log.debug(decObj)
from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
ItemList = [ 'xAOD::TrigCompositeContainer#{}'.format(d) for d in decObj ]
ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(d) for d in decObj ]
ItemList += [ k[0] for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" not in k[0] ]
ItemList += [ k[0] for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" in k[0] ]
ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(k[0].split("#")[1]) for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" in k[0] ]
ItemList += [ "xAOD::EventInfo#EventInfo" ]
ItemList = list(set(ItemList))
else:
ItemList = []
__log.warning("Failed to find L1Decoder, cannot determine Decision names for output configuration")
decObj = []
# find DecisionSummaryMakerAlg
summaryMakerAlg = findAlgorithm(topSequence, "DecisionSummaryMakerAlg")
if not summaryMakerAlg:
__log.warning("Failed to find DecisionSummaryMakerAlg")
import AthenaPoolCnvSvc.WriteAthenaPool
from OutputStreamAthenaPool.OutputStreamAthenaPool import createOutputStream
StreamESD=createOutputStream("StreamESD","myESD.pool.root",True)
StreamESD.ItemList = ItemList
from AthenaConfiguration.AllConfigFlags import ConfigFlags
from AthenaCommon.Configurable import Configurable
Configurable.configurableRun3Behavior+=1
acc, edmSet = triggerOutputCfg(ConfigFlags, decObj, summaryMakerAlg)
Configurable.configurableRun3Behavior-=1
acc.appendToGlobals()
##########################################
# Print top sequence for debugging
##########################################
from AthenaCommon.AlgSequence import dumpSequence, AthSequencer
dumpSequence(topSequence)
##########################################
# Write menu JSON
##########################################
HLTTop = findSubSequence(topSequence, "HLTTop")
from TriggerMenuMT.HLTMenuConfig.Menu.HLTMenuJSON import generateJSON
generateJSON()
......@@ -20,7 +20,11 @@ flags.Detector.GeometryMDT = True
flags.Detector.GeometryTGC = True
flags.Detector.GeometryCSC = True
flags.Detector.GeometryRPC = True
flags.Trigger.writeBS=True # switches on HLTResultMT creation
# Output configuration - currently testing offline workflow
flags.Trigger.writeBS = False
flags.Output.doWriteRDO = True
flags.Output.RDOFileName = 'RDO_TRIG.pool.root'
flags.Trigger.CostMonitoring.doCostMonitoring = True
......
......@@ -30,7 +30,8 @@ class opt:
doDBConfig = None # dump trigger configuration
trigBase = None # file name for trigger config dump
enableCostD3PD = False # enable cost monitoring
doWriteESD = True # Write out an ESD?
doWriteRDOTrigger = False # Write out RDOTrigger?
doWriteBS = True # Write out BS?
doL1Unpacking = True # decode L1 data in input file if True, else setup emulation
doL1Sim = False # (re)run L1 simulation
isOnline = False # isOnline flag (TEMPORARY HACK, should be True by default)
......@@ -278,8 +279,6 @@ rec.doESD = False
rec.doAOD = False
rec.doTruth = False
TriggerFlags.writeBS = True
#-------------------------------------------------------------
# Apply modifiers
#-------------------------------------------------------------
......@@ -482,14 +481,32 @@ if svcMgr.MessageSvc.OutputLevel<INFO:
jobproperties.print_JobProperties('tree&value')
print(svcMgr)
#-------------------------------------------------------------
# Use parts of NewJO
#-------------------------------------------------------------
from AthenaCommon.Configurable import Configurable
Configurable.configurableRun3Behavior=True
Configurable.configurableRun3Behavior+=1
from TriggerJobOpts.TriggerConfig import triggerIDCCacheCreatorsCfg
from AthenaConfiguration.AllConfigFlags import ConfigFlags
# Output flags
isPartition = len(ConfigFlags.Trigger.Online.partitionName) > 0
if opt.doWriteRDOTrigger:
if isPartition:
log.error('Cannot use doWriteRDOTrigger in athenaHLT or partition')
theApp.exit(1)
rec.doWriteRDO = False # RecExCommon flag
ConfigFlags.Output.doWriteRDO = True # new JO flag
ConfigFlags.Output.RDOFileName = 'RDO_TRIG.pool.root' # new JO flag
if opt.doWriteBS:
rec.doWriteBS = True # RecExCommon flag
TriggerFlags.writeBS = True # RecExCommon flag
ConfigFlags.Output.doWriteBS = True # new JO flag
ConfigFlags.Trigger.writeBS = True # new JO flag
# ID Cache Creators
ConfigFlags.lock()
triggerIDCCacheCreatorsCfg(ConfigFlags).appendToGlobals()
Configurable.configurableRun3Behavior=False
Configurable.configurableRun3Behavior-=1
#-------------------------------------------------------------
# Non-ComponentAccumulator Cost Monitoring
......
......@@ -9,7 +9,7 @@ export EVENTS=10
export THREADS=2
export SLOTS=2
export JOBOPTION="TrigUpgradeTest/full_menu.py"
export EXTRA="isOnline=True;doEmptyMenu=True"
export EXTRA="isOnline=True;doEmptyMenu=True;doWriteBS=False;doWriteRDOTrigger=True;"
# This test doesn't produce any output useful for RegTest, so do a dummy reference comparison
export REGTESTEXP="leaving with code"
......
......@@ -10,7 +10,7 @@ export THREADS=1
export SLOTS=1
export INPUT="ttbar"
export JOBOPTION="TrigUpgradeTest/full_menu.py"
export EXTRA="doL1Sim=True;doEmptyMenu=True"
export EXTRA="doL1Sim=True;doEmptyMenu=True;doWriteBS=False;doWriteRDOTrigger=True;"
# This test doesn't produce any output useful for RegTest, so do a dummy reference comparison
export REGTESTEXP="leaving with code"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment