Skip to content

GitLab

Commit 6cc4809e authored by Joerg Stelzer's avatar Joerg Stelzer
Browse files

Fix for failing nightly tests 

Moves HLT menu generation into testHLT_MT.py
parent 42a6afa4
Hide whitespace changes
Inline Side-by-side
Trigger/TrigConfiguration/TrigConfigSvc/python/TrigConfigSvcCfg.py
View file @ 6cc4809e
......@@ -37,16 +37,15 @@ def generateL1Menu( flags=None ):
menuName = TriggerFlags.triggerMenuSetup() if flags is None else flags.Trigger.triggerMenuSetup
log.info("Generating L1 menu %s", menuName)
from TriggerMenuMT.L1.L1MenuConfig import L1MenuConfig
l1cfg = L1MenuConfig( menuName = menuName, outputFile = fileName )
log.info("Writing L1 menu file %s", fileName)
l1cfg.writeJSON()
# also still generate the old L1 xml
return fileName
l1cfg = L1MenuConfig( menuName = menuName )
outfile = l1cfg.writeJSON(outputFile = fileName)
if outfile is not None:
log.info("Wrote L1 menu file %s", outfile)
return outfile
# configuration of L1ConfigSvc
@memoize
def getL1ConfigSvc( flags = None ):
log = logging.getLogger('TrigConfigSvcCfg')
from AthenaCommon.Logging import log
......@@ -55,23 +54,32 @@ def getL1ConfigSvc( flags = None ):
from TrigConfigSvc.TrigConfigSvcConfig import findFileInXMLPATH
from AthenaCommon.AppMgr import theApp
# generate menu file
generateL1Menu( flags=flags )
generatedFile = generateL1Menu( flags=flags )
# configure config svc
l1ConfigSvc = TrigConf__LVL1ConfigSvc( "LVL1ConfigSvc" )
l1XMLFile = findFileInXMLPATH( TriggerFlags.inputLVL1configFile() if flags is None else flags.Trigger.LVL1ConfigFile )
l1ConfigSvc.ConfigSource = "XML"
l1XMLFile = findFileInXMLPATH( TriggerFlags.inputLVL1configFile() if flags is None else flags.Trigger.LVL1ConfigFile )
l1ConfigSvc.XMLMenuFile = l1XMLFile
l1ConfigSvc.InputType = "file"
l1JsonFileName = getL1MenuFileName( flags )
l1ConfigSvc.JsonFileName = l1JsonFileName
theApp.CreateSvc += [ "TrigConf::LVL1ConfigSvc/LVL1ConfigSvc" ]
log.info( "Configured LVL1ConfigSvc with run 2 style input file : %s", l1XMLFile )
log.info( "Configured LVL1ConfigSvc with InputType='file' and JsonFileName=%s", l1JsonFileName )
if generatedFile is None:
l1ConfigSvc.InputType = "none"
l1ConfigSvc.JsonFileName = ""
log.info( "Configured LVL1ConfigSvc with InputType='none'" )
else:
l1ConfigSvc.InputType = "file"
l1JsonFileName = generatedFile
l1ConfigSvc.JsonFileName = l1JsonFileName
log.info( "Configured LVL1ConfigSvc with InputType='file' and JsonFileName=%s", l1JsonFileName )
theApp.CreateSvc += [ "TrigConf::LVL1ConfigSvc/LVL1ConfigSvc" ]
return l1ConfigSvc
# configuration of HLTConfigSvc
@memoize
def getHLTConfigSvc( flags = None ):
log = logging.getLogger('TrigConfigSvcCfg')
from AthenaCommon.Logging import log
......@@ -89,14 +97,14 @@ def getHLTConfigSvc( flags = None ):
log.info( "Configured HLTConfigSvc with InputType='file' and JsonFileName=%s", hltJsonFileName )
return hltConfigSvc
# provide both services in new JO
# provide L1 config service in new JO
def L1ConfigSvcCfg( flags ):
from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
acc = ComponentAccumulator()
acc.addService( getL1ConfigSvc( flags ) )
return acc
# provide both services in new JO
# provide HLT config service in new JO
def HLTConfigSvcCfg( flags ):
from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
acc = ComponentAccumulator()
......
Trigger/TrigValidation/TrigUpgradeTest/share/full_menu.py
View file @ 6cc4809e
......@@ -10,26 +10,6 @@ TriggerFlags.triggerMenuSetup = "LS2_v1"
include("TrigUpgradeTest/testHLT_MT.py")
##################################################################
# Generate the menu
##################################################################
from TriggerMenuMT.HLTMenuConfig.Menu.GenerateMenuMT import GenerateMenuMT
menu = GenerateMenuMT()
def signaturesToGenerate():
TriggerFlags.Slices_all_setOff()
for sig in opt.enabledSignatures:
eval(sig)
menu.overwriteSignaturesWith(signaturesToGenerate)
allChainConfigs = menu.generateMT()
##########################################
# Some debug
##########################################
......
Trigger/TrigValidation/TrigUpgradeTest/share/full_menu_l1only.py 0 → 100644
View file @ 6cc4809e
#
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
#
from AthenaCommon.Logging import logging
__log = logging.getLogger('full_menu_l1only')
from TriggerJobOpts.TriggerFlags import TriggerFlags
TriggerFlags.triggerMenuSetup = "LS2_v1"
include("TrigUpgradeTest/testL1_MT.py")
##########################################
# Some debug
##########################################
from AthenaCommon.AlgSequence import dumpSequence, AthSequencer
dumpSequence(topSequence)
##########################################
# Output configuration
##########################################
# Configure BS result only in the following three cases
# 1) Running in partition
# 2) Running with athenaHLT
# 3) Running with athena and saving BS output
import os
onlineWriteBS = os.getenv('TDAQ_PARTITION') is not None # athenaHLT also sets this
offlineWriteBS = False # TODO: cover configuration of BS writing in athena
configureBSResult = onlineWriteBS or offlineWriteBS
if onlineWriteBS and opt.doWriteESD:
__log.error("ESD writing configured in online mode")
from AthenaCommon.CFElements import findAlgorithm,findSubSequence
if opt.doWriteESD:
from RecExConfig.RecFlags import rec
rec.doESD=True
rec.doWriteESD=True
# this part uses parts from the NewJO configuration, it is very hacky for the moment
# try to find L1Decoder
log.info("No L1Decoder")
decObj = []
# Get the EDM list
from TrigEDMConfig.TriggerEDM import getTriggerEDMList
from TrigEDMConfig.TriggerEDMRun3 import persistent
TriggerFlags.EDMDecodingVersion = 3
TriggerFlags.ESDEDMSet = 'BS' if configureBSResult else 'ESD'
edmList = getTriggerEDMList(TriggerFlags.ESDEDMSet(), TriggerFlags.EDMDecodingVersion())
# Build the output ItemList
ItemList = []
for edmType, edmKeys in edmList.iteritems():
edmType = persistent(edmType) if configureBSResult else edmType
for key in edmKeys:
ItemList.append(edmType+'#'+key)
if not configureBSResult:
# Do not serialise EventInfo into BS, as it is already represented by the BS event header
ItemList.append("xAOD::EventInfo#EventInfo")
ItemList.append("xAOD::EventAuxInfo#EventInfoAux.")
for item in decObj:
typeName = 'xAOD::TrigCompositeContainer#{}'.format(item)
typeNameAux = 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(item)
ItemList.append(typeName)
# Temporary workaround for the case when item from decObj is explicitly in TriggerEDM list
# to avoid adding it twice - with and without dynamic variables
if typeNameAux+'-' not in ItemList:
ItemList.append(typeNameAux)
ItemList = list(set(ItemList))
# Configure ESD writing
if opt.doWriteESD:
import AthenaPoolCnvSvc.WriteAthenaPool
from OutputStreamAthenaPool.OutputStreamAthenaPool import createOutputStream
StreamESD=createOutputStream("StreamESD","myESD.pool.root",True)
StreamESD.ItemList = ItemList
Trigger/TrigValidation/TrigUpgradeTest/share/testHLT_MT.py
View file @ 6cc4809e
......@@ -18,7 +18,7 @@
#
class opt:
setupForMC = None # force MC setup
setLVL1XML = 'TriggerMenuMT/LVL1config_LS2_v1.xml' # 'TriggerMenu/LVL1config_Physics_pp_v7.xml' # default for legacy
setMenu = 'LS2_v1'
setDetDescr = None # force geometry tag
setGlobalTag = None # force global conditions tag
useCONDBR2 = True # if False, use run-1 conditions DB
......@@ -35,6 +35,7 @@ class opt:
doL1Sim = False # (re)run L1 simulation
isOnline = False # isOnline flag (TEMPORARY HACK, should be True by default)
doEmptyMenu = False # Disable all chains, except those re-enabled by specific slices
endJobAfterGenerate = False # Finish job after menu generation
#Individual slice flags
doEgammaSlice = True
doMuonSlice = True
......@@ -49,6 +50,8 @@ class opt:
reverseViews = False
enabledSignatures = []
disabledSignatures = []
#
################################################################################
from TriggerJobOpts.TriggerFlags import TriggerFlags
......@@ -367,15 +370,16 @@ elif globalflags.InputFormat.is_bytestream():
# ---------------------------------------------------------------
# Trigger config
# ---------------------------------------------------------------
TriggerFlags.triggerMenuSetup = opt.setMenu
TriggerFlags.readLVL1configFromXML = True
TriggerFlags.outputLVL1configFile = None
from TrigConfigSvc.TrigConfigSvcCfg import generateL1Menu
l1JsonFile = generateL1Menu()
from TrigConfigSvc.TrigConfigSvcCfg import getL1ConfigSvc, getHLTConfigSvc
from TrigConfigSvc.TrigConfigSvcCfg import getL1ConfigSvc
svcMgr += getL1ConfigSvc()
svcMgr += getHLTConfigSvc()
# ---------------------------------------------------------------
# Level 1 simulation
......@@ -403,6 +407,38 @@ if opt.doL1Unpacking:
topSequence += L1EmulationTest()
# ---------------------------------------------------------------
# HLT generation
# ---------------------------------------------------------------
from TriggerMenuMT.HLTMenuConfig.Menu.GenerateMenuMT import GenerateMenuMT
menu = GenerateMenuMT()
# define the function that enable the signatures
def signaturesToGenerate():
TriggerFlags.Slices_all_setOff()
for sig in opt.enabledSignatures:
eval(sig)
menu.overwriteSignaturesWith(signaturesToGenerate)
# generating the HLT structure requires
# the L1Decoder to be defined in the topSequence
menu.generateMT()
if endJobAfterGenerate:
import sys
sys.exit(0)
from TrigConfigSvc.TrigConfigSvcCfg import getHLTConfigSvc
svcMgr += getHLTConfigSvc()
# ---------------------------------------------------------------
# ID conditions
# ---------------------------------------------------------------
if TriggerFlags.doID:
from InDetTrigRecExample.InDetTrigFlags import InDetTrigFlags
InDetTrigFlags.doPixelClusterSplitting = False
......@@ -415,7 +451,7 @@ if TriggerFlags.doID:
# ---------------------------------------------------------------
# Monitoring
# ---------------------------------------------------------------
if not hasattr(svcMgr, 'THistSvc'):
if not hasattr(svcMgr, 'THistSvc'):
from GaudiSvc.GaudiSvcConf import THistSvc
svcMgr += THistSvc()
if hasattr(svcMgr.THistSvc, "Output"):
......
Trigger/TrigValidation/TrigUpgradeTest/share/testL1_MT.py 0 → 100644
View file @ 6cc4809e
# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
################################################################################
# TrigUpgradeTest/testHLT_MT.py
#
# Job options for trigger MT tests to be used as first include
# in signature specific job options. Example:
#
# include('TrigUpgradeTest/testL1_MT.py')
# ... your job options ...
#
# Several flags are supported on the command line to steer these job options.
# See below for a complete list of all flags and their default value. If used
# in athena.py the auto-configuration is used to setup most flags correctly.
#
# Additional "modifiers" can be specified by using
# -c "myModifier=True/False"
# Existing modifiers can be found in "TriggerRelease/python/Modifiers.py"
#
class opt:
setupForMC = None # force MC setup
setLVL1XML = 'TriggerMenuMT/LVL1config_LS2_v1.xml' # 'TriggerMenu/LVL1config_Physics_pp_v7.xml' # default for legacy
setDetDescr = None # force geometry tag
setGlobalTag = None # force global conditions tag
useCONDBR2 = True # if False, use run-1 conditions DB
condOverride = {} # overwrite conditions folder tags e.g. '{"Folder1":"Tag1", "Folder2":"Tag2"}'
doHLT = False # run HLT?
doID = True # TriggerFlags.doID
doCalo = True # TriggerFlags.doCalo
doMuon = True # TriggerFlags.doMuon
doDBConfig = None # dump trigger configuration
trigBase = None # file name for trigger config dump
enableCostD3PD = False # enable cost monitoring
doWriteESD = True # Write out an ESD?
doL1Unpacking = False # decode L1 data in input file if True, else setup emulation
doL1Sim = True # (re)run L1 simulation
isOnline = False # isOnline flag (TEMPORARY HACK, should be True by default)
doEmptyMenu = False # Disable all chains, except those re-enabled by specific slices
#Individual slice flags
doElectronSlice = False
doPhotonSlice = False
doMuonSlice = False
doJetSlice = False
doMETSlice = False
doBjetSlice = False
doTauSlice = False
doCombinedSlice = False
doBphysicsSlice = False
doStreamingSlice = False
reverseViews = False
enabledSignatures = []
disabledSignatures = []
################################################################################
from TriggerJobOpts.TriggerFlags import TriggerFlags
from AthenaCommon.AppMgr import theApp, ServiceMgr as svcMgr
from AthenaCommon.Logging import logging
log = logging.getLogger('testL1_MT.py')
#-------------------------------------------------------------
# Setup options
#-------------------------------------------------------------
log.info('Setup options:')
defaultOptions = [a for a in dir(opt) if not a.startswith('__')]
for option in defaultOptions:
if option in globals():
setattr(opt, option, globals()[option])
print(' %20s = %s' % (option, getattr(opt, option)))
else:
print(' %20s = (Default) %s' % (option, getattr(opt, option)))
import re
## sliceRe = re.compile("^do.*Slice")
## slices = [a for a in dir(opt) if sliceRe.match(a)]
## if opt.doEmptyMenu is True:
## log.info("Disabling all slices")
## for s in slices:
## if s in globals():
## log.info("re-enabling %s ", s)
## setattr(opt, s, globals()[s])
## else:
## setattr(opt, s, False)
## else:
## for s in slices:
## setattr(opt, s, True)
## opt.doTauSlice =False #Wait for ATR-17399
# Setting the TriggerFlags.XXXSlice to use in TriggerMenuMT
# This is temporary and will be re-worked for after M3.5
## for s in slices:
## signature = s[2:].replace('Slice', '')
## if 'Electron' in s or 'Photon' in s:
## signature = 'Egamma'
##
## if eval('opt.'+s) is True:
## enabledSig = 'TriggerFlags.'+signature+'Slice.setAll()'
## opt.enabledSignatures.append( enabledSig )
## else:
## disabledSig = 'TriggerFlags.'+signature+'Slice.setAll()'
## opt.disabledSignatures.append( disabledSig )
#-------------------------------------------------------------
# Setting Global Flags
#-------------------------------------------------------------
from AthenaCommon.GlobalFlags import globalflags
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
from AthenaCommon.BeamFlags import jobproperties
import TriggerRelease.Modifiers
# Auto-configuration for athena
if len(athenaCommonFlags.FilesInput())>0:
import PyUtils.AthFile as athFile
af = athFile.fopen(athenaCommonFlags.FilesInput()[0])
globalflags.InputFormat = 'bytestream' if af.fileinfos['file_type']=='bs' else 'pool'
globalflags.DataSource = 'data' if af.fileinfos['evt_type'][0]=='IS_DATA' else 'geant4'
if opt.setDetDescr is None:
opt.setDetDescr = af.fileinfos.get('geometry',None)
if opt.setGlobalTag is None:
opt.setGlobalTag = af.fileinfos.get('conditions_tag',None) or \
(TriggerFlags.OnlineCondTag() if opt.isOnline else 'CONDBR2-BLKPA-2018-13')
TriggerRelease.Modifiers._run_number = af.fileinfos['run_number'][0]
else: # athenaHLT
globalflags.InputFormat = 'bytestream'
globalflags.DataSource = 'data' if not opt.setupForMC else 'data'
if '_run_number' not in dir():
import PyUtils.AthFile as athFile
from AthenaCommon.AthenaCommonFlags import athenaCommonFlags
af = athFile.fopen(athenaCommonFlags.BSRDOInput()[0])
_run_number = af.run_number[0]
TriggerRelease.Modifiers._run_number = _run_number # noqa, set by athenaHLT
from RecExConfig.RecFlags import rec
rec.RunNumber =_run_number
del _run_number
# Set final Cond/Geo tag based on input file, command line or default
globalflags.DetDescrVersion = opt.setDetDescr or TriggerFlags.OnlineGeoTag()
globalflags.ConditionsTag = opt.setGlobalTag or TriggerFlags.OnlineCondTag()
# Other defaults
jobproperties.Beam.beamType = 'collisions'
jobproperties.Beam.bunchSpacing = 25
globalflags.DatabaseInstance='CONDBR2' if opt.useCONDBR2 else 'COMP200'
athenaCommonFlags.isOnline.set_Value_and_Lock(opt.isOnline)
log.info('Configured the following global flags:')
globalflags.print_JobProperties()
#-------------------------------------------------------------
# Transfer flags into TriggerFlags
#-------------------------------------------------------------
# To turn off HLT for athena running
TriggerFlags.doHLT = bool(opt.doHLT)
# To extract the Trigger configuration
TriggerFlags.Online.doDBConfig = bool(opt.doDBConfig)
if opt.trigBase is not None:
TriggerFlags.Online.doDBConfigBaseName = opt.trigBase
# Setup list of modifiers
# Common modifiers for MC and data
setModifiers = ['noLArCalibFolders',
'ForceMuonDataType',
'useNewRPCCabling',
'enableCostMonitoring',
#'enableCoherentPS',
'useOracle',
'enableHotIDMasking',
'openThresholdRPCCabling',
]
if globalflags.DataSource.is_geant4(): # MC modifiers
setModifiers += ['BFieldFromDCS']
else: # More data modifiers
setModifiers += ['allowCOOLUpdates',
'BFieldAutoConfig',
'useHLTMuonAlign',
#Check for beamspot quality flag
'UseBeamSpotFlagForBjet',
'UseParamFromDataForBjet',
#Use online luminosity
'useOnlineLumi',
#for running with real data
'DisableMdtT0Fit',
#Setup mufast tuning for data
'UseLUTFromDataForMufast',
'UseRPCTimeDelayFromDataForMufast',
#Set muComb/muIso Backextrapolator tuned for real data
'UseBackExtrapolatorDataForMuIso',
#Monitoring for L1 muon group
#'muCTPicheck',
#Monitoring L1Topo at ROB level
#'L1TopoCheck',
'forceTileRODMap',
]
TriggerFlags.doID = opt.doID
TriggerFlags.doMuon = opt.doMuon
TriggerFlags.doCalo = opt.doCalo
#-------------------------------------------------------------
# Modifiers
#-------------------------------------------------------------
modifierList=[]
from TrigConfigSvc.TrigConfMetaData import TrigConfMetaData
meta = TrigConfMetaData()
for mod in dir(TriggerRelease.Modifiers):
if not hasattr(getattr(TriggerRelease.Modifiers,mod),'preSetup'): continue
if mod in dir(): #allow turning on and off modifiers by variable of same name
if globals()[mod]:
if mod not in setModifiers:
setModifiers+=[mod]
else:
if mod in setModifiers: setModifiers.remove(mod)
if mod in setModifiers:
modifierList+=[getattr(TriggerRelease.Modifiers,mod)()]
meta.Modifiers += [mod] # store in trig conf meta data
setModifiers.remove(mod)
if setModifiers:
log.error('Unknown modifier(s): '+str(setModifiers))
# never include this
include.block("RecExCond/RecExCommon_flags.py")
#-------------------------------------------------------------
# Setting DetFlags
#-------------------------------------------------------------
if globalflags.InputFormat.is_bytestream():
TriggerFlags.doLVL1 = False
from AthenaCommon.DetFlags import DetFlags
if TriggerFlags.doLVL1():
DetFlags.detdescr.all_setOn()
if TriggerFlags.doID():
DetFlags.detdescr.ID_setOn()
DetFlags.makeRIO.ID_setOn()
else:
DetFlags.ID_setOff()
if TriggerFlags.doMuon():
DetFlags.detdescr.Muon_setOn()
DetFlags.makeRIO.all_setOn()
else:
DetFlags.Muon_setOff()
if TriggerFlags.doCalo():
DetFlags.detdescr.Calo_setOn()
from LArConditionsCommon.LArCondFlags import larCondFlags
larCondFlags.LoadElecCalib.set_Value_and_Lock(False)
else:
DetFlags.Calo_setOff()
DetFlags.Print()
# RecEx flags
from RecExConfig.RecFlags import rec
rec.doWriteESD = False
rec.doWriteAOD = False
rec.doWriteTAG = False
rec.doESD = False
rec.doAOD = False
rec.doTruth = False
TriggerFlags.writeBS = True
#-------------------------------------------------------------
# Apply modifiers
#-------------------------------------------------------------
for mod in modifierList:
mod.preSetup()
#--------------------------------------------------------------
# Conditions setup.
#--------------------------------------------------------------
from IOVDbSvc.CondDB import conddb #This import will also set up CondInputLoader
conddb.setGlobalTag(globalflags.ConditionsTag())
from AthenaCommon.AlgSequence import AlgSequence
topSequence = AlgSequence()
#--------------------------------------------------------------
# Thread-specific setup
#--------------------------------------------------------------
from AthenaCommon.ConcurrencyFlags import jobproperties
if jobproperties.ConcurrencyFlags.NumThreads() > 0:
from AthenaCommon.AlgScheduler import AlgScheduler
AlgScheduler.CheckDependencies( True )
AlgScheduler.ShowControlFlow( True )
AlgScheduler.ShowDataDependencies( True )
AlgScheduler.EnableVerboseViews( True )
#--------------------------------------------------------------
# Event Info setup
#--------------------------------------------------------------
# If no xAOD::EventInfo is found in a POOL file, schedule conversion from old EventInfo
if globalflags.InputFormat.is_pool():
from RecExConfig.ObjKeyStore import objKeyStore
from PyUtils.MetaReaderPeeker import convert_itemList
objKeyStore.addManyTypesInputFile(convert_itemList(layout='#join'))
if ( not objKeyStore.isInInput("xAOD::EventInfo") ) and ( not hasattr(topSequence, "xAODMaker::EventInfoCnvAlg") ):
from xAODEventInfoCnv.xAODEventInfoCnvAlgDefault import xAODEventInfoCnvAlgDefault
xAODEventInfoCnvAlgDefault