Allow adding extra trigger collections via preExec

Trigger/TriggerCommon/TrigEDMConfig/python/TriggerEDM.py

@@ -10,8 +10,8 @@
 
 from TrigEDMConfig.TriggerEDMRun1 import TriggerL2List,TriggerEFList,TriggerResultsRun1List
 from TrigEDMConfig.TriggerEDMRun2 import TriggerResultsList,TriggerLvl1List,TriggerIDTruth,TriggerHLTList,EDMDetails,EDMLibraries,TriggerL2EvolutionList,TriggerEFEvolutionList
-from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3, AllowedOutputFormats
-
+from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3, AllowedOutputFormats,addExtraCollectionsToEDMList
+from AthenaConfiguration.AllConfigFlags import ConfigFlags as flags
 from AthenaCommon.Logging import logging
 log = logging.getLogger('TriggerEDM')
 
@@ -22,6 +22,12 @@ import six
 #  For Run 3
 #
 #************************************************************
+# Let adding collections via preExec for detailed validation
+if flags.Trigger.ExtraEDMList:
+    log.info( "Adding extra collections to EDM (): {}".format(len(TriggerHLTListRun3), flags.Trigger.ExtraEDMList) )
+    addExtraCollectionsToEDMList(TriggerHLTListRun3, flags.Trigger.ExtraEDMList)
+    log.info( "Number of EDM items after adding extra collections: {}".format(len(TriggerHLTListRun3)) )
+
 
 def getTriggerEDMList(key, runVersion):
     """

Trigger/TriggerCommon/TrigEDMConfig/python/TriggerEDMRun3.py

@@ -534,3 +534,42 @@ def addHLTNavigationToEDMList(edmList, allDecisions, hypoDecisions):
         edmList.extend([
             (typeName,    HLTNavEDMTargets, 'Steer'),
             (typeNameAux, HLTNavEDMTargets, 'Steer')])
+
+def addExtraCollectionsToEDMList(edmList, extraList):
+    """
+    Extend edmList with extraList, keeping track whether a completely new
+    collection is being added, or a dynamic variable is added to an existing collection.
+    The format of extraList is the same as those of TriggerHLTListRun3.
+    """
+    existing_collections = [(c[0].split("#")[1]).split(".")[0] for c in edmList]
+    for item in extraList:
+        colname = (item[0].split("#")[1]).split(".")[0]
+        if colname not in existing_collections:
+            # a new collection is added
+            edmList.append(item)
+            __log.info("added new item to Trigger EDM: {}".format(item))
+        else:
+            if "Aux." in item[0]:
+                # probably an extra dynamic variable is added
+                # new variables to add:
+                dynVars = (item[0].split("#")[1]).split(".")[1:]
+                # find the index of the existing item
+                existing_item_nr = [i for i,s in enumerate(edmList) if colname == (s[0].split("#")[1]).split(".")[0]]
+                if len(existing_item_nr) != 1:
+                    __log.error("Found {} existing edm items corresponding to new item {}, but it must be exactly one!".format(len(existing_item_nr), item))
+                # merge lists of variables
+                existing_dynVars = (edmList[existing_item_nr[0]][0].split("#")[1]).split(".")[1:]
+                dynVars.extend(existing_dynVars)
+                # remove duplicates:
+                dynVars = list(dict.fromkeys(dynVars))
+                newVars = '.'.join(dynVars)
+                typename = item[0].split("#")[0]
+                __log.info("old item in Trigger EDM: {}".format(edmList[existing_item_nr[0]]))
+                targets = edmList[existing_item_nr[0]][1]
+                signature = edmList[existing_item_nr[0]][2]
+                edmList.pop(existing_item_nr[0])
+                edmList.insert(existing_item_nr[0] , (typename + "#" + colname + "." + newVars, targets, signature))
+                __log.info("updated item in Trigger EDM: {}".format(edmList[existing_item_nr[0]]))
+            else:
+                # asking to add some collection which is already in the list - do nothing
+                pass

Trigger/TriggerCommon/TriggerJobOpts/python/TriggerConfig.py

@@ -433,10 +433,17 @@ def triggerPOOLOutputCfg(flags, edmSet):
     return acc
 
 
-def triggerMergeViewsAndAddMissingEDMCfg( edmSet, hypos, viewMakers, decObj, decObjHypoOut ):
+def triggerMergeViewsAndAddMissingEDMCfg( flags, edmSet, hypos, viewMakers, decObj, decObjHypoOut ):
 
     HLTEDMCreatorAlg, HLTEDMCreator=CompFactory.getComps("HLTEDMCreatorAlg","HLTEDMCreator",)
-    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
+    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3, addExtraCollectionsToEDMList
+
+    __log.info( "Number of EDM items in triggerMergeViewsAndAddMissingEDMCfg: {}".format(len(TriggerHLTListRun3)) )
+    if flags.Trigger.ExtraEDMList:
+        __log.info( "Adding extra collections to EDM: {}".format(flags.Trigger.ExtraEDMList) )
+        addExtraCollectionsToEDMList(TriggerHLTListRun3, flags.Trigger.ExtraEDMList)
+        __log.info( "Number of EDM items after adding extra collections: {}".format(len(TriggerHLTListRun3)) )
+
 
     alg = HLTEDMCreatorAlg("EDMCreatorAlg")
 
@@ -577,14 +584,16 @@ def triggerRunCfg( flags, seqName = None, menu=None ):
 
     # Add HLT Navigation to EDM list
     from TrigEDMConfig import TriggerEDMRun3
+    __log.info( "Number of EDM items before adding navigation: {}".format(len(TriggerEDMRun3.TriggerHLTListRun3)) )
     TriggerEDMRun3.addHLTNavigationToEDMList(TriggerEDMRun3.TriggerHLTListRun3, decObj, decObjHypoOut)
+    __log.info( "Number of EDM items after adding navigation: {}".format(len(TriggerEDMRun3.TriggerHLTListRun3)) )
 
     # Configure output writing
     outputAcc, edmSet = triggerOutputCfg( flags, summaryAlg )
     acc.merge( outputAcc )
 
     if edmSet:
-        mergingAlg = triggerMergeViewsAndAddMissingEDMCfg( [edmSet] , hypos, viewMakers, decObj, decObjHypoOut )
+        mergingAlg = triggerMergeViewsAndAddMissingEDMCfg( flags, [edmSet] , hypos, viewMakers, decObj, decObjHypoOut )
         acc.addEventAlgo( mergingAlg, sequenceName="HLTFinalizeSeq" )
 
     return acc

Trigger/TriggerCommon/TriggerJobOpts/python/TriggerConfigFlags.py

@@ -154,6 +154,9 @@ def createTriggerFlags():
     # list of objects to be written to ESD
     flags.addFlag('Trigger.ESDEDMSet', 'ESD')
 
+    # to allow stroing extra EDM items via preExec
+    flags.addFlag('Trigger.ExtraEDMList', [])
+
     # tag to be used for condutions used by HLT code
     flags.addFlag('Trigger.OnlineCondTag', 'CONDBR2-HLTP-2018-01')
 

Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/HLTCFConfig.py

@@ -190,7 +190,7 @@ def makeHLTTree(newJO=False, triggerConfigHLT = None):
     appendCAtoAthena( monAcc )
 
     Configurable.configurableRun3Behavior=1
-    edmAlg = triggerMergeViewsAndAddMissingEDMCfg(['AOD', 'ESD'], hypos, viewMakers, decObj, decObjHypoOut)
+    edmAlg = triggerMergeViewsAndAddMissingEDMCfg(ConfigFlags, ['AOD', 'ESD'], hypos, viewMakers, decObj, decObjHypoOut)
     Configurable.configurableRun3Behavior=0
     # C) Finally, we create the EDM output
     hltFinalizeSeq += conf2toConfigurable(edmAlg)