Reorganising TriggerEDM code to have a separate file for Run 2 and the...

Reorganising TriggerEDM code to have a separate file for Run 2 and the functions, keeping the name TriggerEDM.py for the functions file which is imported in many places the code.

Reconstruction/RecJobTransforms/share/skeleton.ESDtoAOD_tf.py

@@ -40,8 +40,8 @@ if hasattr(runArgs,"outputAODFile"):
     if TriggerFlags.doMT():
         recoLog.info("Scheduling temporary ESDtoAOD propagation of Trigger MT EDM collections")
         # Note this mirrors skeleton.RDOtoRDOTrigger_tf and skeleton.RAWtoESD_py. It should migrate to a getTriggerEDMList style function
-        from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
-        for item in TriggerHLTList:
+        from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
+        for item in TriggerHLTListRun3:
             if "ESD" in item[1]:
                 objKeyStore.addManyTypesStreamESD( [item[0]] )
             if "AOD" in item[1]:

Reconstruction/RecJobTransforms/share/skeleton.RAWtoESD_tf.py

@@ -69,8 +69,8 @@ if hasattr(runArgs,"inputRDO_TRIGFile"):
     from RecExConfig.ObjKeyStore import objKeyStore
     if TriggerFlags.doMT():
         # Note this mirrors skeleton.RDOtoRDOTrigger_tf and skeleton.ESDtoAOD_tf. It should migrate to a getTriggerEDMList style function
-        from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
-        for item in TriggerHLTList:
+        from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
+        for item in TriggerHLTListRun3:
             if "ESD" in item[1]:
                 objKeyStore.addManyTypesStreamESD( [item[0]] )
             if "AOD" in item[1]:

Reconstruction/RecJobTransforms/share/skeleton.RDOtoRDOtrigger.py

@@ -243,8 +243,8 @@ for i in outSequence.getAllChildren():
         StreamRDO.ItemList += preplist(getLvl1ESDList())
         StreamRDO.ItemList += ["TrigInDetTrackTruthMap#*"]
 
-        from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
-        for item in TriggerHLTList:
+        from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
+        for item in TriggerHLTListRun3:
             if "ESD" in item[1] or "AOD" in item[1]:
                 StreamRDO.ItemList += [item[0]]
 

Trigger/TrigValidation/TrigUpgradeTest/share/full_menu.py

@@ -65,12 +65,12 @@ if opt.doESD:
       __log.debug("Decision Objects to export to ESD [hack method - should be replaced with triggerRunCfg()]")
       __log.debug(decObj)
 
-      from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
+      from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
       ItemList  = [ 'xAOD::TrigCompositeContainer#{}'.format(d) for d in decObj ]
       ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(d) for d in decObj ]
-      ItemList += [ k[0] for k in TriggerHLTList if 'ESD' in k[1] and "TrigComposite" not in k[0] ]
-      ItemList += [ k[0] for k in TriggerHLTList if 'ESD' in k[1] and "TrigComposite" in k[0] ]
-      ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(k[0].split("#")[1]) for k in TriggerHLTList if 'ESD' in k[1] and "TrigComposite" in k[0] ]
+      ItemList += [ k[0] for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" not in k[0] ]
+      ItemList += [ k[0] for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" in k[0] ]
+      ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(k[0].split("#")[1]) for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" in k[0] ]
       ItemList += [ "xAOD::EventInfo#EventInfo" ]
       ItemList += [ "xAOD::EventAuxInfo#EventInfoAux." ]
       ItemList = list(set(ItemList))

Trigger/TrigValidation/TrigUpgradeTest/share/full_menu_cf.py

@@ -330,12 +330,12 @@ if l1decoder:
     __log.debug("Decision Objects to export to ESD [hack method - should be replaced with triggerRunCfg()]")
     __log.debug(decObj)
 
-    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
+    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
     ItemList  = [ 'xAOD::TrigCompositeContainer#{}'.format(d) for d in decObj ]
     ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(d) for d in decObj ]
-    ItemList += [ k[0] for k in TriggerHLTList if 'ESD' in k[1] and "TrigComposite" not in k[0] ]
-    ItemList += [ k[0] for k in TriggerHLTList if 'ESD' in k[1] and "TrigComposite" in k[0] ]
-    ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(k[0].split("#")[1]) for k in TriggerHLTList if 'ESD' in k[1] and "TrigComposite" in k[0] ]
+    ItemList += [ k[0] for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" not in k[0] ]
+    ItemList += [ k[0] for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" in k[0] ]
+    ItemList += [ 'xAOD::TrigCompositeAuxContainer#{}Aux.'.format(k[0].split("#")[1]) for k in TriggerHLTListRun3 if 'ESD' in k[1] and "TrigComposite" in k[0] ]
     ItemList += [ "xAOD::EventInfo#EventInfo" ]
 
     ItemList = list(set(ItemList))

Trigger/TriggerCommon/TrigEDMConfig/python/TriggerEDMRun3.py

-# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
+# Copyright (C) 2002-2019 CERN for the benefit of the ATLAS collaboration
 
 
-# definition of trigger EDM for the Run 3, mostly similar to Run 2
-# however there are different set of EDM to stream and different functionalities required
+# Definition of trigger EDM for the Run 3, mostly similar to Run 2 (TriggerEDM.py).
+# Some different sets of EDM to stream and some different functionalities are required.
+
+
+# Concept of categories is kept, categories are: 
+#   Bjet, Bphys, Egamma, ID/Tracking (to be concluded if can be merged), 
+#   Jet, L1, MET, MinBias, Muon Steer, Tau, Calo
+
+# Please note: 
+#   Dynamic varialbes/Container slimming: All dyn vars are removed unless explicitly specified to be kept!
+
 from AthenaCommon.Logging import logging
 __log = logging.getLogger('TriggerEDMRun3Config')
+
+
 def recordable( name ):
     """
     Verify that the name is in the list of recorded objects and conform to the name convention
 
+    In Run 2 it was a delicate process to configure correctly what got recorded
+    as it had to be set in the algorithm that produced it as well in the TriggerEDM.py in a consistent manner.
 
-    In Run 2 it was a delicate process to configure correctly what get recorded
-    as it had to be set in the produce arlgoirhm as well in here in a consistent manner.
-
-    For Run 3 every alg input/output key can be crosschecked agains the list of objects to record defined here.
+    For Run 3 every alg input/output key can be crosschecked against the list of objects to record which is defined here.
     I.e. in the configuration alg developer would do this:
     from TriggerEDM.TriggerEDMRun3 import recordable
 
@@ -29,15 +39,15 @@ def recordable( name ):
         if "Aux" in name and not name[-1] != ".":
             __log.error( "The collection name {0} is Aux but the name does not end with the '.'".format( name ) )
 
-    for entry in TriggerHLTList:
+    for entry in TriggerHLTListRun3:
         if entry[0].split( "#" )[1] == name:
             return name
     msg = "The collection name {0} is not declared to be stored by HLT. Add it to TriggerEDMRun3.py".format( name )
     __log.error("ERROR in recordable() - see following stack trace.")
     raise RuntimeError( msg )
-    return name
 
-TriggerHLTList = [
+
+TriggerHLTListRun3 = [
 
     #framework/steering
     ('xAOD::TrigCompositeContainer#HLTSummary',              'BS ESD AODFULL AODSLIM', 'Steer'),
@@ -75,9 +85,8 @@ TriggerHLTList = [
     ('xAOD::EnergySumRoI#LVL1EnergySumRoI' ,                 'ESD AODFULL AODSLIM AODVERYSLIM AODBLSSLIM', 'L1'),
     ('xAOD::EnergySumRoIAuxInfo#LVL1EnergySumRoIAux.',       'ESD AODFULL AODSLIM AODVERYSLIM AODBLSSLIM', 'L1'),
 
-    # Egamma
-
 
+    # Egamma
     ('xAOD::TrigEMClusterContainer#HLT_L2CaloEMClusters',           'BS ESD AODFULL', 'Egamma', 'inViews:EMCaloViews'), # last arg specifies in which view container the fragments are, look into the proprty of View maker alg for it
     ('xAOD::TrigEMClusterAuxContainer#HLT_L2CaloEMClustersAux.',    'BS ESD AODFULL', 'Egamma'),
     ('xAOD::TrigPhotonContainer#HLT_L2Photons',                     'BS ESD AODFULL', 'Egamma', 'inViews:L2PhotonRecoViews'),
@@ -218,7 +227,6 @@ TriggerHLTList = [
     ('xAOD::TauTrackAuxContainer#HLT_tautrack_MVAAux.',                    'BS ESD AODFULL AODSLIM AODVERYSLIM', 'Tau'),
 
     #bjet
-
     ('xAOD::TrackParticleContainer#HLT_xAODTracks_FS',                     'BS ESD AODFULL', 'Bjet'),
     ('xAOD::TrackParticleAuxContainer#HLT_xAODTracks_FSAux.',              'BS ESD AODFULL', 'Bjet'),
 
@@ -231,12 +239,10 @@ TriggerHLTList = [
     ('xAOD::JetContainer#HLT_GSCJetAux.',                         'BS ESD AODFULL AODSLIM AODVERYSLIM', 'Bjet'),
 
     # MinBias
-
     ('xAOD::TrackParticleContainer#HLT_xAODTracksMinBias',                 'BS ESD AODFULL', 'MinBias'),
     ('xAOD::TrackParticleAuxContainer#HLT_xAODTracksMinBiasAux.',          'BS ESD AODFULL', 'MinBias'),
 
     # ID
-
     # Requested by TrigUpgradeTest/IDCalo.py
     ('xAOD::TrackParticleContainer#HLT_xAODTracks',                        'BS ESD AODFULL', 'ID'),
     ('xAOD::TrackParticleAuxContainer#HLT_xAODTracksAux.',                 'BS ESD AODFULL', 'ID'),
@@ -248,6 +254,7 @@ TriggerHLTList = [
     ('xAOD::TrigCompositeAuxContainer#HLT_SpacePointCountsAux.',     'BS ESD AODFULL AODSLIM', 'MinBias'),
 ]
 
+
 EDMDetails = {}
 
 EDMDetails[ "TrigRoiDescriptorCollection" ]     = {'persistent':"TrigRoiDescriptorCollection_p3"}
@@ -279,3 +286,5 @@ def tpMap():
             continue
         l[tr] = persistent(tr)
     return l
+
+

Trigger/TriggerCommon/TrigEDMConfig/python/testEDM.py

@@ -3,6 +3,7 @@
 
 
 from TriggerEDM import *
+from TriggerEDMRun2 import EDMDetails
 from CLIDComps.clidGenerator import clidGenerator
 cgen = clidGenerator("", False)
 

Trigger/TriggerCommon/TrigEDMConfig/python/testEDMRun3.py

@@ -15,7 +15,7 @@ def main():
   import re
   serializable_names = []
   serializable_names_no_label = []
-  for TriggerSerializable in TriggerHLTList:
+  for TriggerSerializable in TriggerHLTListRun3:
     #print TriggerSerializable
     serializable_name = TriggerSerializable[0]
     serializable_name_no_label = re.sub(r"\#.*", "", serializable_name)

Trigger/TriggerCommon/TriggerJobOpts/python/HLTTriggerResultGetter.py

@@ -371,7 +371,7 @@ class HLTTriggerResultGetter(Configured):
             # log.warning( "HLTTriggerResultGetter - setting up RoiWriter" )
             topSequence += RoiWriter()
             # write out the RoiDescriptorStores
-            from TrigEDMConfig.TriggerEDM import TriggerRoiList
+            from TrigEDMConfig.TriggerEDMRun2 import TriggerRoiList
             objKeyStore.addManyTypesStreamAOD( TriggerRoiList )
 
         #Are we adding operational info objects in ESD?

Trigger/TriggerCommon/TriggerJobOpts/python/TriggerConfig.py

@@ -201,8 +201,8 @@ def triggerOutputStreamCfg( flags, decObj, outputType ):
     # the rest of triger EDM
     itemsToRecord.extend( __TCKeys( "HLTSummary" ) )
 
-    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
-    EDMCollectionsToRecord=filter( lambda x: outputType in x[1] and "TrigCompositeContainer" not in x[0],  TriggerHLTList )
+    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
+    EDMCollectionsToRecord=filter( lambda x: outputType in x[1] and "TrigCompositeContainer" not in x[0],  TriggerHLTListRun3 )
     itemsToRecord.extend( [ el[0] for el in EDMCollectionsToRecord ] )
 
     # summary objects
@@ -221,7 +221,7 @@ def triggerBSOutputCfg( flags, decObj ):
 
 
 
-    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList, persistent
+    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3, persistent
     from TrigOutputHandling.TrigOutputHandlingConf import HLTResultMTMakerAlg # , StreamTagMakerTool, TriggerBitsMakerTool     # TODO add config of these two
     from TrigOutputHandling.TrigOutputHandlingConfig import TriggerEDMSerialiserToolCfg, HLTResultMTMakerCfg
     
@@ -231,7 +231,7 @@ def triggerBSOutputCfg( flags, decObj ):
                                                     "{}#remap_{}Aux.".format( persistent("xAOD::TrigCompositeAuxContainer"), coll )] )
 
     # EDM
-    EDMCollectionsToRecord=filter( lambda x: "BS" in x[1],  TriggerHLTList )    
+    EDMCollectionsToRecord=filter( lambda x: "BS" in x[1],  TriggerHLTListRun3 )    
     for item in EDMCollectionsToRecord:
         typeName, collName = item[0].split("#")
         serialisedTypeColl="{}#{}".format(persistent(typeName), collName)
@@ -256,12 +256,12 @@ def triggerBSOutputCfg( flags, decObj ):
 def triggerMergeViewsAndAddMissingEDMCfg( edmSet, hypos, viewMakers, decObj ):
 
     from TrigOutputHandling.TrigOutputHandlingConf import HLTEDMCreatorAlg, HLTEDMCreator
-    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTList
+    from TrigEDMConfig.TriggerEDMRun3 import TriggerHLTListRun3
 
     alg = HLTEDMCreatorAlg("EDMCreatorAlg")
 
     # configure views merging
-    needMerging = filter( lambda x: len(x) >= 4 and x[3].startswith("inViews:"),  TriggerHLTList )
+    needMerging = filter( lambda x: len(x) >= 4 and x[3].startswith("inViews:"),  TriggerHLTListRun3 )
     __log.info("These collections need merging: {}".format( " ".join([ c[0] for c in needMerging ])) )
     # group by the view collection name/(the view maker algorithm in practice)
     from collections import defaultdict
@@ -295,7 +295,7 @@ def triggerMergeViewsAndAddMissingEDMCfg( edmSet, hypos, viewMakers, decObj ):
         groupedByType = defaultdict( list )
     
         # scan the EDM
-        for el in TriggerHLTList:
+        for el in TriggerHLTListRun3:
             if not any([ outputType in el[1].split() for outputType in edmSet ]):
                 continue
             collType, collName = el[0].split("#")