HLTTriggerResultGetter.py 20 KB
Newer Older
1
# Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
2
3

from TriggerJobOpts.TriggerFlags import TriggerFlags
4
from AthenaConfiguration.AllConfigFlags import ConfigFlags
5
from AthenaCommon.Logging import logging
6

7
from AthenaCommon.AppMgr import ServiceMgr
8
9
10
11
from RecExConfig.Configured import Configured

from RecExConfig.RecFlags import rec

12
13
from TrigRoiConversion.TrigRoiConversionConf import RoiWriter

14
15
16

class xAODConversionGetter(Configured):
    def configure(self):
17
        from AthenaCommon.AlgSequence import AlgSequence
18
19
20
21
22
        topSequence = AlgSequence()

        #schedule xAOD conversions here
        from TrigBSExtraction.TrigBSExtractionConf import TrigHLTtoxAODConversion
        xaodconverter = TrigHLTtoxAODConversion()
23
24
        if ConfigFlags.Trigger.readBS:
            xaodconverter.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")] # contract wiht BSExtraction alg (see below)
25
26
        from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
        xaodconverter.Navigation = HLTNavigationOffline()
27

28
29
        from TrigEDMConfig.TriggerEDM import getPreregistrationList
        from TrigEDMConfig.TriggerEDM import getEFRun1BSList,getEFRun2EquivalentList,getL2Run1BSList,getL2Run2EquivalentList
30
        xaodconverter.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
31

32
33
34
        # we want only containers from Run 1 with the BS tag
        xaodconverter.BStoxAOD.ContainersToConvert = getL2Run1BSList() + getEFRun1BSList()
        xaodconverter.BStoxAOD.NewContainers = getL2Run2EquivalentList() + getEFRun2EquivalentList()
35
36
37
38

        xaodconverter.HLTResultKey="HLTResult_EF"
        topSequence += xaodconverter

39
40
41
42
        # define list of HLT xAOD containers to be written to the output root file
        # (previously this was defined in HLTTriggerResultGetter def configure)
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList
        self.xaodlist = {}
43
        self.xaodlist.update( getTriggerEDMList(ConfigFlags.Trigger.ESDEDMSet, 2 ))
44

45
46
47
48
49
50
        return True
    
        

class ByteStreamUnpackGetter(Configured):
    def configure(self):
51
        log = logging.getLogger("ByteStreamUnpackGetter")
52

53
54
55
56
57
58
59
60
61
62
63
64
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
        hasHLT = TriggerFlags.dataTakingConditions()=='HltOnly' or TriggerFlags.dataTakingConditions()=='FullTrigger'
        if not hasHLT:
            log.info("Will not configure HLT BS unpacking because dataTakingConditions flag indicates HLT was disabled")
            return True

        # Define the decoding sequence
        from TrigHLTResultByteStream.TrigHLTResultByteStreamConf import HLTResultMTByteStreamDecoderAlg
        from TrigOutputHandling.TrigOutputHandlingConf import TriggerEDMDeserialiserAlg
        from AthenaCommon.CFElements import seqAND
        decoder = HLTResultMTByteStreamDecoderAlg()
        deserialiser = TriggerEDMDeserialiserAlg("TrigDeserialiser")
65
        deserialiser.ExtraOutputs += [('xAOD::TrigCompositeContainer' , 'StoreGateSvc+HLTNav_Summary_OnlineSlimmed')]
66
67
68
69
70
71
72
73
74
75
76
77
        decodingSeq = seqAND("HLTDecodingSeq")
        decodingSeq += decoder  # BS -> HLTResultMT
        decodingSeq += deserialiser  # HLTResultMT -> xAOD

        # Append the decoding sequence to topSequence
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
        topSequence += decodingSeq

        log.debug("Configured HLT result BS decoding sequence")
        return True

78
class ByteStreamUnpackGetterRun1or2(Configured):
79
80
    def configure(self):

81
        log = logging.getLogger("ByteStreamUnpackGetterRun1or2")
82
83
84
        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
85
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
86
        # in MC this is always FullTrigger
87
        hasHLT = TriggerFlags.dataTakingConditions() in ('HltOnly', 'FullTrigger')
88
89
90
91
        
        # BS unpacking
        from TrigBSExtraction.TrigBSExtractionConf import TrigBSExtraction
        extr = TrigBSExtraction()
92
93

        # Add fictional output to ensure data dependency in AthenaMT
Tim Martin's avatar
Tim Martin committed
94
95
        # Keeping the run 2 workflow, we run this after we have put the full serialised navigation into xAOD
        extr.ExtraInputs += [("xAOD::TrigNavigation", "StoreGateSvc+TrigNavigation")]
96
        extr.ExtraOutputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
97
98
99
        
        if hasHLT:
            from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
100
101
102
103
            extr.NavigationForL2 = HLTNavigationOffline("NavigationForL2")
            # Ignore the L2 TrigPassBits to avoid clash with EF (ATR-23411)
            extr.NavigationForL2.ClassesFromPayloadIgnore = ["TrigPassBits#passbits"]

104
            extr.Navigation = HLTNavigationOffline()
105
106

            from TrigEDMConfig.TriggerEDM import getEDMLibraries
107
            extr.Navigation.Dlls = getEDMLibraries()            
108
109

            from TrigEDMConfig.TriggerEDM import getPreregistrationList
110
            extr.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
111

112
113
            from eformat import helper as efh
            robIDMap = {}   # map of result keys and their ROB ID
114
            if ConfigFlags.Trigger.EDMVersion == 1:  # Run-1 has L2 and EF result
115
116
                robIDMap["HLTResult_L2"] = efh.SourceIdentifier(efh.SubDetector.TDAQ_LVL2, 0).code()
                robIDMap["HLTResult_EF"] = efh.SourceIdentifier(efh.SubDetector.TDAQ_EVENT_FILTER, 0).code()
117
                extr.L2ResultKey = "HLTResult_L2"
118
                extr.HLTResultKey = "HLTResult_EF"
119
            else:
120
                robIDMap["HLTResult_HLT"] = efh.SourceIdentifier(efh.SubDetector.TDAQ_HLT, 0).code()
121
122
                extr.L2ResultKey = ""
                extr.HLTResultKey = "HLTResult_HLT"
123
124

            # Configure DataScouting
125
            from PyUtils.MetaReaderPeeker import metadata
126
            if 'stream' in metadata:
127
                stream_local = metadata['stream']   # e.g. calibration_DataScouting_05_Jets
128
                if stream_local.startswith('calibration_DataScouting_'):
129
                    ds_tag = '_'.join(stream_local.split('_')[1:3])   # e.g. DataScouting_05
130
131
132
                    ds_id = int(stream_local.split('_')[2])           # e.g. 05
                    robIDMap[ds_tag] = efh.SourceIdentifier(efh.SubDetector.TDAQ_HLT, ds_id).code()
                    extr.DSResultKeys += [ds_tag]
133
134

        else:
135
136
137
138
139
            # if data doesn't have HLT info set HLTResult keys as empty strings to avoid warnings
            # but the extraction algorithm must run
            extr.L2ResultKey = ""
            extr.HLTResultKey = ""
            extr.DSResultKeys = []
140
141
142

        topSequence += extr

143
144
145
146
147
        # Add all HLTResult keys to AddressProvider
        for k in robIDMap.keys():
            ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ f"HLT::HLTResult/{k}" ]

        # Create necessary public tools
148
        from AthenaCommon.AppMgr import ToolSvc
149
        from TrigSerializeTP.TrigSerializeTPConf import TrigSerTPTool
150
        from TrigEDMConfig.TriggerEDM import getTPList
151
        ToolSvc += TrigSerTPTool(TPMap = getTPList((ConfigFlags.Trigger.EDMVersion)))
152
153
        
        from TrigSerializeCnvSvc.TrigSerializeCnvSvcConf import TrigSerializeConvHelper
154
155
156
157
        ToolSvc += TrigSerializeConvHelper(doTP = True)

        from TrigHLTResultByteStream.TrigHLTResultByteStreamConf import HLT__HLTResultByteStreamTool
        ToolSvc += HLT__HLTResultByteStreamTool(HLTResultRobIdMap = robIDMap)
158
159
160
161
162

        return True


class TrigDecisionGetter(Configured):
163
164
165
166
167
168
169
170
171
    def configure(self):
        log = logging.getLogger("TrigDecisionGetter")

        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()

        from TrigDecisionMaker.TrigDecisionMakerConfig import TrigDecisionMakerMT
        tdm = TrigDecisionMakerMT('TrigDecMakerMT')

172
        if not ConfigFlags.Trigger.readBS:
173
174
175
176
177
178
179
180
181
            # Construct trigger bits from HLTNav_summary instead of reading from BS
            from TrigOutputHandling.TrigOutputHandlingConf import TriggerBitsMakerTool
            tdm.BitsMakerTool = TriggerBitsMakerTool()

        topSequence += tdm
        log.info('xTrigDecision writing enabled')

        return True

182
class TrigDecisionGetterRun1or2(Configured):
183
184
185
    #class to setup the writing or just making of TrigDecisionObject
    def configure(self):
        
186
        log = logging.getLogger("TrigDecisionGetterRun1or2")
187
188
189
190
191
192
193
194
195
196
197

        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if hasOnlyLVL1:
        #from RecExConfig.ObjKeyStore import objKeyStore
        #objKeyStore.addStreamESD('TrigDec::TrigDecision','TrigDecision')
        #objKeyStore.addStreamAOD('TrigDec::TrigDecision','TrigDecision')
        
        from RecExConfig.RecFlags import rec
        if ( rec.doWriteESD() or rec.doWriteAOD() or rec.doESD() or rec.doAOD() ) and \
198
               ( not ( rec.readAOD() or rec.readESD() or rec.doWriteBS()) ):
199
            log.info("Will write TrigDecision object to storegate")
200
            
201
            from TrigDecisionMaker.TrigDecisionMakerConfig import WriteTrigDecision
202
            trigDecWriter = WriteTrigDecision()  # noqa: F841
203
204
205
            if (ConfigFlags.Trigger.EDMVersion == 1 or ConfigFlags.Trigger.EDMVersion == 2) and ConfigFlags.Trigger.doEDMVersionConversion:
                from TrigNavTools.NavConverterConfig import createNavConverterAlg
                navCnvAlg = createNavConverterAlg()
206
                navCnvAlg.TrigConfigSvc = "HLTConfigSvcRun3"
207
208
                navCnvAlg.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
                topSequence += navCnvAlg
209

210
#           WritexAODTrigDecision() is called within WriteTrigDecision()
211

212
213
214
215
216
            # inform TD maker that some parts may be missing
            if TriggerFlags.dataTakingConditions()=='Lvl1Only':
                topSequence.TrigDecMaker.doL2=False
                topSequence.TrigDecMaker.doEF=False
                topSequence.TrigDecMaker.doHLT=False
217
218
219
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
220
221
222
            elif TriggerFlags.dataTakingConditions()=='HltOnly':
                from AthenaCommon.AlgSequence import AlgSequence
                topSequence.TrigDecMaker.doL1=False
223
224

            if ConfigFlags.Trigger.EDMVersion == 1:  # Run-1 has L2 and EF result
225
                topSequence.TrigDecMaker.doHLT = False
226
227
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
228
229
230
            else:
                topSequence.TrigDecMaker.doL2 = False
                topSequence.TrigDecMaker.doEF = False
231
232
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
233
234
235
236
                pass
                
        else:
            log.info("Will not write TrigDecision object to storegate")
237
    
238
239
240
241
242
243
244
245
246
247
248
249
        return True
    
    
class HLTTriggerResultGetter(Configured):

    log = logging.getLogger("HLTTriggerResultGetter.py")

    def _AddOPIToESD(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")        
        
        if rec.doESD():
250
251
252
            from PyUtils.MetaReaderPeeker import metadata
            if 'stream' in metadata:
                stream = metadata['stream']
253
                log.debug("the stream found in 'metadata' is %s", stream)
254
255
                if "express" in stream:
                    from TrigEDMConfig.TriggerEDM import getTypeAndKey,EDMDetails
256
                    type,key=getTypeAndKey("TrigOperationalInfo#HLT_EXPRESS_OPI_HLT")
257
                    if 'collection'in EDMDetails[type]:
258
                        colltype = EDMDetails[type]['collection']
259
                        log.info("Adding HLT_EXPRESS_OPI_HLT to ESD for stream %s", stream)
260
261
262
263
                        from RecExConfig.ObjKeyStore import objKeyStore
                        objKeyStore.addStreamESD(colltype, key)
                    return True
            else:
264
                log.warning("Could not determine stream of bytestream file, not adding HLT_EXPRESS_OPI_HLT to ESD.")
265
266
267
268
269
270
271
272
273
        return False

    def configure(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")
        from RecExConfig.ObjKeyStore import objKeyStore

        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
274
275
        log.info("BS unpacking (ConfigFlags.Trigger.readBS): %d", ConfigFlags.Trigger.readBS )
        if ConfigFlags.Trigger.readBS:
276
277
278
279
            if ConfigFlags.Trigger.EDMVersion == 1 or \
               ConfigFlags.Trigger.EDMVersion == 2:
                bs = ByteStreamUnpackGetterRun1or2()  # noqa: F841
            elif ConfigFlags.Trigger.EDMVersion >=3:
280
                bs = ByteStreamUnpackGetter()  # noqa: F841
281
282
            else:
                raise RuntimeError("Invalid EDMVersion=%s " % ConfigFlags.Trigger.EDMVersion)
283

284
        xAODContainers = {}
285

286
        if ConfigFlags.Trigger.EDMVersion == 1:
287
288
            xaodcnvrt = xAODConversionGetter()
            xAODContainers = xaodcnvrt.xaodlist
289

290
291
292
293
294
        if ConfigFlags.Trigger.EDMVersion == 1 or \
           ConfigFlags.Trigger.EDMVersion == 2:
            if rec.doTrigger() or TriggerFlags.doTriggerConfigOnly():
                tdt = TrigDecisionGetterRun1or2()  # noqa: F841
        elif ConfigFlags.Trigger.EDMVersion >= 3:
295
            if ConfigFlags.Trigger.readBS:
296
297
298
                tdt = TrigDecisionGetter()  # noqa: F841
        else:
            raise RuntimeError("Invalid EDMVersion=%s " % ConfigFlags.Trigger.EDMVersion)
299

300
        # Temporary hack to add Run-3 navigation to ESD and AOD
301
        if (rec.doESD() or rec.doAOD()) and ConfigFlags.Trigger.EDMVersion == 3:
302
303
304
305
306
307
308
309
310
            # The hack with wildcards is needed for BS->ESD because we don't know the exact keys
            # of HLT navigation containers before unpacking them from the BS event.
            objKeyStore._store['streamESD'].allowWildCard(True)
            objKeyStore._store['streamAOD'].allowWildCard(True)
            objKeyStore.addManyTypesStreamESD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])
            objKeyStore.addManyTypesStreamAOD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])

311
312
313
314
315
316
317
318
319
        # TrigJetRec additions
        if rec.doWriteESD():
            objKeyStore.addStreamESD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamESD("JetMomentMap","TrigJetRecMomentMap")

        if rec.doWriteAOD():
            objKeyStore.addStreamAOD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamAOD("JetMomentMap","TrigJetRecMomentMap")
                    
320
        if rec.doAOD() or rec.doWriteAOD():
321
322
            # schedule the RoiDescriptorStore conversion
            # log.warning( "HLTTriggerResultGetter - setting up RoiWriter" )
323
324
325
326
            roiWriter = RoiWriter()
            # Add fictional input to ensure data dependency in AthenaMT
            roiWriter.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
            topSequence += roiWriter
327
            # write out the RoiDescriptorStores
328
            from TrigEDMConfig.TriggerEDMRun2 import TriggerRoiList
329
330
            objKeyStore.addManyTypesStreamAOD( TriggerRoiList )

331
332
333
        #Are we adding operational info objects in ESD?
        added=self._AddOPIToESD()
        if added:
334
            log.debug("Operational Info object HLT_EXPRESS_OPI_HLT with extra information about express stream prescaling added to the data.")
335
336
337
338
339
340
        


        # ESD objects definitions
        _TriggerESDList = {}

341
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList 
342
343
344
345
346
        # we have to store xAOD containers in the root file, NOT AOD,
        # if the xAOD container list is not empty
        if(xAODContainers):
            _TriggerESDList.update( xAODContainers )
        else:
347
            _TriggerESDList.update( getTriggerEDMList(ConfigFlags.Trigger.ESDEDMSet,  ConfigFlags.Trigger.EDMVersion) )
348
        
349
        log.info("ESD content set according to the ESDEDMSet flag: %s and EDM version %d", ConfigFlags.Trigger.ESDEDMSet, ConfigFlags.Trigger.EDMVersion)
350
351
352
353

        # AOD objects choice
        _TriggerAODList = {}
        
354
        #from TrigEDMConfig.TriggerEDM import getAODList    
355
        _TriggerAODList.update( getTriggerEDMList(ConfigFlags.Trigger.AODEDMSet,  ConfigFlags.Trigger.EDMVersion) )
356

357
        log.info("AOD content set according to the AODEDMSet flag: %s and EDM version %d", ConfigFlags.Trigger.AODEDMSet, ConfigFlags.Trigger.EDMVersion)
358

359
360
        log.debug("ESD EDM list: %s", _TriggerESDList)
        log.debug("AOD EDM list: %s", _TriggerAODList)
361
        
362
363
364
365
366
367
368
369
370
371
372
373
        # Highlight what is in AOD list but not in ESD list, as this can cause
        # the "different number of entries in branch" problem, when it is in the
        # AOD list but the empty container per event is not created
        # Just compares keys of dicts, which are the class names, not their string keys in StoreGate
        not_in = [ element for element in  _TriggerAODList if element not in _TriggerESDList ]
        if (len(not_in)>0):
            log.warning("In AOD list but not in ESD list: ")
            log.warning(not_in)
        else:
            log.info("AOD list is subset of ESD list - good.")


374
        def _addSlimmingRun2(stream, edm):
375
            from TrigNavTools.TrigNavToolsConfig import navigationThinningSvc
376

377
378
            edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
          
379
380
381
382
            # TimM Sep 2021: In MT the 'reload' slimming option in the R2 navigation thinning service was found to be creating
            # AODs which would crash when trying to return features. We therefore remove this option by using the added 'cleanup_noreload'
            # configuration, see ATR-24141 for details. 
            svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup_noreload', 
383
384
385
386
387
388
389
390
                                          'result':'HLTResult_HLT',
                                          'features':edmlist})

            from OutputStreamAthenaPool.CreateOutputStreams import registerTrigNavThinningSvc
            registerTrigNavThinningSvc (stream, svc)

            log.info("Configured slimming of HLT for %s", stream)
            print(svc)  # noqa: ATL901
391
392
393
            del edmlist


394
395
396
        if ConfigFlags.Trigger.EDMVersion == 1 or ConfigFlags.Trigger.EDMVersion == 2:

            # Run 1, 2 slimming
397
            if ConfigFlags.Trigger.doNavigationSlimming and rec.readRDO() and rec.doWriteAOD():
398
399
400
                _addSlimmingRun2('StreamAOD', _TriggerESDList ) #Use ESD item list also for AOD!
                log.info("configured navigation slimming for AOD output")
                
401
            if ConfigFlags.Trigger.doNavigationSlimming and rec.readRDO() and rec.doWriteESD():
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
                _addSlimmingRun2('StreamESD', _TriggerESDList )                
                log.info("configured navigation slimming for ESD output")

        if ConfigFlags.Trigger.EDMVersion >= 3:
            # Change in the future to 'if EDMVersion >= 3 or doEDMVersionConversion:'

            # Run 3 slimming
            if ConfigFlags.Trigger.doNavigationSlimming: 
                from TrigNavSlimmingMT.TrigNavSlimmingMTConfig import getTrigNavSlimmingMTConfig
                from AthenaCommon.Configurable import Configurable
                Configurable.configurableRun3Behavior += 1
                from AthenaConfiguration.ComponentAccumulator import appendCAtoAthena
                appendCAtoAthena( getTrigNavSlimmingMTConfig(ConfigFlags) )
                Configurable.configurableRun3Behavior -= 1
            else:
                log.info("doNavigationSlimming is False, won't schedule run 3 navigation slimming")
418

419
        objKeyStore.addManyTypesStreamESD( _TriggerESDList )
420
421
422
423
424
425
426
427
        objKeyStore.addManyTypesStreamAOD( _TriggerAODList )        
            
        return True