HLTTriggerResultGetter.py 19.3 KB
Newer Older
1
# Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
2
3

from TriggerJobOpts.TriggerFlags import TriggerFlags
4
from AthenaConfiguration.AllConfigFlags import ConfigFlags
5
6
from AthenaCommon.Logging import logging
from AthenaCommon.GlobalFlags import globalflags
7

8
from AthenaCommon.AppMgr import ServiceMgr
9
10
11
12
from RecExConfig.Configured import Configured

from RecExConfig.RecFlags import rec

13
14
from TrigRoiConversion.TrigRoiConversionConf import RoiWriter

15
16
17

class xAODConversionGetter(Configured):
    def configure(self):
18
        from AthenaCommon.AlgSequence import AlgSequence
19
20
21
22
23
24
25
26
        topSequence = AlgSequence()

        #schedule xAOD conversions here
        from TrigBSExtraction.TrigBSExtractionConf import TrigHLTtoxAODConversion
        xaodconverter = TrigHLTtoxAODConversion()
        
        from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
        xaodconverter.Navigation = HLTNavigationOffline()
27

28
29
        from TrigEDMConfig.TriggerEDM import getPreregistrationList
        from TrigEDMConfig.TriggerEDM import getEFRun1BSList,getEFRun2EquivalentList,getL2Run1BSList,getL2Run2EquivalentList
30
        xaodconverter.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
31

32
33
34
        # we want only containers from Run 1 with the BS tag
        xaodconverter.BStoxAOD.ContainersToConvert = getL2Run1BSList() + getEFRun1BSList()
        xaodconverter.BStoxAOD.NewContainers = getL2Run2EquivalentList() + getEFRun2EquivalentList()
35
36
37
38

        xaodconverter.HLTResultKey="HLTResult_EF"
        topSequence += xaodconverter

39
40
41
42
43
44
        # define list of HLT xAOD containers to be written to the output root file
        # (previously this was defined in HLTTriggerResultGetter def configure)
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList
        self.xaodlist = {}
        self.xaodlist.update( getTriggerEDMList(TriggerFlags.ESDEDMSet(), 2 ))

45
46
47
48
49
50
        return True
    
        

class ByteStreamUnpackGetter(Configured):
    def configure(self):
51
        log = logging.getLogger("ByteStreamUnpackGetter")
52

53
54
55
56
57
58
59
60
61
62
63
64
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
        hasHLT = TriggerFlags.dataTakingConditions()=='HltOnly' or TriggerFlags.dataTakingConditions()=='FullTrigger'
        if not hasHLT:
            log.info("Will not configure HLT BS unpacking because dataTakingConditions flag indicates HLT was disabled")
            return True

        # Define the decoding sequence
        from TrigHLTResultByteStream.TrigHLTResultByteStreamConf import HLTResultMTByteStreamDecoderAlg
        from TrigOutputHandling.TrigOutputHandlingConf import TriggerEDMDeserialiserAlg
        from AthenaCommon.CFElements import seqAND
        decoder = HLTResultMTByteStreamDecoderAlg()
        deserialiser = TriggerEDMDeserialiserAlg("TrigDeserialiser")
65
        deserialiser.ExtraOutputs += [('xAOD::TrigCompositeContainer' , 'StoreGateSvc+HLTNav_Summary_OnlineSlimmed')]
66
67
68
69
70
71
72
73
74
75
76
77
        decodingSeq = seqAND("HLTDecodingSeq")
        decodingSeq += decoder  # BS -> HLTResultMT
        decodingSeq += deserialiser  # HLTResultMT -> xAOD

        # Append the decoding sequence to topSequence
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
        topSequence += decodingSeq

        log.debug("Configured HLT result BS decoding sequence")
        return True

78
class ByteStreamUnpackGetterRun1or2(Configured):
79
80
    def configure(self):

81
        log = logging.getLogger("ByteStreamUnpackGetterRun1or2")
82
83
84
85
        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if TriggerFlags.readBS():
86
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
87
        # in MC this is always FullTrigger
88
        hasHLT = TriggerFlags.dataTakingConditions() in ('HltOnly', 'FullTrigger')
89
90
91
92
        
        # BS unpacking
        from TrigBSExtraction.TrigBSExtractionConf import TrigBSExtraction
        extr = TrigBSExtraction()
93
94
95

        # Add fictional output to ensure data dependency in AthenaMT
        extr.ExtraOutputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
96
97
98
        
        if hasHLT:
            from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
99
100
101
102
            extr.NavigationForL2 = HLTNavigationOffline("NavigationForL2")
            # Ignore the L2 TrigPassBits to avoid clash with EF (ATR-23411)
            extr.NavigationForL2.ClassesFromPayloadIgnore = ["TrigPassBits#passbits"]

103
            extr.Navigation = HLTNavigationOffline()
104
105

            from TrigEDMConfig.TriggerEDM import getEDMLibraries
106
            extr.Navigation.Dlls = getEDMLibraries()            
107
108

            from TrigEDMConfig.TriggerEDM import getPreregistrationList
109
            extr.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
110

111
112
            from eformat import helper as efh
            robIDMap = {}   # map of result keys and their ROB ID
113
            if ConfigFlags.Trigger.EDMVersion == 1:  # Run-1 has L2 and EF result
114
115
                robIDMap["HLTResult_L2"] = efh.SourceIdentifier(efh.SubDetector.TDAQ_LVL2, 0).code()
                robIDMap["HLTResult_EF"] = efh.SourceIdentifier(efh.SubDetector.TDAQ_EVENT_FILTER, 0).code()
116
                extr.L2ResultKey = "HLTResult_L2"
117
                extr.HLTResultKey = "HLTResult_EF"
118
            else:
119
                robIDMap["HLTResult_HLT"] = efh.SourceIdentifier(efh.SubDetector.TDAQ_HLT, 0).code()
120
121
                extr.L2ResultKey = ""
                extr.HLTResultKey = "HLTResult_HLT"
122
123

            # Configure DataScouting
124
            from PyUtils.MetaReaderPeeker import metadata
125
            if 'stream' in metadata:
126
                stream_local = metadata['stream']   # e.g. calibration_DataScouting_05_Jets
127
                if stream_local.startswith('calibration_DataScouting_'):
128
                    ds_tag = '_'.join(stream_local.split('_')[1:3])   # e.g. DataScouting_05
129
130
131
                    ds_id = int(stream_local.split('_')[2])           # e.g. 05
                    robIDMap[ds_tag] = efh.SourceIdentifier(efh.SubDetector.TDAQ_HLT, ds_id).code()
                    extr.DSResultKeys += [ds_tag]
132
133

        else:
134
135
136
137
138
            # if data doesn't have HLT info set HLTResult keys as empty strings to avoid warnings
            # but the extraction algorithm must run
            extr.L2ResultKey = ""
            extr.HLTResultKey = ""
            extr.DSResultKeys = []
139
140
141

        topSequence += extr

142
143
144
145
146
        # Add all HLTResult keys to AddressProvider
        for k in robIDMap.keys():
            ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ f"HLT::HLTResult/{k}" ]

        # Create necessary public tools
147
        from AthenaCommon.AppMgr import ToolSvc
148
        from TrigSerializeTP.TrigSerializeTPConf import TrigSerTPTool
149
        from TrigEDMConfig.TriggerEDM import getTPList
150
        ToolSvc += TrigSerTPTool(TPMap = getTPList((ConfigFlags.Trigger.EDMVersion)))
151
152
        
        from TrigSerializeCnvSvc.TrigSerializeCnvSvcConf import TrigSerializeConvHelper
153
154
155
156
        ToolSvc += TrigSerializeConvHelper(doTP = True)

        from TrigHLTResultByteStream.TrigHLTResultByteStreamConf import HLT__HLTResultByteStreamTool
        ToolSvc += HLT__HLTResultByteStreamTool(HLTResultRobIdMap = robIDMap)
157
158
159
160
161

        return True


class TrigDecisionGetter(Configured):
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    def configure(self):
        log = logging.getLogger("TrigDecisionGetter")

        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()

        from TrigDecisionMaker.TrigDecisionMakerConfig import TrigDecisionMakerMT
        tdm = TrigDecisionMakerMT('TrigDecMakerMT')

        if not TriggerFlags.readBS():
            # Construct trigger bits from HLTNav_summary instead of reading from BS
            from TrigOutputHandling.TrigOutputHandlingConf import TriggerBitsMakerTool
            tdm.BitsMakerTool = TriggerBitsMakerTool()

        topSequence += tdm
        log.info('xTrigDecision writing enabled')

        return True

181
class TrigDecisionGetterRun1or2(Configured):
182
183
184
    #class to setup the writing or just making of TrigDecisionObject
    def configure(self):
        
185
        log = logging.getLogger("TrigDecisionGetterRun1or2")
186
187
188
189
190
191
192
193
194
195
196

        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if hasOnlyLVL1:
        #from RecExConfig.ObjKeyStore import objKeyStore
        #objKeyStore.addStreamESD('TrigDec::TrigDecision','TrigDecision')
        #objKeyStore.addStreamAOD('TrigDec::TrigDecision','TrigDecision')
        
        from RecExConfig.RecFlags import rec
        if ( rec.doWriteESD() or rec.doWriteAOD() or rec.doESD() or rec.doAOD() ) and \
197
               ( not ( rec.readAOD() or rec.readESD() or rec.doWriteBS()) ):
198
            log.info("Will write TrigDecision object to storegate")
199
            
200
            from TrigDecisionMaker.TrigDecisionMakerConfig import WriteTrigDecision
201
            trigDecWriter = WriteTrigDecision()  # noqa: F841
202
203
204
            if (ConfigFlags.Trigger.EDMVersion == 1 or ConfigFlags.Trigger.EDMVersion == 2) and ConfigFlags.Trigger.doEDMVersionConversion:
                from TrigNavTools.NavConverterConfig import createNavConverterAlg
                navCnvAlg = createNavConverterAlg()
205
                navCnvAlg.TrigConfigSvc = "HLTConfigSvcRun3"
206
207
                navCnvAlg.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
                topSequence += navCnvAlg
208

209
#           WritexAODTrigDecision() is called within WriteTrigDecision()
210

211
212
213
214
215
            # inform TD maker that some parts may be missing
            if TriggerFlags.dataTakingConditions()=='Lvl1Only':
                topSequence.TrigDecMaker.doL2=False
                topSequence.TrigDecMaker.doEF=False
                topSequence.TrigDecMaker.doHLT=False
216
217
218
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
219
220
221
            elif TriggerFlags.dataTakingConditions()=='HltOnly':
                from AthenaCommon.AlgSequence import AlgSequence
                topSequence.TrigDecMaker.doL1=False
222
223

            if ConfigFlags.Trigger.EDMVersion == 1:  # Run-1 has L2 and EF result
224
                topSequence.TrigDecMaker.doHLT = False
225
226
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
227
228
229
            else:
                topSequence.TrigDecMaker.doL2 = False
                topSequence.TrigDecMaker.doEF = False
230
231
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
232
233
234
235
                pass
                
        else:
            log.info("Will not write TrigDecision object to storegate")
236
    
237
238
239
240
241
242
243
244
245
246
247
248
        return True
    
    
class HLTTriggerResultGetter(Configured):

    log = logging.getLogger("HLTTriggerResultGetter.py")

    def _AddOPIToESD(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")        
        
        if rec.doESD():
249
250
251
            from PyUtils.MetaReaderPeeker import metadata
            if 'stream' in metadata:
                stream = metadata['stream']
252
                log.debug("the stream found in 'metadata' is %s", stream)
253
254
                if "express" in stream:
                    from TrigEDMConfig.TriggerEDM import getTypeAndKey,EDMDetails
255
                    type,key=getTypeAndKey("TrigOperationalInfo#HLT_EXPRESS_OPI_HLT")
256
                    if 'collection'in EDMDetails[type]:
257
                        colltype = EDMDetails[type]['collection']
258
                        log.info("Adding HLT_EXPRESS_OPI_HLT to ESD for stream %s", stream)
259
260
261
262
                        from RecExConfig.ObjKeyStore import objKeyStore
                        objKeyStore.addStreamESD(colltype, key)
                    return True
            else:
263
                log.warning("Could not determine stream of bytestream file, not adding HLT_EXPRESS_OPI_HLT to ESD.")
264
265
266
267
268
269
270
        return False

    def configure(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")
        from RecExConfig.ObjKeyStore import objKeyStore

271
272
273
274
        # Set AODFULL for data unless it was set explicitly already
        if TriggerFlags.AODEDMSet.isDefault() and globalflags.DataSource()=='data':
            TriggerFlags.AODEDMSet = 'AODFULL'
            
275
276
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
277
        log.info("BS unpacking (TF.readBS): %d", TriggerFlags.readBS() )
278
        if TriggerFlags.readBS():
279
280
281
282
            if ConfigFlags.Trigger.EDMVersion == 1 or \
               ConfigFlags.Trigger.EDMVersion == 2:
                bs = ByteStreamUnpackGetterRun1or2()  # noqa: F841
            elif ConfigFlags.Trigger.EDMVersion >=3:
283
                bs = ByteStreamUnpackGetter()  # noqa: F841
284
285
            else:
                raise RuntimeError("Invalid EDMVersion=%s " % ConfigFlags.Trigger.EDMVersion)
286

287
        xAODContainers = {}
288

289
        if ConfigFlags.Trigger.EDMVersion == 1:
290
291
            xaodcnvrt = xAODConversionGetter()
            xAODContainers = xaodcnvrt.xaodlist
292

293
294
295
296
297
298
299
300
301
        if ConfigFlags.Trigger.EDMVersion == 1 or \
           ConfigFlags.Trigger.EDMVersion == 2:
            if rec.doTrigger() or TriggerFlags.doTriggerConfigOnly():
                tdt = TrigDecisionGetterRun1or2()  # noqa: F841
        elif ConfigFlags.Trigger.EDMVersion >= 3:
            if TriggerFlags.readBS():
                tdt = TrigDecisionGetter()  # noqa: F841
        else:
            raise RuntimeError("Invalid EDMVersion=%s " % ConfigFlags.Trigger.EDMVersion)
302

303
        # Temporary hack to add Run-3 navigation to ESD and AOD
304
        if (rec.doESD() or rec.doAOD()) and ConfigFlags.Trigger.EDMVersion == 3:
305
306
307
308
309
310
311
312
313
            # The hack with wildcards is needed for BS->ESD because we don't know the exact keys
            # of HLT navigation containers before unpacking them from the BS event.
            objKeyStore._store['streamESD'].allowWildCard(True)
            objKeyStore._store['streamAOD'].allowWildCard(True)
            objKeyStore.addManyTypesStreamESD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])
            objKeyStore.addManyTypesStreamAOD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
        # TrigJetRec additions
        if rec.doWriteESD():
            objKeyStore.addStreamESD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamESD("JetMomentMap","TrigJetRecMomentMap")

        if rec.doWriteAOD():
            objKeyStore.addStreamAOD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamAOD("JetMomentMap","TrigJetRecMomentMap")
                    
        # ID truth
        if not rec.readESD() and (not rec.readAOD()) and TriggerFlags.doID() \
                and rec.doTruth():
            try:
                from TrigInDetTruthAlgs.TrigInDetTruthAlgsConfig import \
                    TrigIDTruthMaker
                topSequence += TrigIDTruthMaker()
            except Exception:
                log.warning( "Couldn't set up the trigger ID truth maker" )
                pass

        if rec.doESD() or rec.doAOD():
            from TrigEDMConfig.TriggerEDM import getTrigIDTruthList
            objKeyStore.addManyTypesStreamESD(getTrigIDTruthList(TriggerFlags.ESDEDMSet()))
            objKeyStore.addManyTypesStreamAOD(getTrigIDTruthList(TriggerFlags.AODEDMSet()))

339
        if rec.doAOD() or rec.doWriteAOD():
340
341
            # schedule the RoiDescriptorStore conversion
            # log.warning( "HLTTriggerResultGetter - setting up RoiWriter" )
342
343
344
345
            roiWriter = RoiWriter()
            # Add fictional input to ensure data dependency in AthenaMT
            roiWriter.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
            topSequence += roiWriter
346
            # write out the RoiDescriptorStores
347
            from TrigEDMConfig.TriggerEDMRun2 import TriggerRoiList
348
349
            objKeyStore.addManyTypesStreamAOD( TriggerRoiList )

350
351
352
        #Are we adding operational info objects in ESD?
        added=self._AddOPIToESD()
        if added:
353
            log.debug("Operational Info object HLT_EXPRESS_OPI_HLT with extra information about express stream prescaling added to the data.")
354
355
356
357
358
359
        


        # ESD objects definitions
        _TriggerESDList = {}

360
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList 
361
362
363
364
365
        # we have to store xAOD containers in the root file, NOT AOD,
        # if the xAOD container list is not empty
        if(xAODContainers):
            _TriggerESDList.update( xAODContainers )
        else:
366
            _TriggerESDList.update( getTriggerEDMList(TriggerFlags.ESDEDMSet(),  ConfigFlags.Trigger.EDMVersion) ) 
367
        
368
        log.info("ESD content set according to the ESDEDMSet flag: %s and EDM version %d", TriggerFlags.ESDEDMSet(), ConfigFlags.Trigger.EDMVersion)
369
370
371
372

        # AOD objects choice
        _TriggerAODList = {}
        
373
        #from TrigEDMConfig.TriggerEDM import getAODList    
374
        _TriggerAODList.update( getTriggerEDMList(TriggerFlags.AODEDMSet(),  ConfigFlags.Trigger.EDMVersion) ) 
375

376
        log.info("AOD content set according to the AODEDMSet flag: %s and EDM version %d", TriggerFlags.AODEDMSet(),ConfigFlags.Trigger.EDMVersion)
377

378
379
        log.debug("ESD EDM list: %s", _TriggerESDList)
        log.debug("AOD EDM list: %s", _TriggerAODList)
380
        
381
382
383
384
385
386
387
388
389
390
391
392
        # Highlight what is in AOD list but not in ESD list, as this can cause
        # the "different number of entries in branch" problem, when it is in the
        # AOD list but the empty container per event is not created
        # Just compares keys of dicts, which are the class names, not their string keys in StoreGate
        not_in = [ element for element in  _TriggerAODList if element not in _TriggerESDList ]
        if (len(not_in)>0):
            log.warning("In AOD list but not in ESD list: ")
            log.warning(not_in)
        else:
            log.info("AOD list is subset of ESD list - good.")


393
394
        def _addSlimming(stream, edm):
            from TrigNavTools.TrigNavToolsConfig import navigationThinningSvc
395

396
397
            edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
          
398
399
400
401
402
403
404
405
406
            svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup', 
                                          'result':'HLTResult_HLT',
                                          'features':edmlist})

            from OutputStreamAthenaPool.CreateOutputStreams import registerTrigNavThinningSvc
            registerTrigNavThinningSvc (stream, svc)

            log.info("Configured slimming of HLT for %s", stream)
            print(svc)  # noqa: ATL901
407
408
409
            del edmlist


410
        if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteAOD():
411
            _addSlimming('StreamAOD', _TriggerESDList ) #Use ESD item list also for AOD!
412
413
            log.info("configured navigation slimming for AOD output")
            
414
        if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteESD():
415
416
            _addSlimming('StreamESD', _TriggerESDList )                
            log.info("configured navigation slimming for ESD output")
417

418
        objKeyStore.addManyTypesStreamESD( _TriggerESDList )
419
420
421
422
423
424
425
426
        objKeyStore.addManyTypesStreamAOD( _TriggerAODList )        
            
        return True