HLTTriggerResultGetter.py 19 KB
Newer Older
1
# Copyright (C) 2002-2021 CERN for the benefit of the ATLAS collaboration
2
3

from TriggerJobOpts.TriggerFlags import TriggerFlags
4
from AthenaConfiguration.AllConfigFlags import ConfigFlags
5
6
from AthenaCommon.Logging import logging
from AthenaCommon.GlobalFlags import globalflags
7

8
from AthenaCommon.AppMgr import ServiceMgr
9
10
11
12
from RecExConfig.Configured import Configured

from RecExConfig.RecFlags import rec

13
14
from TrigRoiConversion.TrigRoiConversionConf import RoiWriter

15
16
17

class xAODConversionGetter(Configured):
    def configure(self):
18
        from AthenaCommon.AlgSequence import AlgSequence
19
20
21
22
23
24
25
26
        topSequence = AlgSequence()

        #schedule xAOD conversions here
        from TrigBSExtraction.TrigBSExtractionConf import TrigHLTtoxAODConversion
        xaodconverter = TrigHLTtoxAODConversion()
        
        from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
        xaodconverter.Navigation = HLTNavigationOffline()
27

28
29
        from TrigEDMConfig.TriggerEDM import getPreregistrationList
        from TrigEDMConfig.TriggerEDM import getEFRun1BSList,getEFRun2EquivalentList,getL2Run1BSList,getL2Run2EquivalentList
30
31
        xaodconverter.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
        ## if ConfigFlags.Trigger.EDMVersion == 2:
32
        ##     #        if TriggerFlags.doMergedHLTResult():
33
        ##     #if ConfigFlags.Trigger.EDMVersion == 2: #FPP
34
35
36
        ##     xaodconverter.Navigation.ClassesToPreregister = getHLTPreregistrationList()
        ## else:
        ##     xaodconverter.Navigation.ClassesToPreregister = list(set(getL2PreregistrationList()+getEFPreregistrationList()+getHLTPreregistrationList()))
37
38

        #we attempt to convert the entire old navigation (L2+EF)
39
40
41
42
        #xaodconverter.BStoxAOD.ContainersToConvert = list(set(getL2PreregistrationList()+getEFPreregistrationList()))
        # we want only containers from Run 1 with the BS tag
        xaodconverter.BStoxAOD.ContainersToConvert = getL2Run1BSList() + getEFRun1BSList()
        xaodconverter.BStoxAOD.NewContainers = getL2Run2EquivalentList() + getEFRun2EquivalentList()
43
44
45
46

        xaodconverter.HLTResultKey="HLTResult_EF"
        topSequence += xaodconverter

47
48
49
50
51
52
        # define list of HLT xAOD containers to be written to the output root file
        # (previously this was defined in HLTTriggerResultGetter def configure)
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList
        self.xaodlist = {}
        self.xaodlist.update( getTriggerEDMList(TriggerFlags.ESDEDMSet(), 2 ))

53
54
55
56
57
58
        return True
    
        

class ByteStreamUnpackGetter(Configured):
    def configure(self):
59
        log = logging.getLogger("ByteStreamUnpackGetter")
60

61
62
63
64
65
66
67
68
69
70
71
72
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
        hasHLT = TriggerFlags.dataTakingConditions()=='HltOnly' or TriggerFlags.dataTakingConditions()=='FullTrigger'
        if not hasHLT:
            log.info("Will not configure HLT BS unpacking because dataTakingConditions flag indicates HLT was disabled")
            return True

        # Define the decoding sequence
        from TrigHLTResultByteStream.TrigHLTResultByteStreamConf import HLTResultMTByteStreamDecoderAlg
        from TrigOutputHandling.TrigOutputHandlingConf import TriggerEDMDeserialiserAlg
        from AthenaCommon.CFElements import seqAND
        decoder = HLTResultMTByteStreamDecoderAlg()
        deserialiser = TriggerEDMDeserialiserAlg("TrigDeserialiser")
73
        deserialiser.ExtraOutputs += [('xAOD::TrigCompositeContainer' , 'StoreGateSvc+HLTNav_Summary')]
74
75
76
77
78
79
80
81
82
83
84
85
        decodingSeq = seqAND("HLTDecodingSeq")
        decodingSeq += decoder  # BS -> HLTResultMT
        decodingSeq += deserialiser  # HLTResultMT -> xAOD

        # Append the decoding sequence to topSequence
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
        topSequence += decodingSeq

        log.debug("Configured HLT result BS decoding sequence")
        return True

86
class ByteStreamUnpackGetterRun1or2(Configured):
87
88
    def configure(self):

89
        log = logging.getLogger("ByteStreamUnpackGetterRun1or2")
90
91
92
93
        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if TriggerFlags.readBS():
94
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
        # in MC this is always FullTrigger
        hasHLT = TriggerFlags.dataTakingConditions()=='HltOnly' or TriggerFlags.dataTakingConditions()=='FullTrigger'
        
        if hasHLT:
            # Decide based on the run number whether to assume a merged, or a
            # split HLT:
            if not TriggerFlags.doMergedHLTResult():
                ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [
                    "HLT::HLTResult/HLTResult_L2",
                    "HLT::HLTResult/HLTResult_EF" ]
            else:
                ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [
                    "HLT::HLTResult/HLTResult_HLT" ]
                pass
            pass

        # BS unpacking
        from TrigBSExtraction.TrigBSExtractionConf import TrigBSExtraction
        extr = TrigBSExtraction()
114
115
116

        # Add fictional output to ensure data dependency in AthenaMT
        extr.ExtraOutputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
117
118
119
120
        
        if hasHLT:
            from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
            extr.Navigation = HLTNavigationOffline()
121
122

            from TrigEDMConfig.TriggerEDM import getEDMLibraries
123
            extr.Navigation.Dlls = getEDMLibraries()            
124
125

            from TrigEDMConfig.TriggerEDM import getPreregistrationList
126
            extr.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
127
128
            
            if TriggerFlags.doMergedHLTResult():
129
130
                extr.L2ResultKey=""
                extr.EFResultKey=""
131
132
            else:
                extr.HLTResultKey=""
133
134
135
136

            #
            # Configure DataScouting
            #
137
            from PyUtils.MetaReaderPeeker import metadata
138
139
140
141
142
143
144
            if 'stream' in metadata:
                stream_local = metadata['stream']
                if stream_local.startswith('calibration_DataScouting_') or TriggerFlags.doAlwaysUnpackDSResult():
                    if 'calibration' in stream_local and 'DataScouting_' in stream_local:
                        ds_tag = stream_local[12:27]
                        ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "HLT::HLTResult/"+ds_tag ]
                        extr.DSResultKeys += [ ds_tag ]
145
146

        else:
147
148
149
150
151
            #if data doesn't have HLT info set HLTResult keys as empty strings to avoid warnings
            # but the extr alg must run
            extr.L2ResultKey=""
            extr.EFResultKey=""
            extr.HLTResultKey=""
152
            extr.DSResultKeys=[]
153
154
155
156
157
158
159
160
161

        topSequence += extr
        
        from TrigSerializeTP.TrigSerializeTPConf import TrigSerTPTool
        TrigSerToolTP = TrigSerTPTool('TrigSerTPTool')

        from AthenaCommon.AppMgr import ToolSvc
        ToolSvc += TrigSerToolTP
        from TrigEDMConfig.TriggerEDM import getTPList
162
        TrigSerToolTP.TPMap = getTPList((ConfigFlags.Trigger.EDMVersion))
163
164
165
166
167
168
169
170
171
        
        from TrigSerializeCnvSvc.TrigSerializeCnvSvcConf import TrigSerializeConvHelper
        TrigSerializeConvHelper = TrigSerializeConvHelper(doTP = True)
        ToolSvc += TrigSerializeConvHelper

        return True


class TrigDecisionGetter(Configured):
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    def configure(self):
        log = logging.getLogger("TrigDecisionGetter")

        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()

        from TrigDecisionMaker.TrigDecisionMakerConfig import TrigDecisionMakerMT
        tdm = TrigDecisionMakerMT('TrigDecMakerMT')

        if not TriggerFlags.readBS():
            # Construct trigger bits from HLTNav_summary instead of reading from BS
            from TrigOutputHandling.TrigOutputHandlingConf import TriggerBitsMakerTool
            tdm.BitsMakerTool = TriggerBitsMakerTool()

        topSequence += tdm
        log.info('xTrigDecision writing enabled')

        return True

191
class TrigDecisionGetterRun1or2(Configured):
192
193
194
    #class to setup the writing or just making of TrigDecisionObject
    def configure(self):
        
195
        log = logging.getLogger("TrigDecisionGetterRun1or2")
196
197
198
199
200
201
202
203
204
205
206

        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if hasOnlyLVL1:
        #from RecExConfig.ObjKeyStore import objKeyStore
        #objKeyStore.addStreamESD('TrigDec::TrigDecision','TrigDecision')
        #objKeyStore.addStreamAOD('TrigDec::TrigDecision','TrigDecision')
        
        from RecExConfig.RecFlags import rec
        if ( rec.doWriteESD() or rec.doWriteAOD() or rec.doESD() or rec.doAOD() ) and \
207
               ( not ( rec.readAOD() or rec.readESD() or rec.doWriteBS()) ):
208
            log.info("Will write TrigDecision object to storegate")
209
            
210
            from TrigDecisionMaker.TrigDecisionMakerConfig import WriteTrigDecision
211
            trigDecWriter = WriteTrigDecision()  # noqa: F841
212

213
#           WritexAODTrigDecision() is called within WriteTrigDecision()
214

215
216
217
218
219
            # inform TD maker that some parts may be missing
            if TriggerFlags.dataTakingConditions()=='Lvl1Only':
                topSequence.TrigDecMaker.doL2=False
                topSequence.TrigDecMaker.doEF=False
                topSequence.TrigDecMaker.doHLT=False
220
221
222
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
223
224
225
            elif TriggerFlags.dataTakingConditions()=='HltOnly':
                from AthenaCommon.AlgSequence import AlgSequence
                topSequence.TrigDecMaker.doL1=False
226
            # Decide based on the doMergedHLTResult to assume a merged, or a
227
228
229
            # split HLT:
            if not TriggerFlags.doMergedHLTResult():
                topSequence.TrigDecMaker.doHLT = False
230
231
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
232
233
234
            else:
                topSequence.TrigDecMaker.doL2 = False
                topSequence.TrigDecMaker.doEF = False
235
236
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
237
238
239
240
                pass
                
        else:
            log.info("Will not write TrigDecision object to storegate")
241
    
242
243
244
245
246
247
248
249
250
251
252
253
        return True
    
    
class HLTTriggerResultGetter(Configured):

    log = logging.getLogger("HLTTriggerResultGetter.py")

    def _AddOPIToESD(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")        
        
        if rec.doESD():
254
255
256
257
            from PyUtils.MetaReaderPeeker import metadata
            if 'stream' in metadata:
                stream = metadata['stream']
                log.debug("the stream found in 'metadata' is "+stream)
258
259
                if "express" in stream:
                    from TrigEDMConfig.TriggerEDM import getTypeAndKey,EDMDetails
260
                    type,key=getTypeAndKey("TrigOperationalInfo#HLT_EXPRESS_OPI_HLT")
261
                    if 'collection'in EDMDetails[type]:
262
                        colltype = EDMDetails[type]['collection']
263
                        log.info("Adding HLT_EXPRESS_OPI_HLT to ESD for stream "+stream)
264
265
266
267
                        from RecExConfig.ObjKeyStore import objKeyStore
                        objKeyStore.addStreamESD(colltype, key)
                    return True
            else:
268
                log.warning("Could not determine stream of bytestream file, not adding HLT_EXPRESS_OPI_HLT to ESD.")
269
270
271
272
273
274
275
        return False

    def configure(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")
        from RecExConfig.ObjKeyStore import objKeyStore

276
277
278
279
        # Set AODFULL for data unless it was set explicitly already
        if TriggerFlags.AODEDMSet.isDefault() and globalflags.DataSource()=='data':
            TriggerFlags.AODEDMSet = 'AODFULL'
            
280
281
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
282
        log.info("BS unpacking (TF.readBS): %d", TriggerFlags.readBS() )
283
        if TriggerFlags.readBS():
284
285
286
287
            if ConfigFlags.Trigger.EDMVersion == 1 or \
               ConfigFlags.Trigger.EDMVersion == 2:
                bs = ByteStreamUnpackGetterRun1or2()  # noqa: F841
            elif ConfigFlags.Trigger.EDMVersion >=3:
288
                bs = ByteStreamUnpackGetter()  # noqa: F841
289
290
            else:
                raise RuntimeError("Invalid EDMVersion=%s " % ConfigFlags.Trigger.EDMVersion)
291

292
        xAODContainers = {}
293

294
        if ConfigFlags.Trigger.EDMVersion == 1:
295
296
            xaodcnvrt = xAODConversionGetter()
            xAODContainers = xaodcnvrt.xaodlist
297

298
299
300
301
302
303
304
305
306
        if ConfigFlags.Trigger.EDMVersion == 1 or \
           ConfigFlags.Trigger.EDMVersion == 2:
            if rec.doTrigger() or TriggerFlags.doTriggerConfigOnly():
                tdt = TrigDecisionGetterRun1or2()  # noqa: F841
        elif ConfigFlags.Trigger.EDMVersion >= 3:
            if TriggerFlags.readBS():
                tdt = TrigDecisionGetter()  # noqa: F841
        else:
            raise RuntimeError("Invalid EDMVersion=%s " % ConfigFlags.Trigger.EDMVersion)
307

308
        # Temporary hack to add Run-3 navigation to ESD and AOD
309
        if (rec.doESD() or rec.doAOD()) and ConfigFlags.Trigger.EDMVersion == 3:
310
311
312
313
314
315
316
317
318
            # The hack with wildcards is needed for BS->ESD because we don't know the exact keys
            # of HLT navigation containers before unpacking them from the BS event.
            objKeyStore._store['streamESD'].allowWildCard(True)
            objKeyStore._store['streamAOD'].allowWildCard(True)
            objKeyStore.addManyTypesStreamESD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])
            objKeyStore.addManyTypesStreamAOD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])

319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
        # TrigJetRec additions
        if rec.doWriteESD():
            objKeyStore.addStreamESD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamESD("JetMomentMap","TrigJetRecMomentMap")

        if rec.doWriteAOD():
            objKeyStore.addStreamAOD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamAOD("JetMomentMap","TrigJetRecMomentMap")
                    
        # ID truth
        if not rec.readESD() and (not rec.readAOD()) and TriggerFlags.doID() \
                and rec.doTruth():
            try:
                from TrigInDetTruthAlgs.TrigInDetTruthAlgsConfig import \
                    TrigIDTruthMaker
                topSequence += TrigIDTruthMaker()
            except Exception:
                log.warning( "Couldn't set up the trigger ID truth maker" )
                pass

        if rec.doESD() or rec.doAOD():
            from TrigEDMConfig.TriggerEDM import getTrigIDTruthList
            objKeyStore.addManyTypesStreamESD(getTrigIDTruthList(TriggerFlags.ESDEDMSet()))
            objKeyStore.addManyTypesStreamAOD(getTrigIDTruthList(TriggerFlags.AODEDMSet()))

344
        if rec.doAOD() or rec.doWriteAOD():
345
346
            # schedule the RoiDescriptorStore conversion
            # log.warning( "HLTTriggerResultGetter - setting up RoiWriter" )
347
348
349
350
            roiWriter = RoiWriter()
            # Add fictional input to ensure data dependency in AthenaMT
            roiWriter.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
            topSequence += roiWriter
351
            # write out the RoiDescriptorStores
352
            from TrigEDMConfig.TriggerEDMRun2 import TriggerRoiList
353
354
            objKeyStore.addManyTypesStreamAOD( TriggerRoiList )

355
356
357
        #Are we adding operational info objects in ESD?
        added=self._AddOPIToESD()
        if added:
358
            log.debug("Operational Info object HLT_EXPRESS_OPI_HLT with extra information about express stream prescaling added to the data.")
359
360
361
362
363
364
        


        # ESD objects definitions
        _TriggerESDList = {}

365
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList 
366
367
368
369
370
        # we have to store xAOD containers in the root file, NOT AOD,
        # if the xAOD container list is not empty
        if(xAODContainers):
            _TriggerESDList.update( xAODContainers )
        else:
371
            _TriggerESDList.update( getTriggerEDMList(TriggerFlags.ESDEDMSet(),  ConfigFlags.Trigger.EDMVersion) ) 
372
        
373
        log.info("ESD content set according to the ESDEDMSet flag: %s and EDM version %d", TriggerFlags.ESDEDMSet(), ConfigFlags.Trigger.EDMVersion)
374
375
376
377

        # AOD objects choice
        _TriggerAODList = {}
        
378
        #from TrigEDMConfig.TriggerEDM import getAODList    
379
        _TriggerAODList.update( getTriggerEDMList(TriggerFlags.AODEDMSet(),  ConfigFlags.Trigger.EDMVersion) ) 
380

381
        log.info("AOD content set according to the AODEDMSet flag: %s and EDM version %d", TriggerFlags.AODEDMSet(),ConfigFlags.Trigger.EDMVersion)
382

383
384
        log.debug("ESD EDM list: %s", _TriggerESDList)
        log.debug("AOD EDM list: %s", _TriggerAODList)
385
        
386
387
388
389
390
391
392
393
394
395
396
397
        # Highlight what is in AOD list but not in ESD list, as this can cause
        # the "different number of entries in branch" problem, when it is in the
        # AOD list but the empty container per event is not created
        # Just compares keys of dicts, which are the class names, not their string keys in StoreGate
        not_in = [ element for element in  _TriggerAODList if element not in _TriggerESDList ]
        if (len(not_in)>0):
            log.warning("In AOD list but not in ESD list: ")
            log.warning(not_in)
        else:
            log.info("AOD list is subset of ESD list - good.")


398
399
        def _addSlimming(stream, edm):
            from TrigNavTools.TrigNavToolsConfig import navigationThinningSvc
400

401
402
            edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
          
403
404
405
406
407
408
409
410
411
            svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup', 
                                          'result':'HLTResult_HLT',
                                          'features':edmlist})

            from OutputStreamAthenaPool.CreateOutputStreams import registerTrigNavThinningSvc
            registerTrigNavThinningSvc (stream, svc)

            log.info("Configured slimming of HLT for %s", stream)
            print(svc)  # noqa: ATL901
412
413
414
            del edmlist


415
        if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteAOD():
416
            _addSlimming('StreamAOD', _TriggerESDList ) #Use ESD item list also for AOD!
417
418
            log.info("configured navigation slimming for AOD output")
            
419
        if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteESD():
420
421
            _addSlimming('StreamESD', _TriggerESDList )                
            log.info("configured navigation slimming for ESD output")
422
423
424



425
426

        objKeyStore.addManyTypesStreamESD( _TriggerESDList )
427
428
429
430
431
432
433
434
        objKeyStore.addManyTypesStreamAOD( _TriggerAODList )        
            
        return True