HLTTriggerResultGetter.py 20 KB
Newer Older
1
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
2
3

from TriggerJobOpts.TriggerFlags import TriggerFlags
4
from AthenaConfiguration.AllConfigFlags import ConfigFlags
5
6
from AthenaCommon.Logging import logging
from AthenaCommon.GlobalFlags import globalflags
7

8
from AthenaCommon.AppMgr import ServiceMgr
9
10
11
12
13
from RecExConfig.Configured import Configured

from RecExConfig.RecAlgsFlags import recAlgs
from RecExConfig.RecFlags import rec

14
15
from TrigRoiConversion.TrigRoiConversionConf import RoiWriter

16
17
18

class xAODConversionGetter(Configured):
    def configure(self):
19
        from AthenaCommon.AlgSequence import AlgSequence
20
21
22
23
24
25
26
27
        topSequence = AlgSequence()

        #schedule xAOD conversions here
        from TrigBSExtraction.TrigBSExtractionConf import TrigHLTtoxAODConversion
        xaodconverter = TrigHLTtoxAODConversion()
        
        from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
        xaodconverter.Navigation = HLTNavigationOffline()
28

29
30
        from TrigEDMConfig.TriggerEDM import getPreregistrationList
        from TrigEDMConfig.TriggerEDM import getEFRun1BSList,getEFRun2EquivalentList,getL2Run1BSList,getL2Run2EquivalentList
31
32
        xaodconverter.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
        ## if ConfigFlags.Trigger.EDMVersion == 2:
33
        ##     #        if TriggerFlags.doMergedHLTResult():
34
        ##     #if ConfigFlags.Trigger.EDMVersion == 2: #FPP
35
36
37
        ##     xaodconverter.Navigation.ClassesToPreregister = getHLTPreregistrationList()
        ## else:
        ##     xaodconverter.Navigation.ClassesToPreregister = list(set(getL2PreregistrationList()+getEFPreregistrationList()+getHLTPreregistrationList()))
38
39

        #we attempt to convert the entire old navigation (L2+EF)
40
41
42
43
        #xaodconverter.BStoxAOD.ContainersToConvert = list(set(getL2PreregistrationList()+getEFPreregistrationList()))
        # we want only containers from Run 1 with the BS tag
        xaodconverter.BStoxAOD.ContainersToConvert = getL2Run1BSList() + getEFRun1BSList()
        xaodconverter.BStoxAOD.NewContainers = getL2Run2EquivalentList() + getEFRun2EquivalentList()
44
45
46
47

        xaodconverter.HLTResultKey="HLTResult_EF"
        topSequence += xaodconverter

48
49
50
51
52
53
        # define list of HLT xAOD containers to be written to the output root file
        # (previously this was defined in HLTTriggerResultGetter def configure)
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList
        self.xaodlist = {}
        self.xaodlist.update( getTriggerEDMList(TriggerFlags.ESDEDMSet(), 2 ))

54
55
56
57
58
59
        return True
    
        

class ByteStreamUnpackGetter(Configured):
    def configure(self):
60
        log = logging.getLogger("ByteStreamUnpackGetter")
61

62
63
64
65
66
67
68
69
70
71
72
73
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
        hasHLT = TriggerFlags.dataTakingConditions()=='HltOnly' or TriggerFlags.dataTakingConditions()=='FullTrigger'
        if not hasHLT:
            log.info("Will not configure HLT BS unpacking because dataTakingConditions flag indicates HLT was disabled")
            return True

        # Define the decoding sequence
        from TrigHLTResultByteStream.TrigHLTResultByteStreamConf import HLTResultMTByteStreamDecoderAlg
        from TrigOutputHandling.TrigOutputHandlingConf import TriggerEDMDeserialiserAlg
        from AthenaCommon.CFElements import seqAND
        decoder = HLTResultMTByteStreamDecoderAlg()
        deserialiser = TriggerEDMDeserialiserAlg("TrigDeserialiser")
74
        deserialiser.ExtraOutputs += [('xAOD::TrigCompositeContainer' , 'StoreGateSvc+HLTNav_Summary')]
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
        decodingSeq = seqAND("HLTDecodingSeq")
        decodingSeq += decoder  # BS -> HLTResultMT
        decodingSeq += deserialiser  # HLTResultMT -> xAOD

        # Append the decoding sequence to topSequence
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
        topSequence += decodingSeq

        log.debug("Configured HLT result BS decoding sequence")
        return True

class ByteStreamUnpackGetterRun2(Configured):
    def configure(self):

        log = logging.getLogger("ByteStreamUnpackGetterRun2")
91
92
93
94
        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if TriggerFlags.readBS():
95
        log.info( "TriggerFlags.dataTakingConditions: %s", TriggerFlags.dataTakingConditions() )
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
        # in MC this is always FullTrigger
        hasHLT = TriggerFlags.dataTakingConditions()=='HltOnly' or TriggerFlags.dataTakingConditions()=='FullTrigger'
        
        if hasHLT:
            # Decide based on the run number whether to assume a merged, or a
            # split HLT:
            if not TriggerFlags.doMergedHLTResult():
                ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [
                    "HLT::HLTResult/HLTResult_L2",
                    "HLT::HLTResult/HLTResult_EF" ]
            else:
                ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [
                    "HLT::HLTResult/HLTResult_HLT" ]
                pass
            pass

        # BS unpacking
        from TrigBSExtraction.TrigBSExtractionConf import TrigBSExtraction
        extr = TrigBSExtraction()
115
116
117

        # Add fictional output to ensure data dependency in AthenaMT
        extr.ExtraOutputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
118
119
120
121
        
        if hasHLT:
            from TrigNavigation.TrigNavigationConfig import HLTNavigationOffline
            extr.Navigation = HLTNavigationOffline()
122
123

            from TrigEDMConfig.TriggerEDM import getEDMLibraries
124
            extr.Navigation.Dlls = getEDMLibraries()            
125
126

            from TrigEDMConfig.TriggerEDM import getPreregistrationList
127
            extr.Navigation.ClassesToPreregister = getPreregistrationList(ConfigFlags.Trigger.EDMVersion)
128
129
            
            if TriggerFlags.doMergedHLTResult():
130
131
                extr.L2ResultKey=""
                extr.EFResultKey=""
132
133
            else:
                extr.HLTResultKey=""
134
135
136
137

            #
            # Configure DataScouting
            #
138
            from PyUtils.MetaReaderPeeker import metadata
139
140
141
142
143
144
145
            if 'stream' in metadata:
                stream_local = metadata['stream']
                if stream_local.startswith('calibration_DataScouting_') or TriggerFlags.doAlwaysUnpackDSResult():
                    if 'calibration' in stream_local and 'DataScouting_' in stream_local:
                        ds_tag = stream_local[12:27]
                        ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [ "HLT::HLTResult/"+ds_tag ]
                        extr.DSResultKeys += [ ds_tag ]
146
147

        else:
148
149
150
151
152
            #if data doesn't have HLT info set HLTResult keys as empty strings to avoid warnings
            # but the extr alg must run
            extr.L2ResultKey=""
            extr.EFResultKey=""
            extr.HLTResultKey=""
153
            extr.DSResultKeys=[]
154
155
156
157
158
159
160
161
162

        topSequence += extr
        
        from TrigSerializeTP.TrigSerializeTPConf import TrigSerTPTool
        TrigSerToolTP = TrigSerTPTool('TrigSerTPTool')

        from AthenaCommon.AppMgr import ToolSvc
        ToolSvc += TrigSerToolTP
        from TrigEDMConfig.TriggerEDM import getTPList
163
        TrigSerToolTP.TPMap = getTPList((ConfigFlags.Trigger.EDMVersion))
164
165
166
167
168
        
        from TrigSerializeCnvSvc.TrigSerializeCnvSvcConf import TrigSerializeConvHelper
        TrigSerializeConvHelper = TrigSerializeConvHelper(doTP = True)
        ToolSvc += TrigSerializeConvHelper

169
170
171
172
        #
        # Configure L1Topo validation data algorithm
        #
        if hasHLT and TriggerFlags.doMergedHLTResult() and TriggerFlags.writeL1TopoValData() :
173
174
175
176
            # make sure that CTP_RDO is known (see also ATR-14683)
            ServiceMgr.ByteStreamAddressProviderSvc.TypeNames += [
                "CTP_RDO/CTP_RDO"
                ]
177
178
179
180
            from L1TopoValDataCnv.L1TopoValDataCnvConf import xAODMaker__L1TopoValDataCnvAlg
            L1TopoValDataCvnAlg = xAODMaker__L1TopoValDataCnvAlg()
            topSequence += L1TopoValDataCvnAlg

181
182
183
184
        return True


class TrigDecisionGetter(Configured):
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
    def configure(self):
        log = logging.getLogger("TrigDecisionGetter")

        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()

        from TrigDecisionMaker.TrigDecisionMakerConfig import TrigDecisionMakerMT
        tdm = TrigDecisionMakerMT('TrigDecMakerMT')

        if not TriggerFlags.readBS():
            # Construct trigger bits from HLTNav_summary instead of reading from BS
            from TrigOutputHandling.TrigOutputHandlingConf import TriggerBitsMakerTool
            tdm.BitsMakerTool = TriggerBitsMakerTool()

        topSequence += tdm
        log.info('xTrigDecision writing enabled')

        return True

class TrigDecisionGetterRun2(Configured):
205
206
207
    #class to setup the writing or just making of TrigDecisionObject
    def configure(self):
        
208
        log = logging.getLogger("TrigDecisionGetterRun2")
209
210
211
212
213
214
215
216
217
218
219

        from AthenaCommon.AlgSequence import AlgSequence 
        topSequence = AlgSequence()
        
        #if hasOnlyLVL1:
        #from RecExConfig.ObjKeyStore import objKeyStore
        #objKeyStore.addStreamESD('TrigDec::TrigDecision','TrigDecision')
        #objKeyStore.addStreamAOD('TrigDec::TrigDecision','TrigDecision')
        
        from RecExConfig.RecFlags import rec
        if ( rec.doWriteESD() or rec.doWriteAOD() or rec.doESD() or rec.doAOD() ) and \
220
               ( not ( rec.readAOD() or rec.readESD() or rec.doWriteBS()) ):
221
            log.info("Will write TrigDecision object to storegate")
222
            
223
            from TrigDecisionMaker.TrigDecisionMakerConfig import WriteTrigDecision
224
            trigDecWriter = WriteTrigDecision()  # noqa: F841
225

226
#           WritexAODTrigDecision() is called within WriteTrigDecision()
227

228
229
230
231
232
            # inform TD maker that some parts may be missing
            if TriggerFlags.dataTakingConditions()=='Lvl1Only':
                topSequence.TrigDecMaker.doL2=False
                topSequence.TrigDecMaker.doEF=False
                topSequence.TrigDecMaker.doHLT=False
233
234
235
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
236
237
238
            elif TriggerFlags.dataTakingConditions()=='HltOnly':
                from AthenaCommon.AlgSequence import AlgSequence
                topSequence.TrigDecMaker.doL1=False
239
            # Decide based on the doMergedHLTResult to assume a merged, or a
240
241
242
            # split HLT:
            if not TriggerFlags.doMergedHLTResult():
                topSequence.TrigDecMaker.doHLT = False
243
244
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doHLT = False
245
246
247
            else:
                topSequence.TrigDecMaker.doL2 = False
                topSequence.TrigDecMaker.doEF = False
248
249
                topSequence.TrigNavigationCnvAlg.doL2 = False
                topSequence.TrigNavigationCnvAlg.doEF = False
250
251
252
253
                pass
                
        else:
            log.info("Will not write TrigDecision object to storegate")
254
    
255
256
257
258
259
260
261
262
263
264
265
266
        return True
    
    
class HLTTriggerResultGetter(Configured):

    log = logging.getLogger("HLTTriggerResultGetter.py")

    def _AddOPIToESD(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")        
        
        if rec.doESD():
267
268
269
270
            from PyUtils.MetaReaderPeeker import metadata
            if 'stream' in metadata:
                stream = metadata['stream']
                log.debug("the stream found in 'metadata' is "+stream)
271
272
                if "express" in stream:
                    from TrigEDMConfig.TriggerEDM import getTypeAndKey,EDMDetails
273
                    type,key=getTypeAndKey("TrigOperationalInfo#HLT_EXPRESS_OPI_HLT")
274
                    if 'collection'in EDMDetails[type]:
275
                        colltype = EDMDetails[type]['collection']
276
                        log.info("Adding HLT_EXPRESS_OPI_HLT to ESD for stream "+stream)
277
278
279
280
                        from RecExConfig.ObjKeyStore import objKeyStore
                        objKeyStore.addStreamESD(colltype, key)
                    return True
            else:
281
                log.warning("Could not determine stream of bytestream file, not adding HLT_EXPRESS_OPI_HLT to ESD.")
282
283
284
285
286
287
288
        return False

    def configure(self):

        log = logging.getLogger("HLTTriggerResultGetter.py")
        from RecExConfig.ObjKeyStore import objKeyStore

289
290
291
292
        # Set AODFULL for data unless it was set explicitly already
        if TriggerFlags.AODEDMSet.isDefault() and globalflags.DataSource()=='data':
            TriggerFlags.AODEDMSet = 'AODFULL'
            
293
294
        from AthenaCommon.AlgSequence import AlgSequence
        topSequence = AlgSequence()
295
        log.info("BS unpacking (TF.readBS): %d", TriggerFlags.readBS() )
296
        if TriggerFlags.readBS():
297
            if ConfigFlags.Trigger.EDMVersion <= 2:
298
299
300
                bs = ByteStreamUnpackGetterRun2()  # noqa: F841
            else:
                bs = ByteStreamUnpackGetter()  # noqa: F841
301

302
303
        xAODContainers = {}
#        if not recAlgs.doTrigger():      #only convert when running on old data
304
        if ConfigFlags.Trigger.EDMVersion == 1:
305
306
            xaodcnvrt = xAODConversionGetter()
            xAODContainers = xaodcnvrt.xaodlist
307
308

        if recAlgs.doTrigger() or TriggerFlags.doTriggerConfigOnly():
309
            if ConfigFlags.Trigger.EDMVersion <= 2:
310
311
312
                tdt = TrigDecisionGetterRun2()  # noqa: F841
            else:
                tdt = TrigDecisionGetter()  # noqa: F841
313

314
        # Temporary hack to add Run-3 navigation to ESD and AOD
315
        if (rec.doESD() or rec.doAOD()) and ConfigFlags.Trigger.EDMVersion == 3:
316
317
318
319
320
321
322
323
324
            # The hack with wildcards is needed for BS->ESD because we don't know the exact keys
            # of HLT navigation containers before unpacking them from the BS event.
            objKeyStore._store['streamESD'].allowWildCard(True)
            objKeyStore._store['streamAOD'].allowWildCard(True)
            objKeyStore.addManyTypesStreamESD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])
            objKeyStore.addManyTypesStreamAOD(['xAOD::TrigCompositeContainer#HLTNav*',
                                               'xAOD::TrigCompositeAuxContainer#HLTNav*'])

325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
        # TrigJetRec additions
        if rec.doWriteESD():
            objKeyStore.addStreamESD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamESD("JetMomentMap","TrigJetRecMomentMap")

        if rec.doWriteAOD():
            objKeyStore.addStreamAOD("JetKeyDescriptor","JetKeyMap")
            objKeyStore.addStreamAOD("JetMomentMap","TrigJetRecMomentMap")
                    
        # ID truth
        if not rec.readESD() and (not rec.readAOD()) and TriggerFlags.doID() \
                and rec.doTruth():
            try:
                from TrigInDetTruthAlgs.TrigInDetTruthAlgsConfig import \
                    TrigIDTruthMaker
                topSequence += TrigIDTruthMaker()
            except Exception:
                log.warning( "Couldn't set up the trigger ID truth maker" )
                pass

        if rec.doESD() or rec.doAOD():
            from TrigEDMConfig.TriggerEDM import getTrigIDTruthList
            objKeyStore.addManyTypesStreamESD(getTrigIDTruthList(TriggerFlags.ESDEDMSet()))
            objKeyStore.addManyTypesStreamAOD(getTrigIDTruthList(TriggerFlags.AODEDMSet()))

350
351
352
353
354
355
        if (rec.doESD() or rec.doAOD()) and TriggerFlags.writeL1TopoValData():
            objKeyStore.addManyTypesStreamESD(['xAOD::TrigCompositeContainer#HLT_xAOD__TrigCompositeContainer_L1TopoValData',
                                               'xAOD::TrigCompositeAuxContainer#HLT_xAOD__TrigCompositeContainer_L1TopoValDataAux.'])
            objKeyStore.addManyTypesStreamAOD(['xAOD::TrigCompositeContainer#HLT_xAOD__TrigCompositeContainer_L1TopoValData',
                                               'xAOD::TrigCompositeAuxContainer#HLT_xAOD__TrigCompositeContainer_L1TopoValDataAux.'])
            log.debug("HLT_xAOD__TrigCompositeContainer_L1TopoValData(Aux.) for L1Topo validation added to the data.")
356

357
        if rec.doAOD() or rec.doWriteAOD():
358
359
            # schedule the RoiDescriptorStore conversion
            # log.warning( "HLTTriggerResultGetter - setting up RoiWriter" )
360
361
362
363
            roiWriter = RoiWriter()
            # Add fictional input to ensure data dependency in AthenaMT
            roiWriter.ExtraInputs += [("TrigBSExtractionOutput", "StoreGateSvc+TrigBSExtractionOutput")]
            topSequence += roiWriter
364
            # write out the RoiDescriptorStores
365
            from TrigEDMConfig.TriggerEDMRun2 import TriggerRoiList
366
367
            objKeyStore.addManyTypesStreamAOD( TriggerRoiList )

368
369
370
        #Are we adding operational info objects in ESD?
        added=self._AddOPIToESD()
        if added:
371
            log.debug("Operational Info object HLT_EXPRESS_OPI_HLT with extra information about express stream prescaling added to the data.")
372
373
374
375
376
377
        


        # ESD objects definitions
        _TriggerESDList = {}

378
        from TrigEDMConfig.TriggerEDM import getTriggerEDMList 
379
380
381
382
383
        # we have to store xAOD containers in the root file, NOT AOD,
        # if the xAOD container list is not empty
        if(xAODContainers):
            _TriggerESDList.update( xAODContainers )
        else:
384
            _TriggerESDList.update( getTriggerEDMList(TriggerFlags.ESDEDMSet(),  ConfigFlags.Trigger.EDMVersion) ) 
385
        
386
        log.info("ESD content set according to the ESDEDMSet flag: %s and EDM version %d", TriggerFlags.ESDEDMSet(), ConfigFlags.Trigger.EDMVersion)
387
388
389
390

        # AOD objects choice
        _TriggerAODList = {}
        
391
        #from TrigEDMConfig.TriggerEDM import getAODList    
392
        _TriggerAODList.update( getTriggerEDMList(TriggerFlags.AODEDMSet(),  ConfigFlags.Trigger.EDMVersion) ) 
393

394
        log.info("AOD content set according to the AODEDMSet flag: %s and EDM version %d", TriggerFlags.AODEDMSet(),ConfigFlags.Trigger.EDMVersion)
395

396
397
        log.debug("ESD EDM list: %s", _TriggerESDList)
        log.debug("AOD EDM list: %s", _TriggerAODList)
398
        
399
400
401
402
403
404
405
406
407
408
409
410
        # Highlight what is in AOD list but not in ESD list, as this can cause
        # the "different number of entries in branch" problem, when it is in the
        # AOD list but the empty container per event is not created
        # Just compares keys of dicts, which are the class names, not their string keys in StoreGate
        not_in = [ element for element in  _TriggerAODList if element not in _TriggerESDList ]
        if (len(not_in)>0):
            log.warning("In AOD list but not in ESD list: ")
            log.warning(not_in)
        else:
            log.info("AOD list is subset of ESD list - good.")


411
412
        def _addSlimming(stream, edm):
            from TrigNavTools.TrigNavToolsConfig import navigationThinningSvc
413

414
415
            edmlist = list(y.split('-')[0] for x in edm.values() for y in x) #flatten names
          
416
417
418
419
420
421
422
423
424
            svc = navigationThinningSvc ({'name':'HLTNav_%s'%stream, 'mode':'cleanup', 
                                          'result':'HLTResult_HLT',
                                          'features':edmlist})

            from OutputStreamAthenaPool.CreateOutputStreams import registerTrigNavThinningSvc
            registerTrigNavThinningSvc (stream, svc)

            log.info("Configured slimming of HLT for %s", stream)
            print(svc)  # noqa: ATL901
425
426
427
            del edmlist


428
        if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteAOD():
429
            _addSlimming('StreamAOD', _TriggerESDList ) #Use ESD item list also for AOD!
430
431
            log.info("configured navigation slimming for AOD output")
            
432
        if TriggerFlags.doNavigationSlimming() and rec.readRDO() and rec.doWriteESD():
433
434
            _addSlimming('StreamESD', _TriggerESDList )                
            log.info("configured navigation slimming for ESD output")
435
436
437



438
439

        objKeyStore.addManyTypesStreamESD( _TriggerESDList )
440
441
442
443
444
445
446
447
        objKeyStore.addManyTypesStreamAOD( _TriggerAODList )        
            
        return True