TrigConfigSvcConfig.py 14.9 KB
Newer Older
1
# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
2

3
4
5
6
7
from TrigConfigSvc.TrigConfigSvcConf import (TrigConf__LVL1ConfigSvc,
                                             TrigConf__L1TopoConfigSvc,
                                             TrigConf__HLTConfigSvc,
                                             TrigConf__DSConfigSvc,
                                             TrigConf__TrigConfigSvc)
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# Hack: xml.etree module is hidden by mistake in LCG56c
from PyUtils.xmldict import import_etree
etree = import_etree()
import xml.etree.cElementTree as ET

from os.path import exists, join, abspath

from AthenaCommon.Logging import logging  # loads logger
from PyUtils.Decorators import memoize

@memoize
def findFileInXMLPATH(filename):
    """ Use XMLPATH to find files (cache result through memoize decorator)"""

    if filename=="NONE":
        return filename

    filename = str(filename)

    mlog = logging.getLogger("TrigConfigSvcConfig.py")
    mlog.debug("Searching XML config file for HLT")
30
    if filename.find('./') == 0: ## this expected to be local file, name starts from ./
31
32
33
34
35
36
37
38
39
40
        return filename
    else:
        mlog.debug("Nonlocal XML config file for HLT")
        from os import environ
        ## even if ./ not as file name prefix look first in PWD
        if exists(filename):
            mlog.info(filename+" XML configuration file taken from working directory")
            return filename

        ## search XMLPATH path
41
        if 'XMLPATH' not in environ: ## XMLPATH is not known ... no search is performed
42
43
44
45
            mlog.info("XML file: "+filename + " not found and XMLPATH not given" )
            return filename

        xmlpath = environ['XMLPATH']
46
        paths = xmlpath.split(":")
47
48
49
50
51
52
53
54
55
56
57
58
        for path in paths:

            test = join(path, filename)
            if exists(test):
                mlog.info("Found XML file: " + abspath(test))
                return abspath(test)

            test = join(path, "TriggerMenuXML",filename)
            if exists(test):
                mlog.info("Found XML file: " + abspath(test))
                return abspath(test)

59
60
61
62
63
            test = join(path, "TriggerMenuMT",filename)
            if exists(test):
                mlog.info("Found XML file: " + abspath(test))
                return abspath(test)

64
65
66
67
68
69
70
71
72
        return filename



class DefaultHLTConfigSvc( TrigConf__HLTConfigSvc ):
    #__slots__ = []
    def __init__( self, name="HLTConfigSvc" ):
        super( DefaultHLTConfigSvc, self ).__init__( name )  #

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
    def getAlgorithmsRun2(self):
        """Produces pair of lists with algorithms scheduled for  L2 and EF"""

        mlog = logging.getLogger("TrigConfigSvcConfig.py")
        from TriggerJobOpts.TriggerFlags import TriggerFlags
        if TriggerFlags.readMenuFromTriggerDb():
            from TrigConfigSvc.TrigConfigSvcUtils import getAlgorithmsForMenuRun2
            mlog.info("Will load algos from DB")
            allalgs = getAlgorithmsForMenuRun2(TriggerFlags.triggerDbConnection(),TriggerFlags.triggerDbKeys()[0])
        else:
            mlog.info("Will load algos from xml")
            allalgs = []
            doc = ET.parse(self.XMLMenuFile)
            algs = self.getAllAlgorithms(doc)
            l2TEs, efTEs = self.getTEsByLevel(doc)
88

89
90
91
92
93
94
95
            for te in l2TEs:
                if te in algs.keys():
                    allalgs += algs[te]

            for te in efTEs:
                if te in algs.keys():
                    allalgs += algs[te]
96

97
98
99
        return allalgs


100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

    def getAlgorithmsByLevel(self):
        """Produces pair of lists with algorithms scheduled for  L2 and EF"""

        mlog = logging.getLogger("TrigConfigSvcConfig.py")
        from TriggerJobOpts.TriggerFlags import TriggerFlags
        if TriggerFlags.readMenuFromTriggerDb():
            from TrigConfigSvc.TrigConfigSvcUtils import getAlgorithmsForMenu
            mlog.info("Will load algos from DB")
            l2algs, efalgs = getAlgorithmsForMenu(TriggerFlags.triggerDbConnection(),TriggerFlags.triggerDbKeys()[0])
        else:
            mlog.info("Will load algos from xml")
            l2algs = []
            efalgs = []
            doc = ET.parse(self.XMLMenuFile)
            algs = self.getAllAlgorithms(doc)
            l2TEs, efTEs = self.getTEsByLevel(doc)
117

118
119
120
121
122
123
124
            for te in l2TEs:
                if te in algs.keys():
                    l2algs += algs[te]

            for te in efTEs:
                if te in algs.keys():
                    efalgs += algs[te]
125

126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
        return l2algs, efalgs

    def getTEsByLevel(self, doc = None):
        """Produce 2 lists of all TEs used in signatures of level L2 and EF"""
        l2TEs = []
        efTEs = []
        #print "INFO parsing: " + self.XMLMenuFile
        if not doc: doc = ET.parse(self.XMLMenuFile)

        #print "INFO got chains " + str(chainlist)
        for ch in doc.getiterator("CHAIN"):
            for te in ch.getiterator("TRIGGERELEMENT"):
                if ch.get("level") == "L2":
                    l2TEs.append(te.get("te_name"))
                else:
                    efTEs.append(te.get("te_name"))

        # having TEs of each level which are mentioned in chains we need to scan sequences now
        # and find if there are some recursive sequences to be called
        seqinp = self.getAllSequenceInputs()
        for te in l2TEs:
            if te in seqinp.keys():
                l2TEs.extend(seqinp[te]) # note that this recursive (we extend list on which we loop)

        for te in efTEs:
            if te in seqinp.keys():
                efTEs.extend(seqinp[te]) # note that this recursive (we extend list on which we loop)

        # make them unique
        l2TEs = self.unique(l2TEs)
        efTEs = self.unique(efTEs)

        # eliminate from EF list those TEs mentioned in the L2
        temp = []
        for te in efTEs:
            if te not in l2TEs:
                temp.append(te)
        efTEs = temp
        return l2TEs, efTEs

    def unique(self, l):
        #d = {}
        #for item in l:
        #    d[item] = item
        #return d.keys()
        return list(set(l))


    def getAllSequenceInputs (self, doc = None):
        """ Produce dictionaries where key is outout TE name and values are tuples of TEs """
        if not doc: doc = ET.parse(self.XMLMenuFile)
        inp = {}
        for seq in doc.getiterator("SEQUENCE"):
            #print "INFO Discovered algorithms in the sequence: " + seq.getAttribute("algorithm")
            inp[seq.get("output")] = seq.get("input").split()
        return inp

    def getAllAlgorithms(self, doc = None):
        """ Produce dictionary where key is output TE name of the sequence and value is a list of algos in sequence"""
        #print "INFO parsing: " + self.XMLMenuFile
        if not doc: doc = ET.parse(self.XMLMenuFile)
        #print "INFO getting sequences "
        algos = {}
        for seq in doc.getiterator("SEQUENCE"):
            algos[seq.get("output")] = seq.get("algorithm").split()
191

192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
        return algos



class TestingHLTConfigSvc ( DefaultHLTConfigSvc ):
    def __init__( self, name="HLTConfigSvc" ):
        super( TestingHLTConfigSvc, self ).__init__( name )  #

    def setDefaults(self, handle):
        from AthenaCommon.Constants import VERBOSE
        handle.OutputLevel=VERBOSE


class HLTConfigSvc ( DefaultHLTConfigSvc ):
    def __init__( self, name="HLTConfigSvc" ):
        super( HLTConfigSvc, self ).__init__( name )  #

    def setDefaults(self, handle):
        pass



class DefaultLVL1ConfigSvc ( TrigConf__LVL1ConfigSvc ):
    __slots__ = []
    def __init__( self, name="LVL1ConfigSvc" ):
        super( DefaultLVL1ConfigSvc, self ).__init__( name )  #

    def setDefaults(self, handle):
        pass


class TestingLVL1ConfigSvc ( TrigConf__LVL1ConfigSvc ):
    __slots__ = []
    def __init__( self, name="LVL1ConfigSvc" ):
        super( TestingLVL1ConfigSvc, self ).__init__( name )  #

    def setDefaults(self, handle):
        from AthenaCommon.Constants import VERBOSE
        handle.OutputLevel=VERBOSE




class LVL1ConfigSvc ( DefaultLVL1ConfigSvc ):
    __slots__ = []
    def __init__( self, name="LVL1ConfigSvc" ):
        super( LVL1ConfigSvc, self ).__init__( name )

    @property
    def XMLFile(self):
        '''The xml file name for backward compatibility with old property name'''
        return  self.XMLMenuFile

    @XMLFile.setter
    def XMLFile(self, xmlfile):
        log = logging.getLogger("LVL1ConfigSvc")
        log.warning( "LVL1ConfigSvc property XMLFile will soon be deprecated. Please use XMLMenuFile instead" )
        self.XMLMenuFile = xmlfile
250

251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
    def setDefaults(self, handle):
        pass

class L1TopoConfigSvc ( TrigConf__L1TopoConfigSvc ) :
    """L1Topo configuration"""
    __slots__ = []
    def __init__(self, name="L1TopoConfigSvc" ):
        super (L1TopoConfigSvc, self).__init__(name)

    def setDetaults(self, handle):
        pass

class DSConfigSvc ( TrigConf__DSConfigSvc ) :
    """Detector Store implementation of the TrigConfigSvc"""
    __slots__ = []
    def __init__(self, name="DSConfigSvc" ):
        super (DSConfigSvc, self).__init__(name)

    def setDetaults(self, handle):
        pass

class TrigConfigSvc( TrigConf__TrigConfigSvc ):
    __slots__ = []
    def __init__(self, name="TrigConfigSvc" ):
        super( TrigConfigSvc, self).__init__(name)

    def setDefaults(self, handle):
        pass


# singleton class used for all
# clients of any TrigConfigSvc
# usage:
# from TrigConfigSvc.TrigConfigSvcConfig import SetupTrigConfigSvc
# svc = SetupTrigConfigSvc()
# svc.SetStates( ["ds"] )
# svc.InitialiseSvc()
#
# after InitialiseSvc() is called, the state is fixed
# possible states are: ds, xml
Tomasz Bold's avatar
Tomasz Bold committed
291

292

293
class SetupTrigConfigSvc(object):
294
295
    """ A python singleton class for configuring the trigger configuration services"""

296
    class __impl(object):
297
298
299
300
301
302
        """ Implementation of the singleton interface """

        def __init__(self):
            """
            state == xml -> read the trigger configuration from 2 xml files, one for L1, one for HLT
            stats == ds  -> read the trigger configuration from the detector store = esd header
303
            state == none -> service is not directly serving the run3 configuration
304
305
            """
            self.states = ["xml"]
306
            self.allowedStates = set(['none','xml','ds'])
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
            self.initialised = False

            from AthenaCommon.Logging import logging
            self.mlog = logging.getLogger("TrigConfigSvcConfig.py")

            # svc properties:
            self.l1topoXmlFile = 'NONE'
            self.hltXmlFile = 'HLT_XML_FILE_NOT_SET'
            self.l1XmlFile  = 'L1_XML_FILE_NOT_SET'

        def spam(self):
            """ Test method, return singleton id """
            return id(self)


        def SetStates(self, state):
            if self.initialised:
324
                raise (RuntimeError, 'state cannot be changed anymore, the athena service has already been added!')
325
326
327
328

            if not type(state) is list:
                state = [state]

Tomasz Bold's avatar
Tomasz Bold committed
329
            if not set(state) <= self.allowedStates:
330
                raise (RuntimeError, 'unknown state %s, cannot set it!' % state)
331
332
333
334
335
336
            else:
                self.states = state

        def GetConfigurable(self):
            try:
                self.InitialiseSvc()
337
            except Exception:
338
339
340
341
342
343
344
                self.mlog.debug( 'ok, TrigConfigSvc already initialised' )

            return self.GetPlugin()


        def InitialiseSvc(self):
            if self.initialised:
345
                raise (RuntimeError, 'athena service has already been added, do nothing.')
346
347
348
349
350
351

            self.initialised = True

            from AthenaCommon.AppMgr import ServiceMgr


352
            self.mlog.info( "initialising TrigConfigSvc using state %s", self.states )
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
            if 'xml' in self.states:
                from TriggerJobOpts.TriggerFlags import TriggerFlags


                if TriggerFlags.doLVL2() or TriggerFlags.doEF() or TriggerFlags.doHLT() or TriggerFlags.configForStartup()=='HLToffline':
                    self.mlog.info( "setup HLTConfigSvc and add instance to ServiceMgr (xml file="+self.hltXmlFile+")" )
                    hlt = HLTConfigSvc("HLTConfigSvc")
                    hlt.XMLMenuFile = findFileInXMLPATH(self.hltXmlFile)
                    hlt.doMergedHLT = TriggerFlags.doHLT()
                    ServiceMgr += hlt
                else:
                    self.mlog.info( "Will not setup HLTConfigSvc, since TriggerFlags doLVL2(), doEF(), and doHLT() are all False" )
                    self.states[self.states.index("xml")] = "xmll1"

                self.mlog.info( "setup LVL1ConfigSvc and add instance to ServiceMgr (xml file="+self.l1XmlFile+")" )
                l1 = LVL1ConfigSvc("LVL1ConfigSvc")
                l1.XMLMenuFile = findFileInXMLPATH(self.l1XmlFile)
                ServiceMgr += l1

                self.mlog.info( "setup L1TopoConfigSvc and add instance to ServiceMgr (xml file="+self.l1topoXmlFile+")" )
                l1topo = L1TopoConfigSvc()
                l1topo.XMLMenuFile = findFileInXMLPATH(self.l1topoXmlFile)
                ServiceMgr += l1topo


            if 'ds' in self.states:

                self.mlog.info( "setup DSConfigSvc and add instance to ServiceMgr" )
                ds = DSConfigSvc("DSConfigSvc")
                ServiceMgr += ds

            self.mlog.info( "setup TrigConfigSvc and add instance to ServiceMgr" )
            trigSvc = TrigConfigSvc("TrigConfigSvc")
            trigSvc.PriorityList = self.states
            ServiceMgr += trigSvc
388
389
            from AthenaCommon.AppMgr import theApp
            theApp.CreateSvc += [ ServiceMgr.TrigConfigSvc.getFullName() ]
390
391
392

        def GetPlugin(self):
            if not self.initialised:
393
                raise (RuntimeError, 'athena service has not been added, cannot return plugin!.')
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
            from AthenaCommon.AppMgr import ServiceMgr
            return ServiceMgr.TrigConfigSvc

    # storage for the instance reference
    __instance = None

    def __init__(self):
        """ Create singleton instance """
        # Check whether we already have an instance
        if SetupTrigConfigSvc.__instance is None:
            # Create and remember instance
            SetupTrigConfigSvc.__instance = SetupTrigConfigSvc.__impl()

        # Store instance reference as the only member in the handle
        self.__dict__['_SetupTrigConfigSvc__instance'] = SetupTrigConfigSvc.__instance

    def __getattr__(self, attr):
        """ Delegate access to implementation """
        return getattr(self.__instance, attr)

    def __setattr__(self, attr, value):
        """ Delegate access to implementation """
        return setattr(self.__instance, attr, value)
Tomasz Bold's avatar
Tomasz Bold committed
417
418
419


def TrigConfigSvcCfg( flags ):
420
421
    from TrigConfigSvc.TrigConfigSvcCfg import TrigConfigSvcCfg
    return TrigConfigSvcCfg( flags )
Tomasz Bold's avatar
Tomasz Bold committed
422
423

if __name__ == "__main__":
424
    from AthenaCommon.Configurable import Configurable
425
    Configurable.configurableRun3Behavior=True
426

Tomasz Bold's avatar
Tomasz Bold committed
427
428
429
    from AthenaConfiguration.AllConfigFlags import ConfigFlags
    ConfigFlags.lock()
    acc = TrigConfigSvcCfg( ConfigFlags )
430
    acc.store( open( "test.pkl", "wb" ) )
431
    print("All OK")