Merge branch 'master-MenuAccessPython' into 'master'

Trigger config python access

See merge request !31782

Trigger/TrigConfiguration/TrigConfIO/python/HLTTriggerConfigAccess.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+from TrigConfIO.TriggerConfigAccessBase import TriggerConfigAccess, ConfigType
+
+class HLTMenuAccess(TriggerConfigAccess):
+    """
+    this class provides access to the HLT menu
+    the methods are self-explanatory for people with knowledge of the configuration
+    """
+    def __init__(self, filename = None, dbalias = None, smkey = None ):
+        """
+        accessor needs to be initialized with either a filename or the dbalias and smkey
+        """
+        super(HLTMenuAccess,self).__init__( ConfigType.HLTMENU, mainkey = "chains",
+                                            filename = filename, dbalias = dbalias, dbkey = smkey )
+        self.loader.setQuery([
+            "SELECT HTM_DATA FROM {schema}.HLT_MENU WHERE HTM_ID={dbkey}", # for new db schema
+            "SELECT HMT.HMT_MENU FROM {schema}.HLT_MASTER_TABLE HMT WHERE HMT.HMT_ID={dbkey}"  # for current db schema
+        ])
+        self.load()
+
+    def chainNames(self):
+        return (x["name"] for x in self)
+
+    def chains(self):
+        return iter(self)
+
+    def sequencers(self):
+        return self["sequencers"]
+
+    def printSummary(self):
+        print("HLT menu %s" % self.name())
+        print("Number of chains: %i" % len(self) )
+        print("Number of sequencers: %i" % len(self.sequencers()) )
+
+
+
+class HLTPrescalesSetAccess(TriggerConfigAccess):
+    """
+    this class provides access to the HLT prescales set
+    the methods are self-explanatory for people with knowledge of the configuration
+    """
+    def __init__(self, filename = None, dbalias = None, hltpskey = None ):
+        """
+        accessor needs to be initialized with either a filename or the dbalias and hlpskey
+        """
+        super(HLTPrescalesSetAccess,self).__init__( ConfigType.HLTPS, mainkey = "prescales",
+                                                    filename = filename, dbalias = dbalias, dbkey = hltpskey )
+        self.loader.setQuery([
+            "SELECT HPS_DATA FROM {schema}.HLT_PRESCALE_SET WHERE HPS_ID={dbkey}", # for new db schema
+            ""  # for current db schema
+        ])
+        self.load()
+
+
+    def prescales(self):
+        return self["prescales"]
+
+    def chainNames(self):
+        return iter(self)
+
+    def prescale(self, chainName):
+        return self["prescales"][chainName]["prescale"]
+
+    def enabled(self, chainName):
+        return self["prescales"][chainName]["enabled"]
+
+    def printSummary(self):
+        print("HLT prescales set %s" % self.name())
+        print("Number of prescales: %i" % len(self) )
+        print("Number of enabled prescales: %i" % sum(x["enabled"] for x in self["prescales"].values()) )
+
+
+
+class HLTJobOptionsAccess(TriggerConfigAccess):
+    """
+    this class provides access to the HLT algorithm configuration
+    the methods are self-explanatory for people with knowledge of the configuration
+    """
+    def __init__(self, filename = None, dbalias = None, smkey = None ):
+        """
+        accessor needs to be initialized with either a filename or the dbalias and smkey
+        """
+        super(HLTJobOptionsAccess,self).__init__( ConfigType.HLTJO, mainkey = "properties",
+                                                  filename = filename, dbalias = dbalias, dbkey = smkey )
+        self.loader.setQuery([
+            "SELECT HST_DATA FROM {schema}.HLT_SETUP WHERE HST_ID={dbkey}", # for new db schema
+            "SELECT JO.JO_CONTENT FROM {schema}.JO_MASTER_TABLE JO WHERE JO.JO_ID={dbkey}"  # for current db schema
+        ])
+        self.load()
+
+    def algorithms(self):
+        return self["properties"]
+
+    def algorithmNames(self):
+        return iter(self)
+
+    def properties(self, algName):
+        return self["properties"][algName]
+        
+    def printSummary(self):
+        print("Job options")
+        print("Number of algorithms: %i" % len(self) )
+        print("Number of properties: %i" % sum(len(alg) for alg in self.algorithms().values()) )

Trigger/TrigConfiguration/TrigConfIO/python/L1TriggerConfigAccess.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+from TrigConfIO.TriggerConfigAccessBase import TriggerConfigAccess, ConfigType
+
+
+class L1MenuAccess(TriggerConfigAccess):
+    """
+    this class provides access to the L1Menu
+    the methods are self-explanatory for people with knowledge of the configuration
+    """
+    def __init__(self, filename = None, dbalias = None, smkey = None ):
+        """
+        accessor needs to be initialized with either a filename or the dbalias and smkey
+        """
+        super(L1MenuAccess,self).__init__( ConfigType.L1MENU, mainkey = "items",
+                                           filename = filename, dbalias = dbalias, dbkey = smkey)
+        self.loader.setQuery([
+            "SELECT L1TM_DATA FROM {schema}.L1_MENU WHERE L1TM_ID={dbkey}", # for new db schema
+            "SELECT L1MT.L1MT_MENU FROM {schema}.L1_MASTER_TABLE L1MT where L1MT.L1MT_ID={dbkey}"  # for current db schema
+        ])
+        self.load()
+
+
+    def itemNames(self):
+        return self._config["items"].keys()
+
+    def itemsWithCtpid(self ):
+        return self.items( includeKeys = ['ctpid'] )
+
+    def items(self, includeKeys = [] ):
+        if includeKeys:
+            """ reduce returned dictionary """
+            items = self._config["items"]
+            return { x : {k : items[x][k] for k in includeKeys if k in items[x]} for x in items }
+        else:
+            return self._config["items"]
+
+    def thresholdTypes(self):
+        thrTypes = self._config["thresholds"].keys()
+        if "legacyCalo" in thrTypes:
+            thrTypes.remove("legacyCalo")
+            thrTypes += self._config["thresholds"]["legacyCalo"].keys()
+        return thrTypes
+
+    def thresholds(self, thresholdType = None):
+        if thresholdType:
+            if thresholdType == "internal":
+                return {}
+            if thresholdType in self["thresholds"]:
+                return self["thresholds"][thresholdType]["thresholds"]
+            if thresholdType in self["thresholds"]["legacyCalo"]:
+                return self["thresholds"]["legacyCalo"][thresholdType]["thresholds"]
+            raise RuntimeError("Threshold type %s not known in thresholds section of the menu" % thresholdType)
+        else:
+            thrs = {}
+            for thrType in self.thresholdTypes():
+                thrs.update( self.thresholds(thrType) )
+            return thrs
+
+    def thresholdNames(self, thresholdType = None):
+        if thresholdType == "internal":
+            return self["thresholds"]["internal"]["names"]
+        elif thresholdType is None:
+            return self.thresholds().keys() + self.thresholdNames("internal")
+        else:
+            return self.thresholds(thresholdType).keys()
+
+    def thresholdExtraInfo(self, thresholdType):
+        if thresholdType in self["thresholds"]:
+            thrDef = self["thresholds"][thresholdType]
+        elif thresholdType in self["thresholds"]["legacyCalo"]:
+            thrDef = self["thresholds"]["legacyCalo"][thresholdType]
+        else:
+            raise KeyError("Threshold type %s not known in thresholds section of the menu" % thresholdType)
+        return {k:thrDef[k] for k in thrDef if not k in ("thresholds", "type")}
+
+    def topoAlgorithmTypes(self):
+        return self["topoAlgorithms"].keys()
+
+    def topoAlgorithms(self, topoAlgoType = None):
+        if topoAlgoType:
+            return self["topoAlgorithms"][topoAlgoType]
+        else:
+            d = {}
+            for x in self["topoAlgorithms"].values():
+                for gr in x.values():
+                    d.update(gr)
+            return d
+
+    def topoAlgorithmNames(self, topoAlgoType = None):
+        allAlgs = self.topoAlgorithms(topoAlgoType)
+        return allAlgs.keys()
+
+
+    def boardNames(self):
+        return iter(self["boards"])
+
+    def boards(self):
+        return self["boards"]
+
+    def board(self, boardName):
+        return self["boards"][boardName]
+
+    def connectorNames(self):
+        return iter(self["connectors"])
+
+    def connectors(self):
+        return self["connectors"]
+
+    def connector(self, connnectorName):
+        return self["connectors"][connectorName]
+
+    def ctp(self):
+        return self["ctp"]
+
+    def ctpInputs(self, inputType):
+        """ inputType should be 'optical', 'electrical' or 'ctpin'
+        """
+        return self["ctp"]["inputs"][inputType]
+
+    def printSummary(self):
+        print("L1 menu %s" % self.name())
+        print("Number of items: %i" % len(self) )
+        print("Number of threshold types: %i" % len(self.thresholdTypes()) )
+        print("Number of thresholds: %i" % len(self.thresholds()) )
+        print("Number of topo algorithms: %i" % len(self.topoAlgorithms()))
+        print("Number of boards: %i (%i are legacy boards)" % ( len(self.boards()), sum(["legacy" in b for b in self.boards().values()]) ))
+        print("Number of connectors: %i (%i are legacy connetors)" % ( len(self.connectors()), sum(["legacy" in c for c in self.connectors().values()]) ))
+        print("CTP has %i optical, %i electrical, and %i CTPIN inputs" % ( len(self.ctpInputs("optical")), len(self.ctpInputs("electrical")), 
+                                                                           len(set(self.ctpInputs("ctpin")["slot7"].values() + 
+                                                                                   self.ctpInputs("ctpin")["slot8"].values() + 
+                                                                                   self.ctpInputs("ctpin")["slot9"].values()) - set([""])) ))
+
+
+
+
+class L1PrescalesSetAccess(TriggerConfigAccess):
+    """
+    this class provides access to the L1 prescales set
+    the methods are self-explanatory for people with knowledge of the configuration
+    """
+    @staticmethod
+    def calcPrescaleFromCut(cut):
+        """
+        turns cut value (which is what the hardware is configured with), into a float prescale value
+        """
+        return 0xFFFFFF / ( 0x1000000 - cut );
+
+    def __init__(self, filename = None, dbalias = None, l1pskey = None ):
+        """
+        accessor needs to be initialized with either a filename or the dbalias and l1pskey
+        """
+        super(L1PrescalesSetAccess,self).__init__( ConfigType.L1PS, mainkey = "cutValues",
+                                                   filename = filename, dbalias = dbalias, dbkey = l1pskey )
+        self.loader.setQuery([
+            "SELECT L1PS_DATA FROM {schema}.L1_PRESCALE_SET WHERE L1PS_ID={dbkey}", # for new db schema
+            "SELECT L1MT.L1MT_MENU FROM {schema}.L1_MASTER_TABLE L1MT where L1MT.L1MT_ID={dbkey}"  # for current db schema
+        ])
+        self.load()
+
+
+    def itemNames(self):
+        return self["cutValues"].keys()
+    def cutValues(self):
+        return self["cutValues"]
+    def cut(self, itemName):
+        return self["cutValues"][itemName]["cut"]
+    def prescale(self, itemName):
+        return L1PrescalesSetAccess.calcPrescaleFromCut( self.cut(itemName) );
+
+    def enabled(self, itemName):
+        return self["cutValues"][itemName]["enabled"]
+
+    def printSummary(self):
+        print("L1 prescales set %s" % self.name())
+        print("Number of prescales: %i" % len(self) )
+        print("Number of enabled prescales: %i" % sum(x["enabled"] for x in self["cutValues"].values()) )
+
+
+class BunchGroupSetAccess(TriggerConfigAccess):
+    """
+    this class provides access to the L1 bunchgroup set
+    the methods are self-explanatory for people with knowledge of the configuration
+    """
+    def __init__(self, filename = None, dbalias = None, smkey = None ):
+        loader = self._getLoader( configType = ConfigType.L1PS, filename = filename, dbalias = dbalias, dbkey = smkey )
+        super(BunchGroupSetAccess,self).__init__(loader, mainkey = "bunchgroups")
+
+
+

Trigger/TrigConfiguration/TrigConfIO/python/TriggerConfigAccessBase.py

+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+import os
+import json
+import six
+import xml.etree.ElementTree as ET
+from collections import OrderedDict as odict
+
+def getFileType(filename):
+    filetype = "unknown"
+    with open(filename, 'r') as fp:
+        config = json.load(fp)
+        filetype = config['filetype']
+    return filetype
+
+from enum import Enum
+class ConfigType(Enum):
+    NONE = ("Config", "None")
+    L1MENU = ("L1Menu", "l1menu")
+    HLTMENU = ("HLTMenu", "hltmenu")
+    L1PS = ("L1PrescalesSet", "l1prescale")
+    HLTPS = ("HLTPrescalesSet", "hltprescale")
+    BGS = ("L1BunchGroupsSet", "bunchgroupset")
+    HLTJO = ("HLTJobOptions", "joboptions")
+    def __init__(self, basename, filetype):
+        self.basename = basename
+        self.filetype = filetype
+    def __eq__(self, other):
+        if isinstance(other,six.string_types):
+            return self.filetype == other
+        else:
+            return self.filetype == other.filetype
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+class ConfigLoader(object):
+    """ 
+    ConfigLoader derived classes hold the information of the configuration source
+    and define the method to load the configuration
+    """ 
+    def __init__(self,configType):
+        self.configType = configType
+    def confirmConfigType(self,config):
+        """
+        checks that the in-file specification of the configuration type matches the expected type
+        """
+        if config['filetype'] != self.configType:
+            raise RuntimeError("Can not load file with filetype '%s' when expecting '%s'" % (config['filetype'], self.configType.filetype))
+        
+
+class ConfigFileLoader(ConfigLoader):
+    def __init__(self, configType, filename ):
+        super(ConfigFileLoader,self).__init__(configType) 
+        self.filename = filename
+    def load(self):
+        with open(self.filename, 'r') as fp:
+            config = json.load(fp, object_pairs_hook = odict)
+            self.confirmConfigType(config)
+        return config
+    def setQuery(self, query):
+        pass
+    def getWriteFilename(self):
+        outfn = os.path.basename(self.filename)
+        if outfn.endswith(".json"):
+            outfn = outfn.rsplit('.',1)[0]
+        return outfn + ".out.json"
+
+class ConfigDBLoader(ConfigLoader):
+    def __init__(self, configType, dbalias, dbkey):
+        super(ConfigDBLoader,self).__init__(configType)
+        self.dbalias = dbalias
+        self.dbkey = dbkey
+        self.query = None
+        self.schema = None
+    def setQuery(self, query):
+        """
+        query template can be a single query or a list of queries (to deal with different schemata)
+        (internally stored as list of queries, which are tried in order)
+        """
+        self.query = [ query ] if isinstance(query,six.string_types) else query
+    @staticmethod
+    def getResolvedFileName(filename, pathenv=""):
+        """ looks for file, first absolute, then by resolving envvar pathenv"""
+        if os.access(filename,os.R_OK):
+            return filename
+        pathlist = os.getenv(pathenv,'').split(os.pathsep)
+        for path in pathlist:
+            f = os.path.join( path, filename )
+            if os.access( f, os.R_OK ):
+                return f
+        raise RuntimeError("Can't read file %s, neither locally nor in %s" % (filename, pathenv) )
+    @staticmethod
+    def getConnectionParameters(dbalias):
+        dblookupFile = ConfigDBLoader.getResolvedFileName("dblookup.xml", "CORAL_DBLOOKUP_PATH")
+        dbp = ET.parse(dblookupFile)
+        listOfServices = None
+        for logSvc in dbp.iter("logicalservice"):
+            if logSvc.attrib["name"] != dbalias:
+                continue
+            listOfServices = [ serv.attrib["name"] for serv in logSvc.iter("service") ]
+            if len(listOfServices) == 0:
+                raise RuntimeError("DB %s has no services listed in %s" % (dbalias, dblookupFile))
+            break
+        if listOfServices is None:
+            raise RuntimeError("DB %s not available in %s" % (dbalias, dblookupFile))
+        # now get the account and pw for oracle connections
+        credentials = odict().fromkeys(listOfServices)
+        authFile = ConfigDBLoader.getResolvedFileName("authentication.xml", "CORAL_AUTH_PATH")
+        for svc in filter(lambda s : s.startswith("oracle:"), listOfServices):
+            ap = ET.parse(authFile)
+            count = 0
+            for con in filter( lambda c: c.attrib["name"]==svc, ap.iter("connection")):
+                credentials[svc] = dict([(par.attrib["name"],par.attrib["value"]) for par in con])
+                count += 1
+            if count==0:
+                raise RuntimeError("No credentials found for connection %s from service %s for db %s" % (con,svc,dbalias))
+            if count>1:
+                raise RuntimeError("More than 1 connection found in %s for service %s" % (authfile, svc))
+        return credentials
+    @staticmethod
+    def getConnection(credentials):
+        for connSvc, userpw in credentials.items():
+            try:
+                if connSvc.startswith("oracle:"):
+                    from cx_Oracle import connect
+                    [tns,schema] = connSvc.split("/")[-2:]
+                    connection = connect(userpw["user"], userpw["password"], tns, threaded=False)
+                elif connSvc.startswith("frontier:"):
+                    raise NotImplementedError("Python-loading of trigger configuration from Frontier has not yet been implemented (%s)" % type(self))
+                return connection, schema
+            except Exception as e:
+                raise RuntimeError(e)
+    def load(self):
+        from cx_Oracle import DatabaseError
+        credentials = ConfigDBLoader.getConnectionParameters(self.dbalias)
+        connection, self.schema = ConfigDBLoader.getConnection(credentials)
+        cursor = connection.cursor()
+        qdict = { "schema" : self.schema, "dbkey" : self.dbkey }
+        failures = []
+        config = None
+        for q in self.query:
+            try:
+                cursor.execute( q.format(**qdict) )
+            except DatabaseError as e:
+                failures += [ (q.format(**qdict), str(e)) ]
+            else:
+                configblob = cursor.fetchall()[0][0]
+                config = json.loads(str(configblob), object_pairs_hook = odict)
+                break
+        if not config:
+            for q,f in failures:
+                print("Failed query: %s\nFailure: %s" % (q,f))
+            raise RuntimeError("Query failed")
+        self.confirmConfigType(config)
+        return config
+    # proposed filename when writing config to file
+    def getWriteFilename(self):
+        return "{basename}_{schema}_{dbkey}.json".format(basename = self.configType.basename, schema = self.schema, dbkey = self.dbkey)
+
+
+
+class TriggerConfigAccess(object):
+    """ 
+    base class to hold the configuration (OrderedDict) 
+    and provides basic functions to access and print
+    """
+    def __init__(self, configType, mainkey, filename = None, dbalias = None, dbkey = None):
+        self._getLoader(configType = configType, filename = filename, dbalias = dbalias, dbkey = dbkey)
+        self._mainkey = mainkey
+        self._config = None
+
+    def _getLoader(self, configType, filename = None, dbalias = None, dbkey = None ):
+        if filename:
+            self.loader = ConfigFileLoader( configType, filename )
+        elif dbalias and dbkey:
+            self.loader = ConfigDBLoader( configType, dbalias, dbkey )
+        else:
+            raise RuntimeError("Neither input file nor db alias and key provided")
+
+    def load(self):
+        self._config = self.loader.load()
+
+    def __str__(self):
+        return str(self._config)
+
+    def __iter__(self):
+        return iter(self[self._mainkey])
+
+    def __getitem__(self, item):
+        return self._config[item]
+
+    def __len__(self):
+        return len(self[self._mainkey])
+
+    def config(self):
+        """ returns the configuration """
+        return self._config
+
+    def prettyPrint(self):
+        if self._config:
+            print(json.dumps(self._config, indent = 4, separators=(',', ': ')))
+    
+    def name(self):
+        return self["name"]
+
+    def filetype(self):
+        return self["filetype"]
+
+    def printSummary(self):
+        """ print summary info, should be overwritten by derived classes """
+        print("Configuration name: %s" % self.name())
+        print("Configuration size: %s" % len(self))
+
+    def writeFile(self, filename = None):
+        if not filename:
+            filename = self.loader.getWriteFilename()
+        with open(filename, 'w') as fh:
+            json.dump(self.config(), fh, indent = 4, separators=(',', ': '))
+            print("Wrote file %s" % filename)

Trigger/TrigConfiguration/TrigConfIO/scripts/TriggerMenuRW.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
+
+from TrigConfIO.TriggerConfigAccessBase import getFileType, ConfigType
+from TrigConfIO.L1TriggerConfigAccess import L1MenuAccess, L1PrescalesSetAccess, BunchGroupSetAccess
+from TrigConfIO.HLTTriggerConfigAccess import HLTMenuAccess, HLTPrescalesSetAccess, HLTJobOptionsAccess
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-v",       help="increase output verbosity", action="count", default=0)
+    parser.add_argument("--file",   dest="filename", help="json file (menu, prescale set, or bunch group set)")
+    parser.add_argument("--db",     dest="dbalias", help="TriggerDB connection alias")
+    parser.add_argument("--smk",    type = int, help="SuperMaster key (when accessing menu or job options")
+    parser.add_argument("--l1psk",  type = int, help="L1 PrescalesSet key (when accessing L1 prescales")
+    parser.add_argument("--hltpsk", type = int, help="HLT PrescalesSet key (when accessing L1 prescales")
+    parser.add_argument("--bgsk",   type = int, help="BunchGroupSet key (when accessing bunch groups")
+    parser.add_argument("--print",  dest="doPrint", help="Prints the loaded information", action="store_true", default = False)
+    parser.add_argument("--write",  dest="doWrite", help="Writes the loaded information to a file", action="store_true", default = False)
+    
+    args = parser.parse_args()
+    
+    if args.filename:
+        filetype = getFileType(args.filename)
+        if filetype == ConfigType.L1MENU.filetype:
+            cfg = L1MenuAccess( filename = args.filename )
+        elif filetype == ConfigType.L1PS.filetype:
+            cfg = L1PrescalesSetAccess( filename = args.filename )
+        elif filetype == ConfigType.BGS.filetype:
+            cfg = BunchGroupSetAccess( filename = args.filename )
+        elif filetype == ConfigType.HLTMENU.filetype:
+            cfg = HLTMenuAccess( filename = args.filename )
+        elif filetype == ConfigType.HLTPS.filetype:
+            cfg = HLTPrescalesSetAccess( filename = args.filename )
+        elif filetype == ConfigType.HLTJO.filetype:
+            cfg = HLTJobOptionsAccess( filename = args.filename )
+        else:
+            print("Can't read file %s of unknown filetype '%s'" % (args.filename, filetype))
+            return 1
+    elif args.dbalias:
+        if args.smk:
+            cfg  = [ L1MenuAccess( dbalias = args.dbalias, smkey = args.smk ) ]
+            cfg += [ HLTMenuAccess( dbalias = args.dbalias, smkey = args.smk ) ]
+            cfg += [ HLTJobOptionsAccess( dbalias = args.dbalias, smkey = args.smk ) ]
+        elif args.l1psk:
+            cfg  = L1PrescalesSetAccess( dbalias = args.dbalias, l1pskey = args.l1psk )
+        elif args.hltpsk:
+            cfg  = HLTPrescalesSetAccess( dbalias = args.dbalias, hltpskey = args.hltpsk )
+        else:
+            raise NotImplementedError("DB access for this type of key")
+    else:
+        print("Either a file or dbalias and key need to be specified")
+        return 1
+
+
+
+    if args.doPrint:
+        if type(cfg) is list:
+            for c in cfg:
+                c.printSummary()
+        else:
+            cfg.printSummary()
+
+    if args.doWrite:
+        if type(cfg) is list:
+            for c in cfg:
+                c.writeFile()
+        else:
+            cfg.writeFile()
+        
+
+
+
+    return 0
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())