diff --git a/Hlt/TCKUtils/doc/release.notes b/Hlt/TCKUtils/doc/release.notes index f0f2e5fa4d265703c983e44fbce775630e6da951..466767f93653c678bed9017067d560b270c38870 100755 --- a/Hlt/TCKUtils/doc/release.notes +++ b/Hlt/TCKUtils/doc/release.notes @@ -5,6 +5,21 @@ ! and the underlying information !----------------------------------------------------------------------------- +! 2016-06-01 - Rosen Matev + - Add a more sophisticated TCKsh.py script, called by TCKsh and iTCKsh + - currently, you can specify non-default paths to a cdb file + +! 2016-05-25 - Rosen Matev + - Fixed blank lines in diff + - Fixed triplication of output from dump + - Improve diff() + - can write diff to file with an argument + - a more human readable output can be produced with human=True + - Improve dump() + - can now output to an open file object + - can output only a sub-tree + - print TisTosSpecs + !========================= TCKUtils v1r21 2016-04-20 ========================= ! 2016-04-18 - Stefano Perazzini diff --git a/Hlt/TCKUtils/python/TCKUtils/backend.py b/Hlt/TCKUtils/python/TCKUtils/backend.py new file mode 100644 index 0000000000000000000000000000000000000000..7fd9eeea0e46660675eb80fd07ff3d126f9e3668 --- /dev/null +++ b/Hlt/TCKUtils/python/TCKUtils/backend.py @@ -0,0 +1,642 @@ +import os +import weakref +from pprint import pprint, pformat +from multiprocessing.managers import BaseManager + +import GaudiPython +import cppyy +from Gaudi.Configuration import * +from Configurables import ConfigTreeEditor, PropertyConfigSvc + +# Decorate the MD5 object +MD5 = cppyy.gbl.Gaudi.Math.MD5 +MD5.__str__ = MD5.str +MD5.__hash__ = lambda x: x.str().__hash__() +MD5.__eq__ = lambda self, other: self.str() == str(other) +MD5.__ne__ = lambda self, other: self.str() != str(other) +digest = MD5.createFromStringRep + +ConfigTreeNodeAlias = cppyy.gbl.ConfigTreeNodeAlias +ConfigTreeNodeAlias.alias_type.__str__ = ConfigTreeNodeAlias.alias_type.str +alias = ConfigTreeNodeAlias.alias_type +topLevelAlias = ConfigTreeNodeAlias.createTopLevel +TCK = ConfigTreeNodeAlias.createTCK + + +def _appMgr(): + # print 'starting appMgr @ pid = %s' % os.getpid() + ApplicationMgr().AppName = "" + ApplicationMgr().OutputLevel = ERROR + appMgr = GaudiPython.AppMgr() + appMgr.initialize() + return appMgr + + +def _tck(x): + if isinstance(x, basestring) and x.upper().startswith('0X'): + return int(x, 16) + return int(x) + + +def _digest(x): + if type(x) == str: + x = digest(x) + return x + + +class AccessSvcSingleton(object): + __app = None + __pcs = None + __cas = None + __cte = None + + def reset(self): + if self._app(): + self._app().finalize() + AccessSvcSingleton.__pcs = None + AccessSvcSingleton.__cas = None + AccessSvcSingleton.__cte = None + AccessSvcSingleton.__app = None + + def _app(self): + return AccessSvcSingleton.__app + + def _pcs(self): + return AccessSvcSingleton.__pcs + + def _cas(self): + return AccessSvcSingleton.__cas + + def _cte(self): + if callable(AccessSvcSingleton.__cte): + AccessSvcSingleton.__cte = AccessSvcSingleton.__cte(self) + return AccessSvcSingleton.__cte + + def __init__(self, create=False, createConfigTreeEditor=None, cas=None): + if create: + if (AccessSvcSingleton.__pcs or AccessSvcSingleton.__cas): + raise LogicError('re-entry of singleton creation') + pcs = PropertyConfigSvc(ConfigAccessSvc=cas.getFullName()) + cte = ConfigTreeEditor( + PropertyConfigSvc=pcs.getFullName(), ConfigAccessSvc=cas.getFullName()) + AccessSvcSingleton.__app = _appMgr() + self._app().createSvc(cas.getFullName()) + AccessSvcSingleton.__cas = self._app().service( + cas.getFullName(), 'IConfigAccessSvc') + self._app().createSvc(pcs.getFullName()) + AccessSvcSingleton.__pcs = self._app().service( + pcs.getFullName(), 'IPropertyConfigSvc') + AccessSvcSingleton.__cte = lambda x: self._app().toolsvc().create( + cte.getFullName(), interface='IConfigTreeEditor') + + def resolveTCK(self, tck): + """Return the id (digest) for a TCK.""" + tck = _tck(tck) + for i in self._cas().configTreeNodeAliases(alias('TCK/')): + if _tck(i.alias().str().split('/')[-1]) == tck: + return digest(i.ref().str()) + raise KeyError('Not a valid TCK: {:#x}'.format(tck)) + + def _2id(self, id): + if type(id) is int: + id = '0x%08x' % id + if type(id) is str and len(id) == 32: + id = _digest(id) + # if we're not a valid id at this point, maybe we're a TCK... + if type(id) is not MD5: + id = self.resolveTCK(id) + return id + + def resolveConfigTreeNode(self, id): + if type(id) is not MD5: + id = self._2id(id) + return self._pcs().resolveConfigTreeNode(id) if (id and id.valid()) else None + + def writeConfigTreeNode(self, node): + return self._cas().writeConfigTreeNode(node) + + def resolvePropertyConfig(self, id): + return self._pcs().resolvePropertyConfig(id) if (id and id.valid()) else None + + def writePropertyConfig(self, node): + return self._cas().writePropertyConfig(node) + + def collectLeafRefs(self, id): + if type(id) is not MD5: + id = self._2id(id) + for ids in self._pcs().collectLeafRefs(id): + yield PropCfg(ids) + + def resolveConfigTreeNodeAliases(self, a): + if type(a) is not type(alias): + a = alias(a) + return self._pcs().configTreeNodeAliases(a) + + def configTreeNodeAliases(self, alias): + return self._cas().configTreeNodeAliases(alias) + + def writeConfigTreeNodeAlias(self, alias): + return self._cas().writeConfigTreeNodeAlias(alias) + + def updateAndWrite(self, id, updates, label, copy=False): + # For copy we still need the next bit, but the updates are empty + if not updates and not copy: + return + if updates: + print 'updateAndWrite updates:' + pprint(dict(updates)) + mod_map = cppyy.gbl.std.multimap( + 'std::string', 'std::pair<std::string, std::string>') + value_pair = cppyy.gbl.std.pair('std::string', 'std::string') + insert_pair = cppyy.gbl.std.pair( + 'const std::string', 'std::pair<std::string, std::string>') + mods = mod_map() + for algname, props in updates.iteritems(): + for k, v in props.iteritems(): + vp = value_pair(k, v if type(v) == str else repr(v)) + ip = insert_pair(algname, vp) + mods.insert(ip) + + return self._cte().updateAndWrite(id, mods, label) + + +def createAccessSvcSingleton(cas, createConfigTreeEditor=False): + return AccessSvcSingleton(True, createConfigTreeEditor, cas) + + +class Configuration: + " A class representing a configuration entry " + + def __init__(self, alias, svc): + self.info = {'id': alias.ref().str(), 'TCK': [], 'label': '<NONE>'} + self.info.update(zip(['release', 'hlttype'], + alias.alias().str().split('/')[1:3])) + x = svc.resolveConfigTreeNode(alias.ref()) + if not x: + print 'Configuration: failed to resolve configtreenode %s - %s ' % (alias.alias().str(), alias.ref()) + label = x.label() + self.info.update({'label': label}) + + def __getitem__(self, label): + return self.info[label] + + def update(self, d): + self.info.update(d) + + def printSimple(self, prefix=' '): + if not self.info['TCK']: + print prefix + '%10s : %s : %s' % ('<NONE>', self.info['id'], self.info['label']) + else: + for tck in self.info['TCK']: + if type(tck) == int: + tck = '0x%08x' % tck + print prefix + '%10s : %s : %s' % (tck, self.info['id'], self.info['label']) + + def PVSS(self): + for tck in self.info['TCK']: + if type(tck) == str and not tck.startswith('0x'): + tck = int(tck) + if type(tck) != str: + tck = '0x%08x' % tck + return '%20s : %10s : %s : %s\n' % (self.info['hlttype'], tck, self.info['id'], self.info['label']) + + +class TreeNode(object): + " A class representing a ConfigTreeNode " + # use flyweight pattern, and use digest to identify objects... + import weakref + _pool = weakref.WeakValueDictionary() + _nreq = 0 + _noid = 0 + _cm = 0 + # TODO: add functionality to flush _pool + + def __new__(cls, id=None): + TreeNode._nreq = TreeNode._nreq + 1 + if not id: + TreeNode._noid = TreeNode._noid + 1 + return object.__new__(cls) + if type(id) != MD5 and type(id) != int: + print id, type(id) + id = digest(id) + obj = TreeNode._pool.get(id) + if not obj: + TreeNode._cm = TreeNode._cm + 1 + obj = AccessSvcSingleton().resolveConfigTreeNode(id) + TreeNode._pool[id] = obj + if not obj: + print 'failed to resolve ConfigTreeNode %s' % id + return obj + + +class PropCfg(object): + " A class representing a PropertyConfig " + # use flyweight pattern, and use digest to identify objects... + __slots__ = ('__weakref__', 'name', 'type', 'kind', + 'fullyQualifiedName', 'digest', 'props') + + import weakref + _pool = weakref.WeakValueDictionary() + _nreq = 0 + _noid = 0 + _cm = 0 + # TODO: make a singleton svc which we use to resolve IDs if not existent yet... + # TODO: add functionality to flush _pool + + def __new__(cls, id=None): + PropCfg._nreq = PropCfg._nreq + 1 + if not id: + PropCfg._noid = PropCfg._noid + 1 + # added to make it possible to recv in parent process... + return object.__new__(cls) + if type(id) != MD5: + print id, type(id) + id = digest(id) + obj = PropCfg._pool.get(id) + if not obj: + PropCfg._cm = PropCfg._cm + 1 + x = AccessSvcSingleton().resolvePropertyConfig(id) + if not x: + print 'failed to resolve PropertyConfig ' + id + obj = object.__new__(cls) + obj.name = x.name() + obj.type = x.type() + obj.kind = x.kind() + obj.fullyQualifiedName = x.fullyQualifiedName() + obj.digest = id + obj.props = dict((i.first, i.second) for i in x.properties()) + PropCfg._pool[id] = obj + if not obj: + print 'invalid PropCfg %s' % id + return obj + + def properties(self): return self.props + + def fqn(self): return self.fullyQualifiedName + ' (' + self.kind + ')' + + def fmt(self): + return ["'%s':%s\n" % (k, v) for k, v in self.props.iteritems()] + + def evaled_props(self): + def try_eval(x): + try: + return eval(x, {}, {}) + except (SyntaxError, NameError): + return x + except: + print x + raise + return {k: try_eval(v) for k, v in self.props.iteritems()} + + +class Tree(object): + __slots__ = ('__weakref__', 'digest', 'label', 'leaf', 'nodes') + _pool = weakref.WeakValueDictionary() + + def __new__(cls, id=None): + if not id: + return object.__new__(cls) + if not isinstance(id, (MD5, int)): + print id, type(id) + id = digest(id) + obj = cls._pool.get(id) + if not obj: + obj = object.__new__(cls) + cls._pool[id] = obj + node = TreeNode(id) + if not node: + print 'invalid TreeNode for ' + id + obj.digest = node.digest().str() + obj.label = node.label() + leaf = node.leaf() + obj.leaf = PropCfg(leaf) if leaf.valid() else None + obj.nodes = [cls(id) for id in node.nodes()] + return obj + + def prnt(self, depth=0, file=None): + s = ' --> ' + str(self.leaf.digest) if self.leaf else '' + indent = depth * ' ' + print >>file, indent + str(self.digest) + (30 - len(indent)) * ' ' + s + for tree in self.nodes: + tree.prnt(depth + 1, file) + + def _inorder(self, visited=None): + if visited is not None: + if self in visited: + return + visited.add(self) + yield self + for i in self.nodes: + for x in i._inorder(visited): + yield x + + def iter(self, duplicates=False): + return self._inorder(None if duplicates else set()) + + def __iter__(self): + return self._inorder(set()) + + def iterleafs(self, duplicates=False): + """Return a generator yielding leafs. + + Args: + duplicates: whether to traverse the full tree (slow!) + + Yields: + leaf objects of type PropCfg + + """ + return (i.leaf for i in self.iter(duplicates) if i.leaf) + + def leafs(self): + return {i.name: i for i in self.iterleafs()} + + +class RemoteAccess(object): + _svc = None + + def __init__(self, cas): + # print 'remote(%s) created at pid=%s and cas %s' % + # (self,os.getpid(),cas.getType()) + RemoteAccess._svc = createAccessSvcSingleton(cas=cas) + + def __del__(self): + RemoteAccess._svc.reset() + + def rgetConfigTree(self, id): + # maybe prefetch all leafs by invoking + # RemoteAccess._svc.collectLeafRefs(id) + # benchmark result: makes no difference whatsoever... + + # from time import clock + # x = clock() + t = Tree(id) + # print '<< remote(%s) at pid=%s: rgetConfigTree(%s) : lookup time: %s s.' % (self, os.getpid(), id, clock() - x) + return t + + def rgetConfigurations(self): + # print 'remote(%s) at pid=%s: rgetConfigurations()' % + # (self,os.getpid()) + svc = RemoteAccess._svc + info = dict() + # print 'reading TOPLEVEL' + for i in svc.configTreeNodeAliases(alias('TOPLEVEL/')): + x = Configuration(i, svc) + info[i.alias().str()] = x + # print 'reading TCK' + for i in svc.configTreeNodeAliases(alias('TCK/')): + tck = _tck(i.alias().str().split('/')[-1]) + id = i.ref().str() + for k in info.values(): + if k.info['id'] == id: + k.info['TCK'].append(tck) + # print 'reading TAG' + for i in svc.configTreeNodeAliases(alias('TAG/')): + tag = i.alias().str().split('/')[1:] + id = i.ref().str() + for k in info.values(): + if k.info['id'] == id: + k.update({'TAG': tag}) + return info + + def rupdateProperties(self, id, updates, label, copy=False): + if not label: + print 'please provide a reasonable label for the new configuration' + return None + svc = RemoteAccess._svc + # either we got (some form of) an ID, and we reverse engineer back the alias (provided it is unique!) + # or we got an alias directly... + + if type(id) == ConfigTreeNodeAlias: + a = id.alias().str() + id = id.ref() + # this is fine for TOPLEVEL, but not TCK... + else: + id = svc._2id(id) + if not id.valid(): + raise RuntimeWarning('not a valid id : %s' % id) + a = [i.alias().str() for i in svc.configTreeNodeAliases( + alias('TOPLEVEL/')) if i.ref() == id] + if len(a) != 1: + print 'something went wrong: no unique toplevel match for ' + str(id) + return + a = a[0] + (release, hlttype) = a.split('/', 3)[1:3] + newId = svc.updateAndWrite(id, updates, label, copy) + noderef = svc.resolveConfigTreeNode(newId) + top = topLevelAlias(release, hlttype, noderef) + svc.writeConfigTreeNodeAlias(top) + print 'wrote ' + str(top.alias()) + return str(newId) + + def rupdateL0TCK(self, id, l0config, label, extra): + svc = RemoteAccess._svc + id = svc._2id(id) + if not id.valid(): + raise RuntimeWarning('not a valid id : %s' % id) + a = [i.alias().str() for i in svc.configTreeNodeAliases( + alias('TOPLEVEL/')) if i.ref() == id] + if len(a) != 1: + print 'something went wrong: no unique toplevel match for ' + str(id) + return + (release, hlttype) = a[0].split('/', 3)[1:3] + + from collections import defaultdict + updates = defaultdict(dict) + # check L0 config in source config + for cfg in svc.collectLeafRefs(id): + # check for either a MultiConfigProvider with the right setup, + # or for a template with the right TCK in it... + if cfg.name == 'ToolSvc.L0DUConfig': + if cfg.type != 'L0DUConfigProvider': + raise KeyError( + "Can only update configuration which use L0DUConfigProvider, not %s" % cfg.type) + # check that all specified properties exist in cfg + for (k, v) in l0config.iteritems(): + if k not in cfg.props: + raise KeyError( + 'Specified property %s not in store' % k) + updates['ToolSvc.L0DUConfig'].update({k: v}) + + if extra: + updates.update(extra) + + newId = svc.updateAndWrite(id, updates, label) + noderef = svc.resolveConfigTreeNode(newId) + if not noderef: + print 'oops, could not find node for %s ' % newId + top = topLevelAlias(release, hlttype, noderef) + svc.writeConfigTreeNodeAlias(top) + print 'wrote ' + str(top.alias()) + return str(newId) + + def rresolveTCK(self, tck): + svc = RemoteAccess._svc + return svc.resolveTCK(tck) + + def rcreateTCKEntries(self, d): + svc = RemoteAccess._svc + for tck, id in d.iteritems(): + id = _digest(id) + tck = _tck(tck) + # check whether L0 part of the TCK is specified + l0tck = tck & 0xffff + hlt1, hlt2 = (False, False) + masks = {(True, False): 1 << 28, + (False, True): 1 << 29, + (True, True): 0} + if l0tck: + l0tck = '0x%04X' % l0tck + for cfg in svc.collectLeafRefs(id): + if l0tck and cfg.name == 'ToolSvc.L0DUConfig': + # check for either a MultiConfigProvider with the right setup, + # or for a template with the right TCK in it... + if cfg.type not in ['L0DUMultiConfigProvider', 'L0DUConfigProvider']: + raise KeyError( + "not a valid L0DU config provider: %s" % cfg.type) + if cfg.type == 'L0DUMultiConfigProvider' and l0tck not in cfg.props['registerTCK']: + raise KeyError('requested L0TCK %s not known by L0DUMultiConfigProvider in config %s; known L0TCKs: %s' % ( + l0tck, id, cfg.props['registerTCK'])) + elif cfg.type == 'L0DUConfigProvider' and l0tck != cfg.props['TCK']: + raise KeyError('requested L0TCK %s not known by L0DUConfigProvider in config %s; known L0TCK: %s' % ( + l0tck, id, cfg.props['TCK'])) + if cfg.type == 'GaudiSequencer' and cfg.name == 'HltDecisionSequence': + hlt1 = 'GaudiSequencer/Hlt1' in cfg.props['Members'] + hlt2 = 'GaudiSequencer/Hlt2' in cfg.props['Members'] + + # Check and set HLT1/HLT2 bits + # If most significant bit is set check if it is really a techincal TCK or a PID-less TCK + # this check is moved from + # Kernel/HltInterfaces/src/ConfigTreeNodeAlias.cpp + if (tck & (1 << 31)): + if (tck & (3 << 30)): + print 'Creating PID-less TCK %x' % (tck) + elif (tck & 0xFFFF): + # check if the technical TCK respect the rules + raise ValeError( + 'The requested TCK does not match the rules: technical TCK must have L0 bits set to zero') + else: + mask = masks[(hlt1, hlt2)] + # In case bits were already assigned, check if they are correct + if ((tck & (3 << 28)) | mask) != mask: + raise ValueError('Wrong HLT1 and HLT2 specific bits set in tck: %x instead of %x for highest four bits.' % ( + tck >> 28, mask >> 28)) + # Apply HLT1/HLT2 mask + tck |= mask + + print 'creating mapping TCK: 0x%08x -> ID: %s' % (tck, id) + ref = svc.resolveConfigTreeNode(id) + alias = TCK(ref, tck) + svc.writeConfigTreeNodeAlias(alias) + + def rcopyTree(self, nodeRef): + svc = RemoteAccess._svc + + def __nodes(n): + node = svc.resolveConfigTreeNode(n) + from itertools import chain + # depth first traversal -- this insures we do not write + # incomplete data objects... + for j in chain.from_iterable(__nodes(i) for i in node.nodes()): + yield j + yield node + for i in __nodes(nodeRef): + leafRef = i.leaf() + if leafRef.valid(): + leaf = svc.resolvePropertyConfig(leafRef) + assert leaf + newRef = svc.writePropertyConfig(leaf) + assert leafRef == newRef + n = svc.writeConfigTreeNode(i) + assert n == i.digest() + + def rcopy(self, glob=None): + svc = RemoteAccess._svc + from copy import deepcopy + from itertools import chain + aliases = [deepcopy(i) for label in ('TOPLEVEL/', 'TCK/') + for i in svc.configTreeNodeAliases(alias(label))] + if glob: + from fnmatch import fnmatch + aliases = filter(lambda i: fnmatch(i.alias().str(), glob), aliases) + # Now, split them into + # TOPLEVEL (just copy) + top = filter(lambda i: i.alias().str( + ).startswith('TOPLEVEL/'), aliases) + # The rest: find corresponding TOPLEVEL, add it to the toplevel list, + # re-create alias afterwards + other = filter(lambda i: not i.alias().str( + ).startswith('TOPLEVEL/'), aliases) + assert len(top) + len(other) == len(aliases) + toplevelaliases = svc.configTreeNodeAliases(alias('TOPLEVEL/')) + for i in other: + top += [deepcopy(j) for j in toplevelaliases if j.ref() == i.ref() and j not in top] + l = len(top) + print '# of TOPLEVEL items to copy: %s' % l + # TODO: sort on the (integer) numbers appearing in the string... + for k, i in enumerate(sorted(top, key=lambda x: x.alias().str()), 1): + print '[%s/%s] copying tree %s' % (k, l, i.alias()) + empty = dict() + node = svc.resolveConfigTreeNode(i.ref()) + newid = self.rupdateProperties(i, empty, node.label(), True) + assert newid == i.ref().str() + for i in other: + print 'copying alias %s -> %s ' % (i.alias().str(), i.ref()) + svc.writeConfigTreeNodeAlias(i) + print 'done copying... ' + + +class AccessMgr(BaseManager): + pass + +AccessMgr.register('Access', RemoteAccess) + + +class AccessProxy(object): + _manager = None + _access = None + _cas = None + + def __init__(self): + # print 'creating proxy in pid = %s' % os.getpid() + pass + # TODO: access should be seperately for each cas instance... + # worse: since Configurables are singletons, they may have + # changed since the last time used. Hence, have to + # check that that hasn't happend, so we need a local + # copy of the state of cas, and compare to that.... + # and if different, let's just shutdown the remote + # and start again... + # (shouldn't have to flush PropertyConfig/ConfigTreeNode) + + def access(self, cas): + if not self._valid(cas): + self.flush() + if not AccessProxy._manager: + AccessProxy._manager = AccessMgr() + AccessProxy._manager.start() + # print 'proxy started manager' + if not AccessProxy._access: + # print 'proxy requesting access to remote -- cas = + # %s'%(cas.getType()) + AccessProxy._access = AccessProxy._manager.Access(cas) + AccessProxy._cas = cas + AccessProxy._properties = cas.getProperties() + + return AccessProxy._access + + def _valid(self, cas): + if not AccessProxy._access or not AccessProxy._cas: + return True + if cas != AccessProxy._cas: + return False # different configurable! + return cas.getProperties() == AccessProxy._properties + + # make flush visible such that eg. createTCKEntries can flush the remote + # and force re-reading... + def flush(self): + AccessProxy._cas = None + AccessProxy._properties = None + AccessProxy._access = None + AccessProxy._manager.shutdown() + AccessProxy._manager = None diff --git a/Hlt/TCKUtils/python/TCKUtils/utils.py b/Hlt/TCKUtils/python/TCKUtils/utils.py index 47a12b28e904fc7398977e586d56118a841bb0c7..1762af8ca54bb59c49ac2c4d46a0001333c2918e 100755 --- a/Hlt/TCKUtils/python/TCKUtils/utils.py +++ b/Hlt/TCKUtils/python/TCKUtils/utils.py @@ -1,61 +1,50 @@ -import GaudiPython -import cppyy -from Gaudi.Configuration import * -from Configurables import ConfigStackAccessSvc, ConfigDBAccessSvc, ConfigZipFileAccessSvc, ConfigTarFileAccessSvc, ConfigFileAccessSvc, ConfigCDBAccessSvc -from Configurables import ConfigTreeEditor, PropertyConfigSvc +"""Utilities for working with TCKs. + +TCK numbering +Bit 31 is assigned to "technical" +Bit 30 is assigned to "for MC" +Bit 29 is assigned to "HLT2 only" +Bit 28 is assigned to "HLT1 only" +Neither 29 nor 28 is "old style" Hlt1 + Hlt2 +""" + +import os +import re +import contextlib +from pprint import pprint, pformat + +from Configurables import (ConfigStackAccessSvc, + ConfigDBAccessSvc, + ConfigZipFileAccessSvc, + ConfigTarFileAccessSvc, + ConfigFileAccessSvc, + ConfigCDBAccessSvc) +import backend + + +def cdb_file(filename): + return os.path.join(os.environ['HLTTCKROOT'], filename) -# pick the default config access svc -import os.path -def cdb_file(filename) : - return os.path.join( os.environ['HLTTCKROOT'], filename ) def cdb_file_exists(filename): - return os.path.isfile( cdb_file(filename) ) - -if cdb_file_exists('config.cdb') : - from Configurables import ConfigCDBAccessSvc as ConfigAccessSvc -else : - from Configurables import ConfigTarFileAccessSvc as ConfigAccessSvc - -cdb_filename = cdb_file('config.cdb') if cdb_file_exists('config.cdb') else cdb_file('config.tar') -print 'using %s as input.' % cdb_filename - -from pprint import pprint - -## TCK numbering -## Bit 31 is assigned to "technical" -## Bit 30 is assigned to "for MC" -## Bit 29 is assigned to "HLT2 only" -## Bit 28 is assigned to "HLT1 only" -## Neither 29 nor 28 is "old style" Hlt1 + Hlt2 - -### add some decoration... -MD5 = cppyy.gbl.Gaudi.Math.MD5 -MD5.__str__ = MD5.str -MD5.__hash__ = lambda x : x.str().__hash__() -MD5.__eq__ = lambda self, other: self.str() == str(other) -cppyy.gbl.ConfigTreeNodeAlias.alias_type.__str__ = cppyy.gbl.ConfigTreeNodeAlias.alias_type.str -digest = MD5.createFromStringRep -alias = cppyy.gbl.ConfigTreeNodeAlias.alias_type -topLevelAlias = cppyy.gbl.ConfigTreeNodeAlias.createTopLevel -TCK = cppyy.gbl.ConfigTreeNodeAlias.createTCK -from os import getpid - -def _appMgr() : - #print 'starting appMgr @ pid = %s' % getpid() - ApplicationMgr().AppName = "" - ApplicationMgr().OutputLevel = ERROR - appMgr = GaudiPython.AppMgr() - appMgr.initialize() - return appMgr - -def _tck(x) : - if type(x) == str and x[0:2].upper() == '0X' : return int(x,16) - return int(x) - -def _digest(x) : - if type(x) == str : x = digest( x ) - return x + return os.path.isfile(cdb_file(filename)) + +ConfigAccessSvc = ConfigCDBAccessSvc if cdb_file_exists('config.cdb') \ + else ConfigTarFileAccessSvc + + +def get_default_cas(): + from Gaudi.Configuration import allConfigurables + if 'ConfigAccessSvc' not in allConfigurables: + filename = cdb_file('config.cdb') if cdb_file_exists('config.cdb') \ + else cdb_file('config.tar') + print 'Using {} as input.'.format(filename) + return ConfigAccessSvc('ConfigAccessSvc', File=filename) + else: + return allConfigurables['ConfigAccessSvc'] +default_cas = get_default_cas() + + # utilities to pack and unpack L0 conditions into Condition property... # given the 'Conditions' property of an L0DUCOnfig tool instance, @@ -86,7 +75,7 @@ def _parseL0setting( setting ) : val.update( { key : value } ) return val -def dumpL0( id, cas = ConfigAccessSvc() ) : +def dumpL0( id, cas=default_cas): tree = getConfigTree( id, cas ) l0s = [ i for i in tree if i.leaf and i.leaf.type == 'L0DUConfigProvider' ] for i in l0s : @@ -98,7 +87,7 @@ def dumpL0( id, cas = ConfigAccessSvc() ) : print 100*'*' -def getL0Prescales( id, cas = ConfigAccessSvc() ) : +def getL0Prescales( id, cas=default_cas): tree = getConfigTree( id, cas ) l0s = [ i for i in tree if i.leaf and i.leaf.type == 'L0DUConfigProvider' ] ret = dict() @@ -110,701 +99,320 @@ def getL0Prescales( id, cas = ConfigAccessSvc() ) : ret[ l0tck ][ chan['name'] ] = chan['rate'] return ret -class AccessSvcSingleton(object) : - __app = None - __pcs = None - __cas = None - __cte = None - def reset(self) : - if self._app() : self._app().finalize() - AccessSvcSingleton.__pcs = None - AccessSvcSingleton.__cas = None - AccessSvcSingleton.__cte = None - AccessSvcSingleton.__app = None - - def _app(self) : - return AccessSvcSingleton.__app - def _pcs(self) : - return AccessSvcSingleton.__pcs - def _cas(self) : - return AccessSvcSingleton.__cas - def _cte(self) : - if callable( AccessSvcSingleton.__cte) : - AccessSvcSingleton.__cte = AccessSvcSingleton.__cte(self) - return AccessSvcSingleton.__cte - - def __init__(self,create=False,createConfigTreeEditor=False,cas=ConfigAccessSvc()) : - if create : - if (AccessSvcSingleton.__pcs or AccessSvcSingleton.__cas) : - raise LogicError('re-entry of singleton creation') - pcs = PropertyConfigSvc( ConfigAccessSvc = cas.getFullName() ) - cte = ConfigTreeEditor( PropertyConfigSvc = pcs.getFullName() - , ConfigAccessSvc = cas.getFullName() ) - AccessSvcSingleton.__app = _appMgr() - self._app().createSvc(cas.getFullName()) - AccessSvcSingleton.__cas = self._app().service(cas.getFullName(),'IConfigAccessSvc') - self._app().createSvc(pcs.getFullName()) - AccessSvcSingleton.__pcs = self._app().service(pcs.getFullName(),'IPropertyConfigSvc') - AccessSvcSingleton.__cte = lambda x : self._app().toolsvc().create(cte.getFullName(),interface='IConfigTreeEditor') - - def resolveTCK(self, tck): - """Return the id (digest) for a TCK.""" - tck = _tck(tck) - for i in self._cas().configTreeNodeAliases(alias('TCK/')): - if _tck(i.alias().str().split('/')[-1]) == tck: - return digest(i.ref().str()) - raise KeyError('Not a valid TCK: {:#x}'.format(tck)) - - def _2id(self,id) : - if type(id) is int : id = '0x%08x' % id - if type(id) is str and len(id)==32 : id = _digest(id) - # if we're not a valid id at this point, maybe we're a TCK... - if type(id) is not MD5 : id = self.resolveTCK(id) - return id - def resolveConfigTreeNode(self,id) : - if type(id) is not MD5 : - id = self._2id(id) - return self._pcs().resolveConfigTreeNode(id) if (id and id.valid()) else None - def writeConfigTreeNode(self,node) : - return self._cas().writeConfigTreeNode(node) - def resolvePropertyConfig(self,id) : - return self._pcs().resolvePropertyConfig(id) if (id and id.valid()) else None - def writePropertyConfig(self,node) : - return self._cas().writePropertyConfig(node) - def collectLeafRefs(self,id) : - if type(id) is not MD5 : - id = self._2id(id) - for ids in self._pcs().collectLeafRefs( id ) : - yield PropCfg(ids) - def resolveConfigTreeNodeAliases(self, a ) : - if type(a) is not type(alias) : a = alias(a) - return self._pcs().configTreeNodeAliases( a ) - def configTreeNodeAliases( self, alias ) : - return self._cas().configTreeNodeAliases( alias ) - def writeConfigTreeNodeAlias(self,alias) : - return self._cas().writeConfigTreeNodeAlias(alias) - def updateAndWrite(self, id, updates, label, copy = False) : - # For copy we still need the next bit, but the updates are empty - if not updates and not copy: - return - if updates: - print 'updateAndWrite updates:' - pprint(dict(updates)) - mod_map = cppyy.gbl.std.multimap('std::string', 'std::pair<std::string, std::string>') - value_pair = cppyy.gbl.std.pair('std::string', 'std::string') - insert_pair = cppyy.gbl.std.pair('const std::string', 'std::pair<std::string, std::string>') - mods = mod_map() - for algname, props in updates.iteritems() : - for k, v in props.iteritems() : - vp = value_pair(k, v if type(v) == str else repr(v)) - ip = insert_pair(algname, vp) - mods.insert(ip) - - return self._cte().updateAndWrite(id,mods,label) - - -def createAccessSvcSingleton( cas = ConfigAccessSvc(), createConfigTreeEditor = False ) : - return AccessSvcSingleton( create = True, createConfigTreeEditor = createConfigTreeEditor, cas = cas ) - - -def _lookupProperty(table,algname,property) : + +def _getProperty(table, algname, property): if algname not in table : raise KeyError("no algorithm named %s in specified config"%algname ) properties = table[algname].properties() if property not in properties: raise KeyError("could not locate property %s for algorithm %s in specified config"%(property,algname) ) return properties[property] -def getProperty(id,algname,property, cas ) : +def getProperty(id, algname, property, cas=default_cas): + # getConfigTree(t).leafs() tables = xget( [ id ], cas ) - return _lookupProperty(tables[id],algname,property) - - - -### and now define the routines visible from the outside world... - -class Configuration : - " A class representing a configuration entry " - def __init__(self,alias,svc) : - self.info = { 'id' : alias.ref().str() , 'TCK' : [], 'label' : '<NONE>' } - self.info.update( zip(['release','hlttype'],alias.alias().str().split('/')[1:3])) - x = svc.resolveConfigTreeNode( alias.ref() ) - if not x : - print 'Configuration: failed to resolve configtreenode %s - %s ' % ( alias.alias().str(), alias.ref() ) - label = x.label(); - self.info.update( { 'label' : label } ) - def __getitem__(self,label) : - return self.info[label] - def update(self,d) : - self.info.update( d ) - def printSimple(self,prefix=' ') : - if not self.info['TCK'] : - print prefix + '%10s : %s : %s'%('<NONE>',self.info['id'],self.info['label']) - else : - for tck in self.info['TCK'] : - if type(tck) == int : tck = '0x%08x' % tck - print prefix + '%10s : %s : %s'%(tck,self.info['id'],self.info['label']) - def PVSS(self) : - for tck in self.info['TCK'] : - if type(tck) == str and not tck.startswith('0x') : tck = int(tck) - if type(tck) != str : tck = '0x%08x' % tck - return '%20s : %10s : %s : %s\n'%(self.info['hlttype'],tck,self.info['id'],self.info['label']) - - -class TreeNode(object) : - " A class representing a ConfigTreeNode " - # use flyweight pattern, and use digest to identify objects... - import weakref - _pool = weakref.WeakValueDictionary() - _nreq = 0 - _noid = 0 - _cm = 0 - # TODO: add functionality to flush _pool - def __new__(cls, id = None) : - TreeNode._nreq = TreeNode._nreq + 1 - if not id: - TreeNode._noid = TreeNode._noid+1 - return object.__new__(cls) - if type(id) != MD5 and type(id) != int : - print id,type(id) - id = digest(id) - obj = TreeNode._pool.get( id ) - if not obj : - TreeNode._cm = TreeNode._cm+1 - obj = AccessSvcSingleton().resolveConfigTreeNode(id) - TreeNode._pool[id] = obj - if not obj : - print 'failed to resolve ConfigTreeNode %s'%id - return obj - - -class PropCfg(object) : - " A class representing a PropertyConfig " - # use flyweight pattern, and use digest to identify objects... - import weakref - _pool = weakref.WeakValueDictionary() - _nreq = 0 - _noid = 0 - _cm = 0 - # TODO: make a singleton svc which we use to resolve IDs if not existent yet... - # TODO: add functionality to flush _pool - def __new__(cls, id = None) : - PropCfg._nreq = PropCfg._nreq + 1 - if not id : - PropCfg._noid = PropCfg._noid +1 - return object.__new__(cls) # added to make it possible to recv in parent process... - if type(id) != MD5 : - print id,type(id) - id = digest(id) - obj = PropCfg._pool.get( id ) - if not obj : - PropCfg._cm = PropCfg._cm + 1 - x = AccessSvcSingleton().resolvePropertyConfig(id) - if not x : - print 'failed to resolve PropertyCondif %s'%id - obj = object.__new__(cls) - obj.name = x.name() - obj.type = x.type() - obj.kind = x.kind() - obj.fullyQualifiedName = x.fullyQualifiedName() - obj.digest = id - obj.props = dict( (i.first,i.second) for i in x.properties() ) - PropCfg._pool[id] = obj - if not obj : - print 'invalid PropCfg %s'%id - return obj - def properties(self) : return self.props - def fqn(self) : return self.fullyQualifiedName + ' ('+self.kind+')' - def fmt(self) : - return [ "'%s':%s\n"%(k,v) for k,v in self.props.iteritems() ] - -class Tree(object): - def __init__(self,id,parent=None ) : - self.parent = parent - self.depth = self.parent.depth+1 if self.parent else 0 - node = TreeNode(id) - if not node : print 'invalid TreeNode for %s' % id - self.digest = node.digest().str() - self.label = node.label() - leaf = node.leaf() - self.leaf = PropCfg( leaf ) if leaf.valid() else None - self.nodes = [ Tree(id=id,parent=self) for id in node.nodes() ] - #TODO: add direct access to leafs 'in' this tree by name - # i.e. notify 'top level parent' of 'my' leafs... - def prnt(self): - s = ' --> ' + str(self.leaf.digest) if self.leaf else '' - indent = self.depth*' ' - print indent + str(self.digest) + (30-len(indent))*' ' + s - for i in self.nodes: i.prnt() - # define in order traversal - def __iter__(self): - return self._inorder() - def _inorder(self) : - yield self - for i in self.nodes: - for x in i._inorder() : yield x - def iterleafs(self) : - for i in self._inorder() : # this _can_ give duplicate leafs!!! - if i.leaf is None : continue - yield i.leaf - def leafs(self) : - d = dict() - for i in self.iterleafs() : - if i.name not in d : d[ i.name ] = i - return d - - - #TODO: add direct access to leafs 'in' this tree by name - # and use it to implement an efficient __contains__ + return _getProperty(tables[id],algname,property) # TODO: rewrite in terms of trees... # that should make it a lot faster for almost identical # trees... -def diff( lhs, rhs , cas = ConfigAccessSvc() ) : - table = xget( [ lhs, rhs ] , cas ) - setl = set( table[lhs].keys() ) - setr = set( table[rhs].keys() ) - onlyInLhs = setl - setr - if len(onlyInLhs)>0 : - print 'only in %s: ' % lhs - for i in onlyInLhs : print ' ' + i - onlyInRhs = setr - setl - if len(onlyInRhs)>0 : - print 'only in %s:' % rhs - for i in onlyInRhs : print ' ' + i - for i in setl & setr : - (l,r) = ( table[lhs][i], table[rhs][i] ) - if l.digest != r.digest : - from difflib import unified_diff - print ''.join( unified_diff(l.fmt(), r.fmt(), - l.fqn(), r.fqn(), - lhs, rhs, n=0) ) - -def copy( source = ConfigAccessSvc() , target = ConfigDBAccessSvc(ReadOnly=False), glob = None ) : - if source == target : +def diff(lhs, rhs, file=None, human=False, cas=default_cas): + ltree = getConfigTree(lhs) + rtree = getConfigTree(rhs) + + lnodes = {i.leaf.name: i for i in ltree if i.leaf} + rnodes = {i.leaf.name: i for i in rtree if i.leaf} + + lleafs = set(lnodes.keys()) + rleafs = set(rnodes.keys()) + + onlyInLhs = lleafs - rleafs + onlyInRhs = rleafs - lleafs + + lhs_nice = hex(lhs) if isinstance(lhs, int) else lhs + rhs_nice = hex(rhs) if isinstance(rhs, int) else rhs + + with smart_open(file) as f: + if len(onlyInLhs) > 0: + f.write('only in %s:\n' % lhs_nice) + for i in onlyInLhs: + f.write(' ' + i + '\n') + + if len(onlyInRhs) > 0: + f.write('only in %s:\n' % rhs_nice) + for i in onlyInRhs: + f.write(' ' + i + '\n') + dump(rnodes[i], file=f, cas=cas) + for i in lleafs & rleafs: + l, r = lnodes[i].leaf, rnodes[i].leaf + if l.digest != r.digest: + from difflib import unified_diff, ndiff + if not human: + lines = unified_diff(l.fmt(), r.fmt(), + l.fqn(), r.fqn(), + lhs_nice, rhs_nice, n=0) + f.write(''.join(lines) + '\n') + else: + f.write('--- {}\t{}\n'.format(l.fqn(), lhs_nice)) + f.write('+++ {}\t{}\n'.format(r.fqn(), rhs_nice)) + lps = l.evaled_props() + rps = r.evaled_props() + for k in set(lps.keys() + rps.keys()): + lp = lps.get(k) + rp = rps.get(k) + if lp != rp: + f.write('@@ -{} +{} @@\n'.format(k if lp else '', k if rp else '')) + lines = ndiff(pformat(lp).splitlines(), pformat(rp).splitlines(), None, None) + f.write('\n'.join(lines) + '\n') + + +def copy(source=default_cas, target=None, glob=None): + from Gaudi.Configuration import DEBUG + if not target: + target = ConfigDBAccessSvc(ReadOnly=False) + if source == target: print 'WARNING: source and target are the same -- no-op ...' return - r = AccessProxy().access( cas = ConfigStackAccessSvc( ConfigAccessSvcs = [ target.getFullName(), source.getFullName() ] - , OutputLevel=DEBUG ) - ).rcopy(glob) - AccessProxy().flush() + stack_cas = ConfigStackAccessSvc( + ConfigAccessSvcs=[target.getFullName(), source.getFullName()], + OutputLevel=DEBUG) + r = backend.AccessProxy().access(cas=stack_cas).rcopy(glob) + backend.AccessProxy().flush() return r -def listComponents( id, cas = ConfigAccessSvc() ) : - tree = getConfigTree( id, cas ) - for i in tree : - if i.leaf : - s = i.depth*3*' ' + i.leaf.name - print s + (80-len(s))*' ' + str(i.leaf.digest) -def getAlgorithms( id, cas = ConfigAccessSvc() ) : - tree = getConfigTree( id, cas ) - x = '' - for i in tree : - if i.leaf and i.leaf.kind =='IAlgorithm': - s = i.depth*3*' ' + i.leaf.name - x = x + s + (80-len(s))*' ' + str(i.leaf.digest) + '\n' - return x - -def listAlgorithms( id, cas = ConfigAccessSvc() ) : - print getAlgorithms(id,cas) - -def getProperties( id, algname='',property='',cas = ConfigAccessSvc() ) : - retlist= dict() - tree = getConfigTree( id, cas ) + +def _getComponents(tree, depth=0, kinds=None): + leaf = tree.leaf + if leaf and (not kinds or leaf.kind in kinds): + yield (depth, leaf.name, str(leaf.digest)) + # # Don't write anything below the WriterFilter + # if leaf and leaf.name == 'WriterFilter': + # return + for i in tree.nodes: + for x in _getComponents(i, depth + 1, kinds=kinds): + yield x + + +def getComponents(id, kinds=None, cas=default_cas): + tree = getConfigTree(id, cas) + return _getComponents(tree, kinds=kinds) + + +def getAlgorithms(id, cas=default_cas): + return getComponents(id, kinds=['IAlgorithm'], cas=cas) + + +def listComponents(id, kinds=None, cas=default_cas): + for depth, name, digest in getComponents(id, kinds, cas): + s = depth * ' ' + name + print s + (80-len(s))*' ' + digest + + +def listAlgorithms(id, cas=default_cas): + listComponents(id, kinds=['IAlgorithm'], cas=cas) + + +def getProperties(id, algname='', property='', cas=default_cas): + retlist = dict() + tree = getConfigTree(id, cas) import re - if algname : + if algname: reqNode = re.compile(algname) - matchNode = lambda x : reqNode.match(x.leaf.fullyQualifiedName) - else : - matchNode = None - if property : + matchNode = lambda leaf: reqNode.match(leaf.fullyQualifiedName) + else: + matchNode = lambda leaf: True + if property: reqProp = re.compile(property) - matchProp = lambda x : reqProp.match(x) - else : - matchProp = None - # generate a unique ID for this traversal, so that we do not - # repeat leafs... - import uuid - sig = uuid.uuid4().hex - for i in tree : - if not i.leaf or (matchNode and not matchNode(i)) : continue - if hasattr(i.leaf,sig) : continue - setattr(i.leaf,sig,True) - pdict = dict() - for (k,v) in i.leaf.properties().iteritems() : - if matchProp and not matchProp(k) : continue - pdict[k] = v - if pdict : retlist[ i.leaf.name ] = pdict + matchProp = lambda x: reqProp.match(x) + else: + matchProp = lambda x: True + + for leaf in tree.iterleafs(): + if not leaf or not matchNode(leaf): + continue + pdict = {k: v for k, v in leaf.properties().iteritems() if matchProp(k)} + if pdict: + retlist[leaf.name] = pdict return retlist -def listProperties( id, algname='',property='',cas = ConfigAccessSvc() ) : - for (c,d) in getProperties(id,algname,property,cas).iteritems() : - print '\n Requested Properties for %s' % c - for k,v in d.iteritems() : - print " '%s':%s" % (k,v) -def getTCKInfo(x) : - for (i,j) in getConfigurations().iteritems() : - if x in j['TCK'] : return (j['hlttype'],j['release']) +def listProperties(id, algname='', property='', cas=default_cas): + for c, d in getProperties(id, algname, property, cas).iteritems(): + print '\nRequested Properties for %s' % c + for k, v in d.iteritems(): + print " '%s':%s" % (k, v) + + +def getConfigurations(cas=default_cas): + return backend.AccessProxy().access(cas).rgetConfigurations() + + +def getTCKInfo(x, cas=default_cas): + for i in getConfigurations(cas).itervalues(): + if x in i['TCK']: + return (i['hlttype'], i['release']) return None -def getReleases( cas = ConfigAccessSvc() ) : - return set( [ i['release'] for i in getConfigurations(cas).itervalues() ] ) -def getHltTypes( release, cas = ConfigAccessSvc() ) : - info = getConfigurations( cas ) - return set( [ i['hlttype'] for i in info.itervalues() if i['release']==release ] ) -def getTCKs( release = None, hlttype = None, cas = ConfigAccessSvc() ) : - info = getConfigurations( cas ) - pred = lambda x : x['TCK'] and ( not release or x['release'] == release ) and ( not hlttype or x['hlttype'] == hlttype ) + +def getReleases(cas=default_cas): + return set([i['release'] for i in getConfigurations(cas).itervalues()]) + + +def getHltTypes(release, cas=default_cas): + info = getConfigurations(cas) + return set([i['hlttype'] for i in info.itervalues() if i['release'] == release]) + + +def getTCKs(release=None, hlttype=None, cas=default_cas): + def pred(x): + return (x['TCK'] and (not release or x['release'] == release) and + (not hlttype or x['hlttype'] == hlttype)) + info = getConfigurations(cas) result = [] - for i in [ x for x in info.itervalues() if pred(x) ]: - for tck in i['TCK'] : - result.append( ('0x%08x'%tck,i['label']) ) + for i in filter(pred, info.itervalues()): + for tck in i['TCK']: + result.append(('0x%08x' % tck, i['label'])) return result -def getTCKList( cas = ConfigAccessSvc() ) : - info = getConfigurations( cas ) + + +def getTCKList(cas=default_cas): + info = getConfigurations(cas) result = [] - for i in info.itervalues() : - for tck in i['TCK'] : result.append( '0x%08x'%tck ) + for i in info.itervalues(): + for tck in i['TCK']: + result.append('0x%08x' % tck) return result -def getRoutingBits( id , cas = ConfigAccessSvc() ) : - # should be a map... so we try to 'eval' it + + +def getRoutingBits(id, stages=('Hlt1', 'Hlt2', 'Hlt'), cas=default_cas): rbs = {} - for stage in ('Hlt1', 'Hlt2', 'Hlt'): - for p in ['RoutingBits','routingBitDefinitions'] : - try : + for stage in stages: + for p in ['RoutingBits', 'routingBitDefinitions']: + try: prop = getProperty(id, '%sRoutingBitsWriter' % stage, p, cas) - except KeyError : + except KeyError: continue rbs.update(eval(prop)) return rbs if rbs else None -## TODO: is a string the best thing to return??? -def getAlgorithms( id, cas = ConfigAccessSvc() ) : - tree = getConfigTree( id, cas ) - tempstr = '' - for i in tree : - if i.leaf and i.leaf.kind =='IAlgorithm': - s = i.depth*3*' ' + i.leaf.name - tempstr = tempstr + s + (80-len(s))*' ' + str(i.leaf.digest) + '\n' - return tempstr - -def dump( id, properties = None, lines = None, file = None, cas = ConfigAccessSvc() ) : - if not properties : - properties = [ 'RoutingBits', 'AcceptFraction', 'FilterDescriptor' - , 'Preambulo', 'Code', 'InputLocations','Input','Inputs' - , 'DaughtersCuts', 'CombinationCut', 'MotherCut', 'DecayDescriptor' - , 'OutputSelection', 'Output' ] - tree = getConfigTree( id, cas ) - def len1(line): - _i = line.rfind('\n') - return len(line) if _i<0 else len(line)-(_i+1) - - def prettyPrintStreamer(code) : - code.translate(None,'\n') - return ('\n' + (_tab+25+18)*' ' + '>> ' ).join( [ i.strip() for i in code.split('>>') ] ) - - def prettyPrintDict(code,trItem) : - try : - cuts = eval(code) # should be a dict - return "{ "+ ('\n' + (_tab+25+18)*' ' + ', ' ).join( [ trItem(k,v) for (k,v) in cuts.iteritems() ] ) + '\n'+(_tab+25+18)*' '+"}" - except : - return code - - def prettyPrintList(code,trItem = None, skipEmpty = True) : - try : - l = eval(code) - if skipEmpty and not l : return '' - if len(l)<2 : return code - if trItem : - l = [ trItem(i) for i in l ] - return "[ "+('\n' + (_tab+25+18)*' ' + ', ' ).join( l )+'\n'+(_tab+25+18)*' '+']' - except : - return code - - trtable = { 'Code' : prettyPrintStreamer - , 'DaughtersCuts' : lambda x : prettyPrintDict(x, lambda k,v : "'%s' : '%s'"%(k,v) ) - , 'Inputs' : prettyPrintList - , 'InputLocations' : prettyPrintList - , 'Preambulo' : prettyPrintList - , 'FilterDescriptor' : lambda x : prettyPrintList(x,lambda y : "'%s'"%y, True) - , 'RoutingBits' : lambda x : prettyPrintDict(x, lambda k,v : "%2d : \"%s\""%(k,v) ) - } - import re - show = not lines - from sys import stdout - file = open(file,'w') if file else sys.stdout - for i in tree : - if not i.leaf or i.leaf.kind != 'IAlgorithm' : continue - if lines and i.leaf.type in [ 'Hlt::Line', 'HltLine' ] : - show = re.match(lines,i.leaf.name) - if not show : continue - _tab = 50 - line = i.depth * ' ' + i.leaf.name - if len1(line)>( _tab-1): line += '\n'+ _tab*' ' - else : line += (_tab-len1(line))*' ' - line += '%-25.25s'%i.leaf.type - for k,v in [ (k,v) for k,v in i.leaf.props.iteritems() if k in properties and v ]: - if _tab+25 < len1(line) : line+= '\n'+(_tab+25)*' ' - if k in trtable.keys() : v = trtable[k](v) - if v : line += '%-15s : %s' % ( k, v) - file.write(line+'\n') - - -class RemoteAccess(object) : - _svc = None - def __init__( self, cas ) : - #print 'remote(%s) created at pid=%s and cas %s' % (self,getpid(),cas.getType()) - RemoteAccess._svc = createAccessSvcSingleton( cas = cas ) - def __del__( self ) : - RemoteAccess._svc.reset() - def rgetConfigTree( self, id ) : - # maybe prefetch all leafs by invoking - #RemoteAccess._svc.collectLeafRefs(id) - # benchmark result: makes no difference whatsoever... - #from time import clock - #x = clock() - t = Tree(id) - #print '<< remote(%s) at pid=%s: rgetConfigTree(%s) : lookup time: %s s.' % (self,getpid(),id, clock()-x) - return t - def rgetConfigurations( self ) : - #print 'remote(%s) at pid=%s: rgetConfigurations()' % (self,getpid()) - svc = RemoteAccess._svc - info = dict() - #print 'reading TOPLEVEL' - for i in svc.configTreeNodeAliases( alias( 'TOPLEVEL/') ) : - x = Configuration( i,svc ) - info[ i.alias().str() ] = x - #print 'reading TCK' - for i in svc.configTreeNodeAliases( alias( 'TCK/' ) ) : - tck = _tck(i.alias().str().split('/')[-1]) - id = i.ref().str() - for k in info.values() : - if k.info['id'] == id : k.info['TCK'].append(tck) - #print 'reading TAG' - for i in svc.configTreeNodeAliases( alias( 'TAG/' ) ) : - tag = i.alias().str().split('/')[1:] - id = i.ref().str() - for k in info.values() : - if k.info['id'] == id : k.update( { 'TAG' : tag } ) - return info - def rupdateProperties(self, id, updates, label, copy = False ) : - if not label : - print 'please provide a reasonable label for the new configuration' - return None - svc = RemoteAccess._svc - # either we got (some form of) an ID, and we reverse engineer back the alias (provided it is unique!) - # or we got an alias directly... - - if type(id) == cppyy.gbl.ConfigTreeNodeAlias : - a = id.alias().str() - id = id.ref() - # this is fine for TOPLEVEL, but not TCK... - else : - id = svc._2id(id) - if not id.valid() : raise RuntimeWarning('not a valid id : %s' % id ) - a = [ i.alias().str() for i in svc.configTreeNodeAliases( alias('TOPLEVEL/') ) if i.ref() == id ] - if len(a) != 1 : - print 'something went wrong: no unique toplevel match for ' + str(id) +@contextlib.contextmanager +def smart_open(filename=None): + if not filename: + import sys + fh, close = sys.stdout, False + elif isinstance(filename, basestring): + fh, close = open(filename, 'w'), True + else: + fh, close = filename, False + + try: + yield fh + finally: + if close: + fh.close() + + +_default_properties = [ + 'RoutingBits', 'AcceptFraction', 'FilterDescriptor', + 'Preambulo', 'Code', 'InputLocations', 'Input', 'Inputs', 'TisTosSpecs', + 'DaughtersCuts', 'CombinationCut', 'MotherCut', 'DecayDescriptor', + 'OutputSelection', 'Output', + 'BankTypes', 'KillFromAll', 'DefaultIsKill', +] + + +_tab = 50 + +def _len1(line): + _i = line.rfind('\n') + return len(line) if _i<0 else len(line)-(_i+1) + +def prettyPrintStreamer(code) : + code.translate(None,'\n') + return ('\n' + (_tab+25+18)*' ' + '>> ' ).join( [ i.strip() for i in code.split('>>') ] ) + +def prettyPrintDict(code,trItem) : + try : + cuts = eval(code) # should be a dict + return "{ "+ ('\n' + (_tab+25+18)*' ' + ', ' ).join( [ trItem(k,v) for (k,v) in cuts.iteritems() ] ) + '\n'+(_tab+25+18)*' '+"}" + except : + return code + +def prettyPrintList(code,trItem = None, skipEmpty = True) : + try : + l = eval(code) + if skipEmpty and not l : return '' + if len(l)<2 : return code + if trItem : + l = [ trItem(i) for i in l ] + return "[ "+('\n' + (_tab+25+18)*' ' + ', ' ).join( l )+'\n'+(_tab+25+18)*' '+']' + except : + return code + +_trtable = { 'Code' : prettyPrintStreamer + , 'DaughtersCuts' : lambda x : prettyPrintDict(x, lambda k,v : "'%s' : '%s'"%(k,v) ) + , 'Inputs' : prettyPrintList + , 'InputLocations' : prettyPrintList + , 'Preambulo' : prettyPrintList + , 'FilterDescriptor' : lambda x : prettyPrintList(x,lambda y : "'%s'"%y, True) + , 'RoutingBits' : lambda x : prettyPrintDict(x, lambda k,v : "%2d : \"%s\""%(k,v) ) + } + + +def prettyPrintLeaf(leaf, depth, properties): + line = depth * ' ' + leaf.name + if _len1(line) > (_tab - 1): + line += '\n' + _tab*' ' + else: + line += (_tab-_len1(line))*' ' + line += '%-25.25s' % leaf.type + for k, v in leaf.props.iteritems(): + if k in properties and v: + if _tab + 25 < _len1(line): + line += '\n'+(_tab+25)*' ' + if k in _trtable.keys(): + prettyPrint = _trtable.get(k) + if prettyPrint: + v = prettyPrint(v) + if v: + line += '%-15s : %s' % (k, v) + return line + + +def _dump(tree, depth=0, properties=_default_properties, lines=None, file=None): + leaf = tree.leaf + if leaf and leaf.kind == 'IAlgorithm': + if lines and leaf.type in ['Hlt::Line', 'HltLine']: + if not re.match(lines, leaf.name): return - a = a[0] - (release,hlttype) = a.split('/',3)[1:3] - newId = svc.updateAndWrite(id, updates, label, copy) - noderef = svc.resolveConfigTreeNode( newId ) - top = topLevelAlias( release, hlttype, noderef ) - svc.writeConfigTreeNodeAlias(top) - print 'wrote ' + str(top.alias()) - return str(newId) - def rupdateL0TCK(self, id, l0config, label, extra ) : - svc = RemoteAccess._svc - id = svc._2id(id) - if not id.valid() : raise RuntimeWarning('not a valid id : %s' % id ) - a = [ i.alias().str() for i in svc.configTreeNodeAliases( alias('TOPLEVEL/') ) if i.ref() == id ] - if len(a) != 1 : - print 'something went wrong: no unique toplevel match for ' + str(id) - return - (release,hlttype) = a[0].split('/',3)[1:3] - - from collections import defaultdict - updates = defaultdict(dict) - # check L0 config in source config - for cfg in svc.collectLeafRefs( id ): - # check for either a MultiConfigProvider with the right setup, - # or for a template with the right TCK in it... - if cfg.name == 'ToolSvc.L0DUConfig' : - if cfg.type != 'L0DUConfigProvider': - raise KeyError("Can only update configuration which use L0DUConfigProvider, not %s" % cfg.type ) - # check that all specified properties exist in cfg - for (k,v) in l0config.iteritems() : - if k not in cfg.props : raise KeyError('Specified property %s not in store'%k) - updates['ToolSvc.L0DUConfig'].update({k : v}) - - if extra: - updates.update(extra) - - newId = svc.updateAndWrite(id, updates, label) - noderef = svc.resolveConfigTreeNode( newId ) - if not noderef : print 'oops, could not find node for %s ' % newId - top = topLevelAlias( release, hlttype, noderef ) - svc.writeConfigTreeNodeAlias(top) - print 'wrote ' + str(top.alias()) - return str(newId) - def rresolveTCK(self, tck ) : - svc = RemoteAccess._svc - return svc.resolveTCK(tck) - - def rcreateTCKEntries(self, d ) : - svc = RemoteAccess._svc - for tck,id in d.iteritems() : - id = _digest(id) - tck = _tck(tck) - # check whether L0 part of the TCK is specified - l0tck = tck & 0xffff - hlt1, hlt2 = (False, False) - masks = {(True, False) : 1 << 28, (False, True) : 1 << 29, (True, True) : 0} - if l0tck : - l0tck = '0x%04X' % l0tck - for cfg in svc.collectLeafRefs( id ) : - if l0tck and cfg.name == 'ToolSvc.L0DUConfig' : - # check for either a MultiConfigProvider with the right setup, - # or for a template with the right TCK in it... - if cfg.type not in [ 'L0DUMultiConfigProvider', 'L0DUConfigProvider' ] : - raise KeyError("not a valid L0DU config provider: %s" % cfg.type ) - if cfg.type == 'L0DUMultiConfigProvider' and l0tck not in cfg.props['registerTCK'] : - raise KeyError('requested L0TCK %s not known by L0DUMultiConfigProvider in config %s; known L0TCKs: %s' % ( l0tck, id, cfg.props['registerTCK'] )) - elif cfg.type == 'L0DUConfigProvider' and l0tck != cfg.props['TCK'] : - raise KeyError('requested L0TCK %s not known by L0DUConfigProvider in config %s; known L0TCK: %s' % ( l0tck, id, cfg.props['TCK'] )) - if cfg.type == 'GaudiSequencer' and cfg.name == 'HltDecisionSequence': - hlt1 = 'GaudiSequencer/Hlt1' in cfg.props['Members'] - hlt2 = 'GaudiSequencer/Hlt2' in cfg.props['Members'] - - # Check and set HLT1/HLT2 bits - # If most significant bit is set check if it is really a techincal TCK or a PID-less TCK - # this check is moved from Kernel/HltInterfaces/src/ConfigTreeNodeAlias.cpp - if (tck & (1 << 31)): - if (tck & (3 << 30)): - print 'Creating PID-less TCK %x' % (tck) - elif (tck & 0xFFFF): - #check if the technical TCK respect the rules - raise ValeError('The requested TCK does not match the rules: technical TCK must have L0 bits set to zero') - else: - mask = masks[(hlt1, hlt2)] - # In case bits were already assigned, check if they are correct - if ((tck & (3 << 28)) | mask) != mask: - raise ValueError('Wrong HLT1 and HLT2 specific bits set in tck: %x instead of %x for highest four bits.' % (tck >> 28, mask >> 28)) - # Apply HLT1/HLT2 mask - tck |= mask - - print 'creating mapping TCK: 0x%08x -> ID: %s' % (tck,id) - ref = svc.resolveConfigTreeNode( id ) - alias = TCK( ref, tck ) - svc.writeConfigTreeNodeAlias(alias) - - def rcopyTree( self, nodeRef ) : - svc = RemoteAccess._svc - def __nodes( n ) : - node = svc.resolveConfigTreeNode( n ) - from itertools import chain - # depth first traversal -- this insures we do not write - # incomplete data objects... - for j in chain.from_iterable( __nodes(i) for i in node.nodes() ) : - yield j - yield node - for i in __nodes( nodeRef ) : - leafRef = i.leaf() - if leafRef.valid() : - leaf = svc.resolvePropertyConfig(leafRef) - assert leaf - newRef = svc.writePropertyConfig(leaf) - assert leafRef == newRef - n = svc.writeConfigTreeNode(i) - assert n == i.digest() - - def rcopy( self, glob = None ) : - svc = RemoteAccess._svc - from copy import deepcopy - from itertools import chain - aliases = [ deepcopy(i) for label in ( 'TOPLEVEL/','TCK/' ) for i in svc.configTreeNodeAliases( alias(label) ) ] - if glob : - from fnmatch import fnmatch - aliases = filter( lambda i : fnmatch(i.alias().str(),glob), aliases) - # Now, split them into - # TOPLEVEL (just copy) - top = filter( lambda i : i.alias().str().startswith('TOPLEVEL/'), aliases ) - # The rest: find corresponding TOPLEVEL, add it to the toplevel list, re-create alias afterwards - other = filter( lambda i : not i.alias().str().startswith('TOPLEVEL/'), aliases ) - assert len(top)+len(other) == len(aliases) - toplevelaliases = svc.configTreeNodeAliases( alias('TOPLEVEL/') ) - for i in other : - top += [ deepcopy(j) for j in toplevelaliases if j.ref() == i.ref() and j not in top ] - l = len(top) - print '# of TOPLEVEL items to copy: %s' % l - for k,i in enumerate( sorted( top, key = lambda x : x.alias().str() ), 1 ):# TODO: sort on the (integer) numbers appearing in the string... - print '[%s/%s] copying tree %s' % (k,l, i.alias()) - empty = dict() - node = svc.resolveConfigTreeNode( i.ref() ) - newid = self.rupdateProperties(i,empty,node.label(), True) - assert newid == i.ref().str() - for i in other: - print 'copying alias %s -> %s ' % (i.alias().str(),i.ref() ) - svc.writeConfigTreeNodeAlias( i ) - print 'done copying... ' - - -from multiprocessing.managers import BaseManager -class AccessMgr(BaseManager): - pass - -AccessMgr.register('Access',RemoteAccess) - - -class AccessProxy( object ) : - _manager = None - _access = None - _cas = None - def __init__( self ) : - #print 'creating proxy in pid = %s' % os.getpid() - pass - # TODO: access should be seperately for each cas instance... - # worse: since Configurables are singletons, they may have - # changed since the last time used. Hence, have to - # check that that hasn't happend, so we need a local - # copy of the state of cas, and compare to that.... - # and if different, let's just shutdown the remote - # and start again... - # (shouldn't have to flush PropertyConfig/ConfigTreeNode) - def access( self, cas ) : - if not self._valid(cas) : self.flush() - if not AccessProxy._manager : - AccessProxy._manager = AccessMgr() - AccessProxy._manager.start() - #print 'proxy started manager' - if not AccessProxy._access : - #print 'proxy requesting access to remote -- cas = %s'%(cas.getType()) - AccessProxy._access = AccessProxy._manager.Access( cas ) - AccessProxy._cas = cas - AccessProxy._properties = cas.getProperties() - - return AccessProxy._access - def _valid( self, cas ) : - if not AccessProxy._access or not AccessProxy._cas: return True - if cas != AccessProxy._cas : return False # different configurable! - return cas.getProperties() == AccessProxy._properties - def flush( self ) : # make flush visible such that eg. createTCKEntries can flush the remote and force re-reading... - AccessProxy._cas = None - AccessProxy._properties = None - AccessProxy._access = None - AccessProxy._manager.shutdown() - AccessProxy._manager = None - - -def resolveTCK( tck, cas=ConfigAccessSvc()) : - return AccessProxy().access(cas).rresolveTCK( tck ) - -def getConfigTree(id, cas = ConfigAccessSvc()): - if 'forest' not in dir(getConfigTree) : getConfigTree.forest = dict() - if id not in getConfigTree.forest : - tree = AccessProxy().access(cas).rgetConfigTree( id ) + line = prettyPrintLeaf(leaf, depth, properties) + file.write(line + '\n') + + # Don't write anything below the WriterFilter + if leaf and leaf.name == 'WriterFilter': + return + + for i in tree.nodes: + _dump(i, depth + 1, properties, lines, file) + + +def dump(id, properties=_default_properties, lines=None, file=None, cas=default_cas): + tree = id if isinstance(id, backend.Tree) else getConfigTree(id, cas) + with smart_open(file) as f: + _dump(tree, properties=properties, lines=lines, file=f) + + + + +def resolveTCK( tck, cas=default_cas): + return backend.AccessProxy().access(cas).rresolveTCK( tck ) + + +def getConfigTree(id, cas=default_cas): + if not hasattr(getConfigTree, 'forest'): + getConfigTree.forest = dict() + if id not in getConfigTree.forest: + tree = backend.AccessProxy().access(cas).rgetConfigTree(id) getConfigTree.forest[id] = tree if tree.digest != id : # in case we got a TCK, the remote side resolves this to an ID @@ -814,19 +422,29 @@ def getConfigTree(id, cas = ConfigAccessSvc()): getConfigTree.forest[tree.digest] = tree return getConfigTree.forest[id] -def getConfigurations( cas = ConfigAccessSvc() ) : - return AccessProxy().access(cas).rgetConfigurations() +def get( ids , cas=default_cas): + if 'forest' not in dir(xget) : xget.forest = dict() + fetch = [ id for id in ids if id not in xget.forest.keys() ] + if fetch : + for t in fetch : xget.forest[t] = getConfigTree(t).leafs() + forest = dict() + for id in ids : forest[id] = xget.forest[id] + return forest + -def updateProperties(id,updates,label='', cas = ConfigAccessSvc() ) : - ret = AccessProxy().access(cas).rupdateProperties( id,updates,label ) - if ret : AccessProxy().flush() # explicit flush in case we wrote something + +def updateProperties(id,updates,label='', cas=default_cas): + ret = backend.AccessProxy().access(cas).rupdateProperties( id,updates,label ) + if ret : backend.AccessProxy().flush() # explicit flush in case we wrote something return ret -def updateL0TCK(id, l0tck, label='', cas = ConfigAccessSvc(), extra = None ) : + +def updateL0TCK(id, l0tck, label='', cas=default_cas, extra=None): + from Gaudi.Configuration import importOptions if not label : print 'please provide a reasonable label for the new configuration' return None - l0tck = '0x%04X'%_tck(l0tck) + l0tck = '0x%04X' % backend._tck(l0tck) importOptions('$L0TCK/L0DUConfig.opts') from Configurables import L0DUMultiConfigProvider,L0DUConfigProvider if l0tck not in L0DUMultiConfigProvider('L0DUConfig').registerTCK : @@ -834,40 +452,32 @@ def updateL0TCK(id, l0tck, label='', cas = ConfigAccessSvc(), extra = None ) : configProvider = L0DUConfigProvider('ToolSvc.L0DUConfig.TCK_%s'%l0tck) l0config = configProvider.getValuedProperties() l0config['TCK'] = l0tck - ret = AccessProxy().access(cas).rupdateL0TCK(id,l0config,label,extra) - if ret : AccessProxy().flush() + ret = backend.AccessProxy().access(cas).rupdateL0TCK(id,l0config,label,extra) + if ret : backend.AccessProxy().flush() return ret -def createTCKEntries(d, cas = ConfigAccessSvc() ) : - ret = AccessProxy().access(cas).rcreateTCKEntries(d) - AccessProxy().flush() +def createTCKEntries(d, cas=default_cas): + ret = backend.AccessProxy().access(cas).rcreateTCKEntries(d) + backend.AccessProxy().flush() return ret -def xget( ids , cas = ConfigAccessSvc() ) : - if 'forest' not in dir(xget) : xget.forest = dict() - fetch = [ id for id in ids if id not in xget.forest.keys() ] - if fetch : - for t in fetch : xget.forest[t] = getConfigTree(t).leafs() - forest = dict() - for id in ids : forest[id] = xget.forest[id] - return forest -def getHlt1Lines( id , cas = ConfigAccessSvc() ) : +def getHlt1Lines( id , cas=default_cas): # should be a list... so we try to 'eval' it return eval(getProperty(id,'Hlt1','Members',cas)) -def getHlt2Lines( id , cas = ConfigAccessSvc() ) : +def getHlt2Lines( id , cas=default_cas): # should be a list... so we try to 'eval' it return eval(getProperty(id,'Hlt2','Members',cas)) -def getHltLines( id , cas = ConfigAccessSvc() ) : +def getHltLines( id , cas=default_cas): return getHlt1Lines(id,cas) + getHlt2Lines(id,cas) -def getHlt1Decisions( id , cas = ConfigAccessSvc() ) : +def getHlt1Decisions( id , cas=default_cas): table = xget( [ id ], cas )[id] - lines = eval(_lookupProperty(table,'Hlt1','Members')) - return [ _lookupProperty(table,i.split('/')[-1],'DecisionName') for i in lines ] + lines = eval(_getProperty(table,'Hlt1','Members')) + return [ _getProperty(table,i.split('/')[-1],'DecisionName') for i in lines ] def _sortReleases( release ): version = release.split('_')[ 1 ] @@ -898,25 +508,19 @@ def printReleases( rel ) : pprint(rel) def printHltTypes( rt ) : pprint(rt) def printTCKs( tcks ) : pprint(tcks) -def listConfigurations( cas = ConfigAccessSvc() ) : +def listConfigurations( cas=default_cas): return printConfigurations( getConfigurations(cas) ) -def listReleases( cas = ConfigAccessSvc() ) : +def listReleases( cas=default_cas): return printReleases( getReleases() ) -def listHltTypes( release, cas = ConfigAccessSvc() ) : +def listHltTypes( release, cas=default_cas): return printHltTypes( getHltTypes(release) ) -def listTCKs( release, hlttype, cas = ConfigAccessSvc() ) : +def listTCKs( release, hlttype, cas=default_cas): return printTCKs( getTCKs(release,hlttype) ) -def listRoutingBits( id, cas = ConfigAccessSvc() ) : +def listRoutingBits( id, cas=default_cas): pprint(getRoutingBits(id,cas)) -def listHlt1Lines( id, cas = ConfigAccessSvc() ) : +def listHlt1Lines( id, cas=default_cas): pprint(getHlt1Lines(id,cas)) -def listHlt2Lines( id, cas = ConfigAccessSvc() ) : +def listHlt2Lines( id, cas=default_cas): pprint(getHlt2Lines(id,cas)) -def listHlt1Decisions( id, cas = ConfigAccessSvc() ) : +def listHlt1Decisions( id, cas=default_cas): pprint(getHlt1Decisions(id,cas)) - - -###### do the actual work... - -if __name__ == '__main__' : - listConfigurations() diff --git a/Hlt/TCKUtils/scripts/TCKsh b/Hlt/TCKUtils/scripts/TCKsh index 6251679c3a363776ffbb993129e5c712ee5335a2..79f9b7cfa2263a5f2bde80eb632843daf3524c82 100755 --- a/Hlt/TCKUtils/scripts/TCKsh +++ b/Hlt/TCKUtils/scripts/TCKsh @@ -1,2 +1,7 @@ #!/bin/sh -exec `which python` -i -c 'from TCKUtils.utils import *' +if [[ "$@" == "--help" ]] || [[ "$@" == "-h" ]]; then + interactive='' +else + interactive='-i' +fi +python ${interactive} `which TCKsh.py` -- "$@" diff --git a/Hlt/TCKUtils/scripts/TCKsh.py b/Hlt/TCKUtils/scripts/TCKsh.py new file mode 100644 index 0000000000000000000000000000000000000000..cdb536752e1e05968400b475869e7fb257a267c4 --- /dev/null +++ b/Hlt/TCKUtils/scripts/TCKsh.py @@ -0,0 +1,48 @@ +import os +import argparse + +parser = argparse.ArgumentParser(description='Explore TCK database') +parser.add_argument('files', nargs='*', help='config.cdb/tar files (use - for the default)') +parser.add_argument('--write', action='store_true', help='Open the first file for writing') + +args = parser.parse_args() +if not args.files and args.write: + parser.error('"--write" requires at least one file to be specified') + +from Configurables import (ConfigStackAccessSvc, + ConfigZipFileAccessSvc, + ConfigTarFileAccessSvc, + ConfigCDBAccessSvc) + + +def __create_file_cas(name, path, mode): + if path == '-': + return ConfigCDBAccessSvc(name, Mode=mode) + ext = os.path.splitext(path)[1] + try: + ConfigAccessSvc = { + '.zip': ConfigZipFileAccessSvc, + '.tar': ConfigTarFileAccessSvc, + '.cdb': ConfigCDBAccessSvc, + }[ext] + except KeyError: + raise ValueError("Extension '{}' not recognized".format(ext)) + if 'Write' not in mode and not os.path.isfile(path): + raise IOError("File not found '{}'".format(path)) + return ConfigCDBAccessSvc(name, File=path, Mode=mode) + +if len(args.files) == 1: + fn = args.files[0] + mode = 'ReadWrite' if args.write else 'ReadOnly' + __create_file_cas('ConfigAccessSvc', fn, mode) +elif len(args.files) > 1: + svcs = [] + for i, fn in enumerate(args.files): + mode = 'ReadWrite' if i == 0 and args.write else 'ReadOnly' + svc = __create_file_cas('ConfigAccessSvc{}'.format(i), fn, mode) + svcs.append(svc) + ConfigStackAccessSvc('ConfigAccessSvc', ConfigAccessSvcs=svcs) + + +# The only essential thing: +from TCKUtils.utils import * diff --git a/Hlt/TCKUtils/scripts/cdb_diff b/Hlt/TCKUtils/scripts/cdb_diff new file mode 100755 index 0000000000000000000000000000000000000000..fd8efaf96395c878dfa991bc004a57812e18b8e9 --- /dev/null +++ b/Hlt/TCKUtils/scripts/cdb_diff @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import os +import sys +from difflib import unified_diff +from cStringIO import StringIO + +# dummy svc to prevent "Using ... as input" +from Configurables import ConfigCDBAccessSvc +ConfigCDBAccessSvc('ConfigAccessSvc') + +from TCKUtils.utils import * + + +def strConfigurations(cas): + sys.stdout = mystdout = StringIO() + listConfigurations(cas) + sys.stdout = sys.__stdout__ + return mystdout.getvalue() + +if len(sys.argv) == 3: + fromfile, tofile = sys.argv[1:] +else: + fromfile = os.path.expandvars('$HLTTCKROOT/config.cdb') + tofile = sys.argv[1] +a = strConfigurations(ConfigAccessSvc('From', File=fromfile)).splitlines(True) +b = strConfigurations(ConfigAccessSvc('To', File=tofile)).splitlines(True) + +print ''.join(unified_diff(a, b, fromfile, tofile, n=0)) diff --git a/Hlt/TCKUtils/scripts/iTCKsh b/Hlt/TCKUtils/scripts/iTCKsh index ee6b8db15a48704ae66c8d04bfb5b52477beace5..9a362b2fdd6d50189acad7ff1d53b66981565e0e 100644 --- a/Hlt/TCKUtils/scripts/iTCKsh +++ b/Hlt/TCKUtils/scripts/iTCKsh @@ -1,2 +1,7 @@ #!/bin/sh -ipython -i -c 'from TCKUtils.utils import *' +if [[ "$@" == "--help" ]] || [[ "$@" == "-h" ]]; then + interactive='' +else + interactive='-i' +fi +ipython ${interactive} `which TCKsh.py` -- "$@" diff --git a/Hlt/TCKUtils/tests/options/dumpdiff.py b/Hlt/TCKUtils/tests/options/dumpdiff.py new file mode 100644 index 0000000000000000000000000000000000000000..8414f8778c47b2d5c2b46e61b1f4251806ab03aa --- /dev/null +++ b/Hlt/TCKUtils/tests/options/dumpdiff.py @@ -0,0 +1,11 @@ +import os +from TCKUtils.utils import * + +dump(0x11291600, file='dump_0x11291600.txt') +dump(0x21291600, file='dump_0x21291600.txt') + +diff(0x212c1605, 0x212c1606, file='diff_0x212c1605_0x212c1606.txt') +diff(0x21291600, 0x212c1600, file='diff_0x21291600_0x212c1600.txt') + +diff(0x212c1605, 0x212c1606, human=True, file='hdiff_0x212c1605_0x212c1606.txt') +diff(0x21291600, 0x212c1600, human=True, file='hdiff_0x21291600_0x212c1600.txt') diff --git a/Hlt/TCKUtils/tests/options/queries.py b/Hlt/TCKUtils/tests/options/queries.py new file mode 100644 index 0000000000000000000000000000000000000000..1dad3b19d07ac7daf5a4817e99ffb446c7176eb8 --- /dev/null +++ b/Hlt/TCKUtils/tests/options/queries.py @@ -0,0 +1,55 @@ +import unittest +import itertools +from TCKUtils.utils import * + + +class TestQueryMethods(unittest.TestCase): + + def test_getComponents(self): + expected = [ + (7, 'Hlt1ForwardHPT', 'b7aab5b0114f4dc8378148c6148ef46a'), + (8, 'Hlt1ForwardHPT.PatForwardTool', 'a9df2f268ba78325058b7184aa3a43c3'), + (9, 'Hlt1ForwardHPT.PatForwardTool.PatFwdTool', '3d9ba410de73d72172ba775099b28df1') + ] + result = getComponents(0x11291600) + result = list(itertools.islice(result, 298, 301)) + self.assertEqual(result, expected) + + def test_getAlgorithms(self): + expected = [ + (1, 'Hlt', 'c87b1cfc29bb0f766a03a346c15af9c6'), + (2, 'HltDecisionSequence', '70aab61e02f643b518a0cb5cde49296f'), + (3, 'Hlt1', 'bf20bdf0ddee88643841d87002eaed30'), + (4, 'Hlt1TrackMVA', 'f3a75204403fa8914f6243ec3d19eeca') + ] + result = getAlgorithms(0x11291600) + result = list(itertools.islice(result, 0, 4)) + self.assertEqual(result, expected) + + def test_getProperties(self): + expected = {'PatPV3DHltBeamGas.PVOfflineTool': {'BeamSpotRCut': '0.20000000'}} + result = getProperties(0x11291600, '.*PatPV3D.*', 'BeamSpotRCut') + self.assertEqual(result, expected) + + def test_getTCKInfo(self): + expected = ('Physics_pp_May2016', 'MOORE_v25r2') + result = getTCKInfo(0x11291600) + self.assertEqual(result, expected) + + def test_getReleases(self): + releases = getReleases() + self.assertTrue('MOORE_v25r2' in releases) + + def test_getTCKs(self): + expected = ('0x11291600', 'Hlt1, Physics_pp_May2016, 0x1600') + tcks = getTCKs() + tcks2 = getTCKs(release='MOORE_v25r2', hlttype='Physics_pp_May2016') + self.assertTrue(expected in tcks) + self.assertTrue(expected in tcks2) + + def test_getTCKList(self): + tcks = getTCKList() + self.assertTrue('0x11291600' in tcks) + +if __name__ == '__main__': + unittest.main() diff --git a/Hlt/TCKUtils/tests/qmtest/tckutils.qms/dumpdiff.qmt b/Hlt/TCKUtils/tests/qmtest/tckutils.qms/dumpdiff.qmt new file mode 100644 index 0000000000000000000000000000000000000000..4b6dd6056171a94d72f94f089b5eedefee99243a --- /dev/null +++ b/Hlt/TCKUtils/tests/qmtest/tckutils.qms/dumpdiff.qmt @@ -0,0 +1,44 @@ +<?xml version="1.0" ?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'> +<!-- +####################################################### +# SUMMARY OF THIS TEST +# ................... +# Author: Rosen Matev +# Purpose: Check that dump and diff functions work +# Prerequisites: None +####################################################### +--> +<extension class="GaudiTest.GaudiExeTest" kind="test"> + <argument name="program"><text>python</text></argument> + <argument name="args"><set> + <text>../options/dumpdiff.py</text> + </set></argument> + <argument name="validator"><text> + +retcode = os.system('zdiff ../refs/dump_0x11291600.txt.gz dump_0x11291600.txt') +if retcode != 0: + causes.append('Dump 1 differs, check stdout/stderr') + +retcode = os.system('zdiff ../refs/dump_0x21291600.txt.gz dump_0x21291600.txt') +if retcode != 0: + causes.append('Dump 2 differs, check stdout/stderr') + +retcode = os.system('zdiff ../refs/diff_0x212c1605_0x212c1606.txt.gz diff_0x212c1605_0x212c1606.txt') +if retcode != 0: + causes.append('Diff 1 differs, check stdout/stderr') + +retcode = os.system('zdiff ../refs/diff_0x21291600_0x212c1600.txt.gz diff_0x21291600_0x212c1600.txt') +if retcode != 0: + causes.append('Diff 2 differs, check stdout/stderr') + +retcode = os.system('zdiff ../refs/hdiff_0x212c1605_0x212c1606.txt.gz hdiff_0x212c1605_0x212c1606.txt') +if retcode != 0: + causes.append('Diff 3 differs, check stdout/stderr') + +retcode = os.system('zdiff ../refs/hdiff_0x21291600_0x212c1600.txt.gz hdiff_0x21291600_0x212c1600.txt') +if retcode != 0: + causes.append('Diff 4 differs, check stdout/stderr') + + </text></argument> +</extension> + diff --git a/Hlt/TCKUtils/tests/qmtest/tckutils.qms/queries.qmt b/Hlt/TCKUtils/tests/qmtest/tckutils.qms/queries.qmt new file mode 100644 index 0000000000000000000000000000000000000000..67d72de28e5e935310ab53d4d001993519a301cd --- /dev/null +++ b/Hlt/TCKUtils/tests/qmtest/tckutils.qms/queries.qmt @@ -0,0 +1,23 @@ +<?xml version="1.0" ?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'> +<!-- +####################################################### +# SUMMARY OF THIS TEST +# ................... +# Author: Rosen Matev +# Purpose: Check that TCKUtils query (get*) functions work +# Prerequisites: None +####################################################### +--> +<extension class="GaudiTest.GaudiExeTest" kind="test"> + <argument name="program"><text>python</text></argument> + <argument name="args"><set> + <text>../options/queries.py</text> + </set></argument> + <argument name="validator"><text> + +if stderr.strip().splitlines()[-1] != 'OK': + causes.append('Failed tests, check stderr') + + </text></argument> +</extension> + diff --git a/Hlt/TCKUtils/tests/refs/diff_0x21291600_0x212c1600.txt.gz b/Hlt/TCKUtils/tests/refs/diff_0x21291600_0x212c1600.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..513537d8327891345944227b6a1c25acfc949c95 Binary files /dev/null and b/Hlt/TCKUtils/tests/refs/diff_0x21291600_0x212c1600.txt.gz differ diff --git a/Hlt/TCKUtils/tests/refs/diff_0x212c1605_0x212c1606.txt.gz b/Hlt/TCKUtils/tests/refs/diff_0x212c1605_0x212c1606.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..6cc88fe53591fcd884c5c5583d5989eb68d678bb Binary files /dev/null and b/Hlt/TCKUtils/tests/refs/diff_0x212c1605_0x212c1606.txt.gz differ diff --git a/Hlt/TCKUtils/tests/refs/dump_0x11291600.txt.gz b/Hlt/TCKUtils/tests/refs/dump_0x11291600.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..845f2b8c6fbe211032f23db37e1121daa9bc9a31 Binary files /dev/null and b/Hlt/TCKUtils/tests/refs/dump_0x11291600.txt.gz differ diff --git a/Hlt/TCKUtils/tests/refs/dump_0x21291600.txt.gz b/Hlt/TCKUtils/tests/refs/dump_0x21291600.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..848f1d22258be4b3c3ea6e07e197784eb49e1b45 Binary files /dev/null and b/Hlt/TCKUtils/tests/refs/dump_0x21291600.txt.gz differ diff --git a/Hlt/TCKUtils/tests/refs/hdiff_0x21291600_0x212c1600.txt.gz b/Hlt/TCKUtils/tests/refs/hdiff_0x21291600_0x212c1600.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..82cea1b3c5e28963c67e17a702e8a9b817eeb35e Binary files /dev/null and b/Hlt/TCKUtils/tests/refs/hdiff_0x21291600_0x212c1600.txt.gz differ diff --git a/Hlt/TCKUtils/tests/refs/hdiff_0x212c1605_0x212c1606.txt.gz b/Hlt/TCKUtils/tests/refs/hdiff_0x212c1605_0x212c1606.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..8955cbde433cde4222bda49259f096c92fb1ee77 Binary files /dev/null and b/Hlt/TCKUtils/tests/refs/hdiff_0x212c1605_0x212c1606.txt.gz differ