From a9568859625adaf3fc42c2e87a062bec13dc225a Mon Sep 17 00:00:00 2001 From: Walter Lampl <walter.lampl@cern.ch> Date: Mon, 16 Jul 2018 09:16:21 +0000 Subject: [PATCH] ComponentAccumulator w/o addConfig Former-commit-id: 202103473e19508fdf01d3b277abbb311e0f613e --- .../CaloRec/python/CaloCellMakerConfig.py | 32 +- .../CaloRec/python/CaloTopoClusterConfig.py | 27 +- .../CaloTools/python/CaloNoiseToolConfig.py | 14 +- Control/AthenaCommon/python/Configurable.py | 19 +- .../python/ComponentAccumulator.py | 339 ++++++++++-------- .../python/UnifyProperties.py | 51 +++ Database/IOVDbSvc/python/IOVDbSvcConfig.py | 9 +- .../AtlasGeoModel/python/GeoModelConfig.py | 14 +- Event/EventInfoMgt/python/TagInfoMgrConfig.py | 35 ++ .../python/LArBadChannelConfig.py | 38 +- .../LArCabling/python/LArCablingConfig.py | 4 +- .../LArCellRec/python/LArCellBuilderConfig.py | 25 +- .../LArGeoAlgsNV/python/LArGMConfig.py | 13 +- .../TileGeoModel/python/TileGMConfig.py | 8 +- .../L1Decoder/python/L1DecoderConfig.py | 19 +- .../L1Decoder/python/L1MuonConfig.py | 19 +- .../TrigUpgradeTest/python/EgammaCaloMod.py | 72 ++-- .../TrigUpgradeTest/share/newJOtest.py | 32 +- 18 files changed, 469 insertions(+), 301 deletions(-) create mode 100644 Control/AthenaConfiguration/python/UnifyProperties.py create mode 100644 Event/EventInfoMgt/python/TagInfoMgrConfig.py diff --git a/Calorimeter/CaloRec/python/CaloCellMakerConfig.py b/Calorimeter/CaloRec/python/CaloCellMakerConfig.py index 6fb8d62c3ec..01f35296ad0 100644 --- a/Calorimeter/CaloRec/python/CaloCellMakerConfig.py +++ b/Calorimeter/CaloRec/python/CaloCellMakerConfig.py @@ -11,26 +11,31 @@ def CaloCellMakerCfg(configFlags): from LArGeoAlgsNV.LArGMConfig import LArGMCfg from TileGeoModel.TileGMConfig import TileGMCfg - result.addConfig(LArGMCfg,configFlags) - result.addConfig(TileGMCfg,configFlags) + + result.merge(LArGMCfg(configFlags)) + result.merge(TileGMCfg(configFlags)) - theLArCellMaker=result.addConfig(LArCellBuilderCfg,configFlags) + acc,theLArCellMaker=LArCellBuilderCfg(configFlags) + result.merge(acc) - theLArCellCorrectors=result.addConfig(LArCellCorrectorCfg,configFlags) + acc,theLArCellCorrectors=LArCellCorrectorCfg(configFlags) + result.merge(acc) theTileCellBuilder = TileCellBuilder() - result.addEventAlgo(CaloCellMaker(CaloCellMakerToolNames=theLArCellMaker+ - [CaloCellContainerFinalizerTool(),]+theLArCellCorrectors, - CaloCellsOutputName="AllCalo")) - return result + cellAlgo=(CaloCellMaker(CaloCellMakerToolNames=[theLArCellMaker,CaloCellContainerFinalizerTool()]+theLArCellCorrectors, + CaloCellsOutputName="AllCalo")) + return result,cellAlgo if __name__=="__main__": from AthenaCommon.Logging import log from AthenaCommon.Constants import DEBUG - log.setLevel(DEBUG) + + from AthenaCommon.Configurable import Configurable + Configurable.configurableRun3Behavior=1 + from AthenaConfiguration.AllConfigFlags import ConfigFlags ConfigFlags.set("global.InputFiles",["myRDO.pool.root"]) @@ -39,11 +44,14 @@ if __name__=="__main__": cfg=ComponentAccumulator() from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg - cfg.addConfig(PoolReadCfg,ConfigFlags) + cfg.merge(PoolReadCfg(ConfigFlags)) - cfg.addConfig(CaloCellMakerCfg,ConfigFlags) - cfg.getEventAlgo("CaloCellMaker").CaloCellsOutputName="AllCaloNew" + acc,cellMakerAlg=CaloCellMakerCfg(ConfigFlags) + cfg.merge(acc) + cellMakerAlg.CaloCellsOutputName="AllCaloNew" + cfg.addEventAlgo(cellMakerAlg) + f=open("CaloCellMaker.pkl","w") cfg.store(f) f.close() diff --git a/Calorimeter/CaloRec/python/CaloTopoClusterConfig.py b/Calorimeter/CaloRec/python/CaloTopoClusterConfig.py index afc21873354..4d22be69453 100644 --- a/Calorimeter/CaloRec/python/CaloTopoClusterConfig.py +++ b/Calorimeter/CaloRec/python/CaloTopoClusterConfig.py @@ -17,11 +17,12 @@ def CaloTopoClusterCfg(configFlags): from CaloRec.CaloRecConf import CaloTopoClusterMaker, CaloTopoClusterSplitter, CaloClusterMomentsMaker, CaloClusterMaker, CaloClusterSnapshot #, CaloClusterLockVars, CaloClusterPrinter - result.addConfig(LArGMCfg,configFlags) - result.addConfig(TileGMCfg,configFlags) - result.addConfig(CaloNoiseToolCfg,configFlags) + result.merge(LArGMCfg(configFlags)) + result.merge(TileGMCfg(configFlags)) - theCaloNoiseTool=result.getPublicTool("CaloNoiseTool") + #Get CaloNoiseTool + acc,theCaloNoiseTool=CaloNoiseToolCfg(configFlags) + result.merge(acc) # maker tools TopoMaker = CaloTopoClusterMaker("TopoMaker") @@ -94,13 +95,15 @@ def CaloTopoClusterCfg(configFlags): CaloTopoCluster.ClustersOutputName="CaloCalTopoClusters" CaloTopoCluster.ClusterMakerTools = [TopoMaker, TopoSplitter] - - result.addEventAlgo(CaloTopoCluster) - return result + + return result,CaloTopoCluster if __name__=="__main__": + from AthenaCommon.Configurable import Configurable + Configurable.configurableRun3Behavior=1 + from AthenaCommon.Logging import log from AthenaCommon.Constants import DEBUG from AthenaConfiguration.AllConfigFlags import ConfigFlags @@ -114,10 +117,14 @@ if __name__=="__main__": cfg=ComponentAccumulator() from AthenaPoolCnvSvc.PoolReadConfig import PoolReadCfg - cfg.addConfig(PoolReadCfg,ConfigFlags) + cfg.merge(PoolReadCfg(ConfigFlags)) + + topoAcc,topoAlg=CaloTopoClusterCfg(ConfigFlags) + topoAlg.ClustersOutputName="CaloCalTopoClustersNew" - cfg.addConfig(CaloTopoClusterCfg,ConfigFlags) - cfg.getEventAlgo("CaloTopoCluster").ClustersOutputName="CaloCalTopoClustersNew" + cfg.merge(topoAcc) + cfg.addEventAlgo(topoAlg) + f=open("CaloTopoCluster.pkl","w") cfg.store(f) diff --git a/Calorimeter/CaloTools/python/CaloNoiseToolConfig.py b/Calorimeter/CaloTools/python/CaloNoiseToolConfig.py index a1dfb6a1f54..c3fe4f10e23 100644 --- a/Calorimeter/CaloTools/python/CaloNoiseToolConfig.py +++ b/Calorimeter/CaloTools/python/CaloNoiseToolConfig.py @@ -18,7 +18,7 @@ def CaloNoiseToolCfg(configFlags): if configFlags.get("global.isOnline"): #online mode: folder = "/CALO/Noise/CellNoise" - result.addConfig(addFolders,configFlags,inputFlags,folder,'CALO_ONL') + result.merge(addFolders(configFlags,inputFlags,folder,'CALO_ONL')) caloNoiseToolDB.FolderNames=[folder,] if fixedLumi >= 0 : caloNoiseToolDB.Luminosity = fixedLumi @@ -26,7 +26,7 @@ def CaloNoiseToolCfg(configFlags): else: if useCaloLumi: lumiFolder='/CALO/Noise/PileUpNoiseLumi' - result.addConfig(addFolders,configFlags,lumiFolder,'CALO') + result.merge(addFolders(configFlags,lumiFolder,'CALO')) caloNoiseToolDB.LumiFolderName = lumiFolder caloNoiseToolDB.Luminosity = -1. log.info("online mode: use luminosity from /CALO/Noise/PileUpNoiseLumi to scale pileup noise") @@ -45,7 +45,7 @@ def CaloNoiseToolCfg(configFlags): else: if useCaloLumi: lumiFolder='/CALO/Ofl/Noise/PileUpNoiseLumi' - result.addConfig(addFolders,configFlags,lumiFolder,'CALO_OFL') + result.merge(addFolders(configFlags,lumiFolder,'CALO_OFL')) log.info("offline mode: use luminosity from /CALO/Ofl/Noise/PileuUpNoiseLumi to scale pileup noise") caloNoiseToolDB.LumiFolderName = lumiFolder caloNoiseToolDB.Luminosity=-1. @@ -69,11 +69,11 @@ def CaloNoiseToolCfg(configFlags): caloNoiseToolDB.Luminosity = -1 if useCaloLumi: lumiFolder='/CALO/Ofl/Noise/PileUpNoiseLumi' - result.addConfig(addFolders,configFlags,lumiFolder,'CALO_OFL') + result.merge(addFolders(configFlags,lumiFolder,'CALO_OFL')) log.info("offline mode: use luminosity from /CALO/Ofl/Noise/PileUpNoiseLumi to scale pileup noise") else: lumiFolder = '/TRIGGER/LUMI/LBLESTONL' - result.addConfig(addFolders,configFlags,lumiFolder,'TRIGGER_ONL') + result.merge(addFolders(configFlags,lumiFolder,'TRIGGER_ONL')) log.info("offline mode: use luminosity = f(Lumiblock) to scale pileup noise") caloNoiseToolDB.LumiFolderName = lumiFolder @@ -91,10 +91,10 @@ def CaloNoiseToolCfg(configFlags): pass #end of real data case for (db,fldr) in folders: - result.addConfig(addFolders,configFlags,fldr,db) + result.merge(addFolders(configFlags,fldr,db)) caloNoiseToolDB.FolderNames=[f[1] for f in folders] result.addPublicTool(caloNoiseToolDB) - return result + return result,caloNoiseToolDB diff --git a/Control/AthenaCommon/python/Configurable.py b/Control/AthenaCommon/python/Configurable.py index f0bd982aafe..213e05f0a52 100755 --- a/Control/AthenaCommon/python/Configurable.py +++ b/Control/AthenaCommon/python/Configurable.py @@ -22,6 +22,7 @@ __all__ = [ 'Configurable', 'ConfigurableUser' ] + ## for messaging from Logging import logging log = logging.getLogger( 'Configurable' ) @@ -64,6 +65,8 @@ class Configurable( object ): _printOnce = 0 + configurableRun3Behavior=0 + def __new__ ( cls, *args, **kwargs ): """To Gaudi, any object with the same type/name is the same object. Hence, this is mimicked in the configuration: instantiating a new Configurable @@ -109,7 +112,9 @@ class Configurable( object ): raise NameError( '"%s": type separator "/" no allowed in component name, '\ 'typename is derived from configurable instead' % name ) - if 'AthenaConfiguration.ComponentAccumulator' not in sys.modules.keys(): + #Uncomment the following line for debugging: + #print "cls.configurableRun3Behavior=",cls.configurableRun3Behavior + if cls.configurableRun3Behavior==0: # ordinary recycle case try: conf = cls.configurables[ name ] @@ -165,7 +170,11 @@ class Configurable( object ): (name,conf.__class__.__name__,cls.__name__) ) except KeyError: pass - pass #end if not new configuration approach + else: + #Run 3 style config + #Uncomment this line to verify that RecExCommon doesn't use that + #print "Run 3 style config" + pass #end if not new configuration approach # still here: create a new instance ... conf = object.__new__( cls ) @@ -177,10 +186,12 @@ class Configurable( object ): cls.__init__( conf, *args, **kwargs ) # update normal, per-class cache - cls.configurables[ name ] = conf + if cls.configurableRun3Behavior==0: + cls.configurables[ name ] = conf # update generics super-cache - cls.allConfigurables[ name ] = conf + if cls.configurableRun3Behavior==0: + cls.allConfigurables[ name ] = conf return conf diff --git a/Control/AthenaConfiguration/python/ComponentAccumulator.py b/Control/AthenaConfiguration/python/ComponentAccumulator.py index 7e5fb79efd9..ed445f6118e 100644 --- a/Control/AthenaConfiguration/python/ComponentAccumulator.py +++ b/Control/AthenaConfiguration/python/ComponentAccumulator.py @@ -3,57 +3,33 @@ from AthenaCommon.Logging import logging from AthenaCommon.Configurable import Configurable,ConfigurableService,ConfigurableAlgorithm,ConfigurableAlgTool from AthenaCommon.CFElements import isSequence,findSubSequence,findAlgorithm,flatSequencers,findOwningSequence -from AthenaCommon.AlgSequence import AlgSequence +from AthenaCommon.AlgSequence import AthSequencer from AthenaConfiguration.AthConfigFlags import AthConfigFlags import GaudiKernel.GaudiHandles as GaudiHandles import ast import collections +from UnifyProperties import unifyProperty + class DeduplicationFailed(RuntimeError): pass class ConfigurationError(RuntimeError): pass -class CurrentSequence: - sequence = AlgSequence("AthAlgSeq") - - @staticmethod - def set( newseq ): - #print "CurrentSequence set.... ", newseq.name() - CurrentSequence.sequence = newseq - - @staticmethod - def get( ): - #print "CurrentSequence ....get %s " % CurrentSequence.sequence.name() - return CurrentSequence.sequence - - -_propsToUnify=frozenset(("GeoModelSvc.DetectorTools","CondInputLoader.Load", - "IOVDbSvc.Folders", - "IOVDbSvc.FoldersToMetaData", - "EvtPersistencySvc.CnvServices", - "PoolSvc.ReadCatalog","ProxyProviderSvc.ProviderNames" )) - -def unifyProp(prop1,prop2): - #May want to strip whitespace in case the params are lists of strings - s1=set(prop1) - s2=set(prop2) - su=s1 | s2 - return list(su) +_servicesToCreate=frozenset(('GeoModelSvc','TileInfoLoader')) class ComponentAccumulator(object): def __init__(self): self._msg=logging.getLogger('ComponentAccumulator') - self._sequence=CurrentSequence.get() # sequence of algorithms - self._eventAlgs={} #Unordered list of event processing algorithms per sequence + their private tools + self._sequence=AthSequencer() #(Nested) sequence of event processing algorithms per sequence + their private tools self._conditionsAlgs=[] #Unordered list of conditions algorithms + their private tools self._services=[] #List of service, not yet sure if the order matters here in the MT age self._conditionsInput=set() #List of database folder (as string), eventually passed to IOVDbSvc self._eventInputs=set() #List of items (as strings) to be read from the input (required at least for BS-reading). - self._outputPerStream={} #Dictionary of {streamName,set(items)}, all as strings + self._outputPerStream={} #Dictionary of {streamName,set(items)}, all as strings self._theAppProps=dict() #Properties of the ApplicationMgr @@ -94,31 +70,32 @@ class ComponentAccumulator(object): self._msg.info( self._outputPerStream ) - def addSequence(self, newseq, sequence = None ): + def addSequence(self, newseq, parentName = None ): """ Adds new sequence. If second argument is present then it is added under another sequence """ - seq = CurrentSequence.get() - if sequence: - seq = findSubSequence(seq, sequence ) - if seq is None: - raise ConfigurationError("Missing sequence %s to add new sequence to" % sequence ) + if parentName is None: + parent=self._sequence + else: + parent = findSubSequence(self._sequence, parentName ) + if parent is None: + raise ConfigurationError("Missing sequence %s to add new sequence to" % parentName ) - if findSubSequence( self._sequence, newseq.name() ): + if findSubSequence( parent, newseq.name() ): raise ConfigurationError("Sequence %s already present" % newseq.name() ) - seq += newseq + parent += newseq return newseq + def moveSequence(self, sequence, destination ): - """ moves seqeunce from one sub-sequence to another, primary use case HLT Control Flow """ - start = CurrentSequence.get() - seq = findSubSequence(start, sequence ) + """ moves sequence from one sub-sequence to another, primary use case HLT Control Flow """ + seq = findSubSequence(self._sequence, sequence ) if seq is None: raise ConfigurationError("Can not find sequence to move %s " % sequence ) - owner = findOwningSequence(start, sequence) + owner = findOwningSequence(self._sequence, sequence) if owner is None: raise ConfigurationError("Can not find the sequence owning the %s " % sequence ) - dest = findSubSequence(start, destination ) + dest = findSubSequence(self._sequence, destination ) if dest is None: raise ConfigurationError("Can not find destination sequence %s to move to " % destination ) @@ -126,32 +103,49 @@ class ComponentAccumulator(object): dest += seq return seq - def addEventAlgo(self, algo,sequence=None): - if not isinstance(algo, ConfigurableAlgorithm): - raise TypeError("Attempt to add wrong type: %s as event algorithm" % type( algo ).__name__) - pass - seq = CurrentSequence.get() - if sequence is not None: - seq = findSubSequence( seq, sequence ) - if seq is None: - raise ConfigurationError("Unable to add %s to sequence %s as it is missing", algo.getFullName(), seq.name() ) - - self._msg.debug("Adding %s to sequence %s", algo.getFullName(), seq.name() ) - seq += algo - return algo + def getSequence(self,sequenceName=None): + if sequenceName is None: + return self._sequence + else: + return findSubSequence(self._sequence,sequenceName) - def getEventAlgo(self,name): - """ Looks for an algorithm given the name in current sequence and in nested scopes - NB. Not sure that this is what we expect. Maybe we want to start from the top always? - Limiting to the current scope reduces risk of cross talk. Will see in real life and make adjustments. - """ - algo = findAlgorithm( CurrentSequence.get(), name ) + + def addEventAlgo(self, algorithms,sequenceName=None): + if not isinstance(algorithms,collections.Sequence): + #Swallow both single algorithms as well as lists or tuples of algorithms + algorithms=[algorithms,] + + if sequenceName is None: + seq=self._sequence + else: + seq = findSubSequence(self._sequence, sequenceName ) + if seq is None: + raise ConfigurationError("Can not find sequence %s" % sequenceName ) + + + for algo in algorithms: + if not isinstance(algo, ConfigurableAlgorithm): + raise TypeError("Attempt to add wrong type: %s as event algorithm" % type( algo ).__name__) + + seq+=algo #TODO: Deduplication necessary? + pass + return None + + + def getEventAlgo(self,name,seqName=None): + if seqName is None: + seq=self._sequence + else: + seq = findSubSequence(self._sequence, seqName ) + + algo = findAlgorithm( seq, name ) if algo is None: raise ConfigurationError("Can not find an algorithm of name %s "% name) return algo + def addCondAlgo(self,algo): if not isinstance(algo, ConfigurableAlgorithm): raise TypeError("Attempt to add wrong type: %s as conditions algorithm" % type( algo ).__name__) @@ -203,14 +197,22 @@ class ComponentAccumulator(object): #Note that getattr for a list property works, even if it's not in ValuedProperties if (oldprop!=newprop): #found property mismatch - if isinstance(oldprop,list): #if properties are concatinable, do that! + if isinstance(oldprop,list): #if properties are mergeable, do that! propid="%s.%s" % (comp.getType(),str(prop)) - if propid not in _propsToUnify: - raise DeduplicationFailed("List property %s defined multiple times with conflicting values.\n " % propid \ - +"If this property should be concatinated, consider adding it to the _propsToUnify set") - - mergeprop=unifyProp(oldprop,newprop) + #Try merging this property. Will raise on failure + mergeprop=unifyProperty(propid,oldprop,newprop) setattr(comp,prop,mergeprop) + elif isinstance(oldprop,dict): #Dicts/maps can be unified + #find conflicting keys + doubleKeys= set(oldprop.keys()) & set(prop.keys()) + for k in doubleKeys(): + if oldprop[k]!= prop[k]: + raise DeduplicationFailed("Map-property '%s.%s' defined multiple times with conflicting values for key %s" % \ + (comp.getJobOptName(),str(prop),k)) + pass + mergeprop=oldprop + mergeprop.update(prop) + else: #self._msg.error("component '%s' defined multiple times with mismatching configuration", svcs[i].getJobOptName()) raise DeduplicationFailed("component '%s' defined multiple times with mismatching property %s" % \ @@ -228,8 +230,13 @@ class ComponentAccumulator(object): #end loop over existing components #No component of the same type & name found, simply append - self._msg.debug("Adding service/Tool/CondAlog %s to the job", newComp.getFullName()) - compList.append(newComp) + self._msg.debug("Adding component %s to the job", newComp.getFullName()) + + #The following is to work with internal list of service as well as gobal svcMgr as second parameter + try: + compList.append(newComp) + except: + compList+=newComp return True #True means something got added def getService(self,name): @@ -273,19 +280,34 @@ class ComponentAccumulator(object): self._theAppProps[key]=value pass - def __merge(self,other): + def merge(self,other): """ Merging in the other accumulator """ + if other is None: + return + + if isinstance(other,collections.Sequence): #Check if we got more than one argument + if len(other)==0: + raise ConfigurationError("Merge called with empty sequence as argument") + for par in other[1:]: + if par is not None: #possible improvment: Check type of par and try to merge if applicable (service, public tool) + self._msg.warning("Merge called with a sequence of potentially un-merged components") + raise RuntimeError() + other=other[0] + if not isinstance(other,ComponentAccumulator): raise TypeError("Attempt merge wrong type %s. Only instances of ComponentAccumulator can be added" % type(other).__name__) + if not Configurable.configurableRun3Behavior: + raise ConfigurationError("discoverd Configurable.configurableRun3Behavior=False while working woth ComponentAccumulator") + #destSubSeq = findSubSequence(self._sequence, sequence) #if destSubSeq == None: # raise ConfigurationError( "Nonexistent sequence %s in %s (or its sub-sequences)" % ( sequence, self._sequence.name() ) ) # def mergeSequences( dest, src ): for c in src.getChildren(): if isSequence( c ): - sub = findSubSequence( dest, c.name() ) + sub = findSubSequence( dest, c.name() ) #depth=1 ??? if sub: mergeSequences(sub, c ) else: @@ -301,6 +323,8 @@ class ComponentAccumulator(object): dest += c + #Merge sequences: + mergeSequences(self._sequence,other._sequence) # Merge Conditions inputs self._conditionsInput|=other._conditionsInput @@ -327,50 +351,38 @@ class ComponentAccumulator(object): self.setAppProperty(k,v) #Will warn about overrides - def addConfig(self,fct,configFlags,*args,**kwargs): - """ The heart and soul of configuration system. You need to read the whole documentation. - This method eliminates possibility that a downstream configuration alters the upstream one. - It is done by a two-fold measures: - - the sub-accumulators can not access the upstream accumulators and thus alter any configuration. - The combination process is defined in the __merge method of this class. Less flexibility == robustness. - """ + def appendToGlobals(self): + from AthenaCommon.AppMgr import ToolSvc, ServiceMgr, theApp + + for s in self._services: + self._deduplicate(s,ServiceMgr) + + if s.getJobOptName() in _servicesToCreate \ + and s.getJobOptName() not in theApp.CreateSvc: + theApp.CreateSvc.append(s.getJobOptName()) - #Safety check: Verify that the ConfigFlags are indeed locked: - if not isinstance(configFlags,AthConfigFlags): - raise ConfigurationError("Expected an instance of AthConfigFlags as parameter") + - if not configFlags.locked(): - raise ConfigurationError("You are trying to create a job using unlocked configFlags!") + for t in self._publicTools: + self._deduplicate(t,ToolSvc) + + condseq=AthSequencer ("AthCondSeq") + for c in self._conditionsAlgs: + self._deduplicate(c,condseq) - - currentSeq = seq = CurrentSequence.get() - if 'sequence' in kwargs: - seq = findSubSequence(seq, kwargs['sequence'] ) - if seq is None: - raise ConfigurationError("Can not add algorithms to sequence %s as it does not exist" % kwargs['sequence'] ) - else: - del kwargs['sequence'] - CurrentSequence.set( seq ) + for seqName, algoList in flatSequencers( self._sequence ).iteritems(): + seq=AthSequencer(seqName) + for alg in algoList: + seq+=alg + - self._msg.info("Executing configuration function %s", fct.__name__) - retval=fct(configFlags,*args,**kwargs) - CurrentSequence.set( currentSeq ) + for (k,v) in self._theAppProps.iteritems(): + if k not in [ 'CreateSvc', 'ExtSvc']: + setattr(theApp,k,v) + return - if isinstance(retval,collections.Sequence) and len(retval)>0: - toMerge=retval[0] - toReturn=list(retval[1:]) - elif isinstance(retval,ComponentAccumulator): - toMerge=retval - toReturn=[] - else: - print retval - raise ConfigurationError("Return value of configuration methods must be instaces of ComponentAccumulator or tuples starting with an instance of ComponentAccumulator") - - - self.__merge(toMerge) - return toReturn def appendConfigurable(self,confElem): name=confElem.getJobOptName() # to be FIXED @@ -449,18 +461,13 @@ class ComponentAccumulator(object): #Hack for now: self._jocfg["ApplicationMgr"]["CreateSvc"]=['ToolSvc/ToolSvc', 'AthDictLoaderSvc/AthDictLoaderSvc', 'AthenaSealSvc/AthenaSealSvc', 'CoreDumpSvc/CoreDumpSvc'] - + svcList=ast.literal_eval(self._jocfg["ApplicationMgr"]["ExtSvc"]) - def __addif( name ): - for svc in self._services: - if name == svc.getJobOptName(): - self._jocfg["ApplicationMgr"]["CreateSvc"].append( svc.getFullName() ) - __addif('DetDescrCnvSvc') - __addif('GeoModelSvc') - __addif('TileInfoLoader') - for svc in self._services: + if svc.getJobOptName() in _servicesToCreate: + self._jocfg["ApplicationMgr"]["CreateSvc"].append( svc.getFullName() ) + svcList.append(svc.getFullName()) #for k, v in svc.getValuedProperties().items(): # self._jocat[svcname][k]=str(v) @@ -481,8 +488,25 @@ class ComponentAccumulator(object): pickle.dump( self._jocfg, outfile ) pickle.dump( self._pycomps, outfile ) + + +def CAtoGlobalWrapper(cfgmethod,flags): + Configurable.configurableRun3Behavior+=1 + result=cfgmethod(flags) + Configurable.configurableRun3Behavior-=1 + + result.appendToGlobals() + return + + + + + + + # self test if __name__ == "__main__": + Configurable.configurableRun3Behavior+=1 # trivial case without any nested sequences from AthenaCommon.Configurable import ConfigurablePyAlgorithm # guinea pig algorithms from AthenaCommon.CFElements import seqAND, seqOR, parOR @@ -500,56 +524,61 @@ if __name__ == "__main__": def AlgsConf1(flags): acc = ComponentAccumulator() - acc.addEventAlgo( Algo("Algo1") ) - acc.addEventAlgo( Algo("Algo2") ) - return acc + a1=Algo("Algo1") + a2=Algo("Algo2") + return acc,[a1,a2] def AlgsConf2(flags): acc = ComponentAccumulator() - acc.addConfig( AlgsConf1, flags ) - acc.addEventAlgo( Algo("Algo3") ) - return acc + result,algs=AlgsConf1( flags ) + acc.merge(result) + algs.append(Algo("Algo3")) + return acc,algs acc = ComponentAccumulator() # top level algs - acc.addConfig( AlgsConf2,dummyCfgFlags ) + acc1,algs=AlgsConf2(dummyCfgFlags) + acc.merge(acc1) + acc.addEventAlgo(algs) # checks - assert findAlgorithm(AlgSequence("AthAlgSeq"), "Algo1", 1), "Algorithm not added to a top sequence" - assert findAlgorithm(AlgSequence("AthAlgSeq"), "Algo2", 1), "Algorithm not added to a top sequence" - assert findAlgorithm(AlgSequence("AthAlgSeq"), "Algo3", 1), "Algorithm not added to a top sequence" + assert findAlgorithm(acc.getSequence(), "Algo1", 1), "Algorithm not added to a top sequence" + assert findAlgorithm(acc.getSequence(), "Algo2", 1), "Algorithm not added to a top sequence" + assert findAlgorithm(acc.getSequence(), "Algo3", 1), "Algorithm not added to a top sequence" print( "Simple Configuration construction OK ") def AlgsConf3(flags): acc = ComponentAccumulator() - acc.addEventAlgo( Algo("NestedAlgo1") ) - return acc + na1=Algo("NestedAlgo1") + return acc,na1 def AlgsConf4(flags): - acc = ComponentAccumulator() - acc.addConfig( AlgsConf3,flags ) + acc,na1= AlgsConf3( flags ) NestedAlgo2 = Algo("NestedAlgo2") NestedAlgo2.OutputLevel=7 - acc.addEventAlgo( NestedAlgo2 ) - return acc + return acc,na1,NestedAlgo2 acc.addSequence( seqAND("Nest") ) - acc.addSequence( seqAND("subSequence1"), sequence="Nest" ) - acc.addSequence( parOR("subSequence2"), sequence="Nest" ) - - assert findSubSequence( AlgSequence("AthAlgSeq"), "subSequence1" ), "Adding sub-sequence failed" - assert findSubSequence( AlgSequence("AthAlgSeq"), "subSequence2" ), "Adding sub-sequence failed" - - acc.addSequence( seqAND("sub2Sequence1"), "subSequence1") - acc.addSequence( seqAND("sub3Sequence1"), "subSequence1") - acc.addSequence( seqAND("sub4Sequence1"), "subSequence1") - assert findSubSequence(AlgSequence("AthAlgSeq"), "sub2Sequence1"), "Adding sub-sequence failed" - assert findSubSequence( findSubSequence(AlgSequence("AthAlgSeq"), "subSequence1"), "sub2Sequence1" ), "Adding sub-sequence done in a wrong place" - - acc.addConfig( AlgsConf4, dummyCfgFlags, sequence="sub2Sequence1" ) - assert findAlgorithm(AlgSequence("AthAlgSeq"), "NestedAlgo1" ), "Algorithm added to nested sequence" - assert findAlgorithm(AlgSequence("AthAlgSeq"), "NestedAlgo1", 1 ) is None, "Algorithm mistakenly in top sequence" - assert findAlgorithm( findSubSequence(AlgSequence("AthAlgSeq"), "sub2Sequence1"), "NestedAlgo1", 1 ), "Algorithm not in right sequence" + acc.addSequence( seqAND("subSequence1"), parentName="Nest" ) + acc.addSequence( parOR("subSequence2"), parentName="Nest" ) + + + assert acc.getSequence("subSequence1" ), "Adding sub-sequence failed" + assert acc.getSequence("subSequence2" ), "Adding sub-sequence failed" + + acc.addSequence( seqAND("sub2Sequence1"), parentName="subSequence1") + acc.addSequence( seqAND("sub3Sequence1"), parentName="subSequence1") + acc.addSequence( seqAND("sub4Sequence1"), parentName="subSequence1") + assert acc.getSequence("sub2Sequence1"), "Adding sub-sequence failed" + assert findSubSequence(acc.getSequence("subSequence1"), "sub2Sequence1"), "Adding sub-sequence done in a wrong place" + + accNA1=AlgsConf4(dummyCfgFlags) + acc.merge(accNA1[0]) + acc.addEventAlgo(accNA1[1:],"sub2Sequence1" ) + + assert findAlgorithm(acc.getSequence(), "NestedAlgo1" ), "Algorithm added to nested sequence" + assert findAlgorithm(acc.getSequence(), "NestedAlgo1", 1 ) is None, "Algorithm mistakenly in top sequence" + assert findAlgorithm( findSubSequence(acc.getSequence(), "sub2Sequence1"), "NestedAlgo1", 1 ), "Algorithm not in right sequence" print( "Complex sequences construction also OK ") #acc.printConfig(True) @@ -564,11 +593,13 @@ if __name__ == "__main__": # some of them (in this case hltSteps) did not have properties recorded acc = ComponentAccumulator() acc.addSequence( seqOR("hltTop") ) - acc.addConfig( AlgsConf2, dummyCfgFlags, sequence="hltTop" ) # some algo - acc.addSequence( seqAND("hltSteps"), sequence="hltTop" ) - acc.addSequence( parOR("hltStep_1"), sequence="hltSteps" ) + acc2,algos2=AlgsConf2(dummyCfgFlags) + acc.merge(acc2) + acc.addEventAlgo(algos2,sequenceName="hltTop") # some algo + acc.addSequence( seqAND("hltSteps"), parentName="hltTop" ) + acc.addSequence( parOR("hltStep_1"), parentName="hltSteps" ) acc.addSequence( seqAND("L2CaloEgammaSeq"), "hltStep_1" ) - acc.addSequence( parOR("hltStep_2"), sequence="hltSteps" ) + acc.addSequence( parOR("hltStep_2"), parentName="hltSteps" ) acc.moveSequence( "L2CaloEgammaSeq", "hltStep_2" ) acc.printConfig() diff --git a/Control/AthenaConfiguration/python/UnifyProperties.py b/Control/AthenaConfiguration/python/UnifyProperties.py new file mode 100644 index 00000000000..4649d4bb5ff --- /dev/null +++ b/Control/AthenaConfiguration/python/UnifyProperties.py @@ -0,0 +1,51 @@ +# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration + +# A collection of methods to unify/merge list-properties +# ToDo: Define the merging-method when defining the property + + +def unifySet(prop1,prop2): + #May want to strip whitespace in case the params are lists of strings + s1=set(prop1) + s2=set(prop2) + su=s1 | s2 + return list(su) + +def unifySetOfPairs(prop1,prop2): + + def getPairs(seq): + r=set() + nPairs=int(len(seq)/2) + if (2*nPairs!=len(seq)): + from AthenaConfiguration.ComponentAccumulator import ConfigurationError + raise ConfigurationError("Expected a sequence with even number of elements") + + for i in range(0,nPairs): + r.add((seq[2*i],seq[2*i+1])) + return r + + setOfPairs1=getPairs(prop1) + setOfPairs2=getPairs(prop2) + unionOfPairs=(setOfPairs1 | setOfPairs2) + finallist=[] + for p in unionOfPairs: + finallist+=p + return finallist + +_propsToUnify={"GeoModelSvc.DetectorTools":unifySet, + "CondInputLoader.Load":unifySet, + "IOVDbSvc.Folders":unifySet, + "IOVDbSvc.FoldersToMetaData":unifySet, + "EvtPersistencySvc.CnvServices":unifySet, + "PoolSvc.ReadCatalog":unifySet, + "ProxyProviderSvc.ProviderNames":unifySet, + "TagInfoMgr.ExtraTagValuePairs":unifySetOfPairs, + } + + +def unifyProperty(propname,prop1,prop2): + if propname not in _propsToUnify: + from AthenaConfiguration.ComponentAccumulator import DeduplicationFailed + raise DeduplicationFailed("List property %s defined multiple times with conflicting values.\n " % propname \ + +"If this property should be merged, consider adding it to AthenaConfiguration/UnifyProperties.py") + return _propsToUnify[propname](prop1,prop2) diff --git a/Database/IOVDbSvc/python/IOVDbSvcConfig.py b/Database/IOVDbSvc/python/IOVDbSvcConfig.py index a7c98165833..2cf7568a56b 100644 --- a/Database/IOVDbSvc/python/IOVDbSvcConfig.py +++ b/Database/IOVDbSvc/python/IOVDbSvcConfig.py @@ -72,7 +72,7 @@ def IOVDbSvcCfg(configFlags): result.addService(DBReplicaSvc(COOLSQLiteVetoPattern="/DBRelease/")) - return result + return result,iovDbSvc #Convenience method to add folders: @@ -83,8 +83,7 @@ def addFolders(configFlags,folderstrings,detDb=None,className=None): if isinstance(folderstrings,str): folderstrings=[folderstrings,] - result=ComponentAccumulator() - result.addConfig(IOVDbSvcCfg,configFlags) + result,iovDbSvc=IOVDbSvcCfg(configFlags) #Add class-name to CondInputLoader (if reqired) if className is not None: @@ -95,7 +94,7 @@ def addFolders(configFlags,folderstrings,detDb=None,className=None): #result.addCondAlgo(CondInputLoader(Load=loadFolders)) - iovDbSvc=result.getService("IOVDbSvc") + if detDb is not None: dbname=configFlags.get("IOVDb.DatabaseInstance") @@ -112,7 +111,7 @@ def addFolders(configFlags,folderstrings,detDb=None,className=None): else: iovDbSvc.Folders.append(fs) - return result + return result,None _dblist={ diff --git a/DetectorDescription/GeoModel/AtlasGeoModel/python/GeoModelConfig.py b/DetectorDescription/GeoModel/AtlasGeoModel/python/GeoModelConfig.py index 8853c7b5ee8..743a4bc4747 100644 --- a/DetectorDescription/GeoModel/AtlasGeoModel/python/GeoModelConfig.py +++ b/DetectorDescription/GeoModel/AtlasGeoModel/python/GeoModelConfig.py @@ -12,8 +12,10 @@ def GeoModelCfg(configFlags): result=ComponentAccumulator() from GeoModelSvc.GeoModelSvcConf import GeoModelSvc - result.addService(GeoModelSvc(AtlasVersion=version, - SupportedGeometry = int(relversion[0]))) + gms=GeoModelSvc(AtlasVersion=version, + SupportedGeometry = int(relversion[0])) + + result.addService(gms) from DetDescrCnvSvc.DetDescrCnvSvcConf import DetDescrCnvSvc from GaudiSvc.GaudiSvcConf import EvtPersistencySvc @@ -23,4 +25,10 @@ def GeoModelCfg(configFlags): result.addService(detDescrCnvSvc) result.addService(EvtPersistencySvc("EventPersistencySvc",CnvServices=[detDescrCnvSvc.getName(),])) #No service handle yet??? - return result + from EventInfoMgt.TagInfoMgrConfig import TagInfoMgrCfg + tim_ca,tagInfoMgr=TagInfoMgrCfg(configFlags) + result.addService(tagInfoMgr) + result.merge(tim_ca) + #TagInfoMgr used by GeoModelSvc but no ServiceHandle. Relies on string-name + + return result,gms diff --git a/Event/EventInfoMgt/python/TagInfoMgrConfig.py b/Event/EventInfoMgt/python/TagInfoMgrConfig.py new file mode 100644 index 00000000000..fdd3de9bd9d --- /dev/null +++ b/Event/EventInfoMgt/python/TagInfoMgrConfig.py @@ -0,0 +1,35 @@ +from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator, ConfigurationError +import os +import collections + + +def TagInfoMgrCfg(configFlags,tagValuePairs=[]): + + #Sanity check: + if not isinstance(tagValuePairs,collections.Sequence) or len(tagValuePairs)%2!=0: + raise ConfigurationError("Parameter extraTagValuePairs is supposed to be an even-numbered list of strings") + + result=ComponentAccumulator() + + from EventInfoMgtConf import TagInfoMgr + from SGComps.SGCompsConf import ProxyProviderSvc + from GaudiSvc.GaudiSvcConf import EvtPersistencySvc + + #Build project-version string for the TagInfoMgr + project = os.getenv('AtlasProject',"Unknown") + version = os.getenv('AtlasVersion',"Unknown") + atlasRelease=project+"-"+version + + releasetag=["AtlasRelease", atlasRelease ] + + tagInfoMgr=TagInfoMgr(ExtraTagValuePairs = releasetag+list(tagValuePairs)) + result.addService(tagInfoMgr) + + #Add to EventPersistencySvc + result.addService(EvtPersistencySvc("EventPersistencySvc",CnvServices=[tagInfoMgr.getName(),])) + + #Add to ProxyProviderSvc + result.addService(ProxyProviderSvc(ProviderNames=[tagInfoMgr.getName(),])) + + return result,tagInfoMgr + diff --git a/LArCalorimeter/LArBadChannelTool/python/LArBadChannelConfig.py b/LArCalorimeter/LArBadChannelTool/python/LArBadChannelConfig.py index 558ab72a15e..ef1f1001f02 100644 --- a/LArCalorimeter/LArBadChannelTool/python/LArBadChannelConfig.py +++ b/LArCalorimeter/LArBadChannelTool/python/LArBadChannelConfig.py @@ -4,9 +4,8 @@ from LArCabling.LArCablingConfig import LArOnOffIdMappingCfg from IOVDbSvc.IOVDbSvcConfig import addFolders def LArBadChannelCfg(configFlags): - result=ComponentAccumulator() - result.addConfig(LArOnOffIdMappingCfg,configFlags) + result=LArOnOffIdMappingCfg(configFlags)[0] if configFlags.get("global.isOnline") or configFlags.get("global.isMC"): foldername="/LAR/BadChannels/BadChannels" @@ -19,10 +18,11 @@ def LArBadChannelCfg(configFlags): else: dbname="LAR_OFL" - result.addConfig(addFolders,configFlags,foldername,detDb=dbname,className="CondAttrListCollection") - - result.addCondAlgo(LArBadChannelCondAlg(ReadKey=foldername)) - return result + result.merge(addFolders(configFlags,foldername,detDb=dbname,className="CondAttrListCollection")[0]) + + theLArBadChannelCondAlgo=LArBadChannelCondAlg(ReadKey=foldername) + result.addCondAlgo(theLArBadChannelCondAlgo) + return result,None def LArBadFebCfg(configFlags): @@ -39,21 +39,16 @@ def LArBadFebCfg(configFlags): else: dbname="LAR_OFL" - result.addConfig(addFolders,configFlags,foldername,detDb=dbname,className="AthenaAttributeList") - + result.merge(addFolders(configFlags,foldername,detDb=dbname,className="AthenaAttributeList")[0]) result.addCondAlgo(LArBadFebCondAlg(ReadKey=foldername)) - return result + return result,None -def LArBadChannelMaskerCfg(configFlags,problemsToMask,doMasking=True, - ToolName="LArBadChannelMasker"): - result=ComponentAccumulator() - result.addConfig(LArBadChannelCfg,configFlags) +def LArBadChannelMaskerCfg(configFlags,problemsToMask,doMasking=True,ToolName="LArBadChannelMasker"): + result=LArBadChannelCfg(configFlags)[0] - bcMasker=LArBadChannelMasker(ToolName,ProblemsToMask=problemsToMask, - DoMasking=doMasking) - - return result,bcMasker + bcMasker=LArBadChannelMasker(ToolName,ProblemsToMask=problemsToMask, DoMasking=doMasking) + return result,bcMasker @@ -62,6 +57,8 @@ if __name__=="__main__": from AthenaConfiguration.AllConfigFlags import ConfigFlags from AthenaCommon.Logging import log from AthenaCommon.Constants import DEBUG + from AthenaCommon.Configurable import Configurable + Configurable.configurableRun3Behavior=1 log.setLevel(DEBUG) ConfigFlags.set("global.isMC",False) @@ -70,9 +67,10 @@ if __name__=="__main__": cfg=ComponentAccumulator() - cfg.addConfig(LArBadChannelCfg,ConfigFlags) - cfg.addConfig(LArBadFebCfg,ConfigFlags) - cfg.addConfig(LArBadChannelMaskerCfg,ConfigFlags,["allDead",]) + cfg.merge(LArBadChannelCfg(ConfigFlags)) + cfg.merge(LArBadFebCfg(ConfigFlags)) + acc,privTool=LArBadChannelMaskerCfg(ConfigFlags,["allDead",]) + cfg.merge(acc) f=open("LArBCCondAlgos.pkl","w") cfg.store(f) f.close() diff --git a/LArCalorimeter/LArCabling/python/LArCablingConfig.py b/LArCalorimeter/LArCabling/python/LArCablingConfig.py index bd7c0847882..4df3f1846dc 100644 --- a/LArCalorimeter/LArCabling/python/LArCablingConfig.py +++ b/LArCalorimeter/LArCabling/python/LArCablingConfig.py @@ -23,8 +23,8 @@ def _larCablingCfg(configFlags,algo,folder): folderwithtag=folder result.addCondAlgo(algo(ReadKey=folder)) - result.addConfig(addFolders,configFlags,folderwithtag,className="AthenaAttributeList",detDb=db) - return result + result.merge(addFolders(configFlags,folderwithtag,className="AthenaAttributeList",detDb=db)) + return result,None def LArOnOffIdMappingCfg(configFlags): diff --git a/LArCalorimeter/LArCellRec/python/LArCellBuilderConfig.py b/LArCalorimeter/LArCellRec/python/LArCellBuilderConfig.py index ac799e76f93..877c338b8f1 100644 --- a/LArCalorimeter/LArCellRec/python/LArCellBuilderConfig.py +++ b/LArCalorimeter/LArCellRec/python/LArCellBuilderConfig.py @@ -5,8 +5,7 @@ from LArCellRec.LArCellRecConf import LArCellBuilderFromLArRawChannelTool, LArCe from LArCabling.LArCablingConfig import LArOnOffIdMappingCfg def LArCellBuilderCfg(configFlags): - result=ComponentAccumulator() - result.addConfig(LArOnOffIdMappingCfg,configFlags) + result=LArOnOffIdMappingCfg(configFlags) theLArCellBuilder = LArCellBuilderFromLArRawChannelTool() theLArCellBuilder.addDeadOTX = False #Create flag? Requires bad-feb DB access @@ -20,31 +19,27 @@ def LArCellCorrectorCfg(configFlags): correctionTools=[] if configFlags.get("LAr.RawChannelSource")=="both": - theMerger=result.addConfig(LArCellMerger,configFlags,RawChannelsName="LArRawChannels_FromDigits") - correctionTools.append(theMerger[0]) + theMerger=LArCellMerger(RawChannelsName="LArRawChannels_FromDigits") + correctionTools.append(theMerger) if configFlags.get("LAr.doCellNoiseMasking") or configFlags.get("LAr.doCellSporadicNoiseMasking"): from LArBadChannelTool.LArBadChannelConfig import LArBadChannelMaskerCfg theNoiseMasker=LArCellNoiseMaskingTool() if configFlags.get("LAr.doCellNoiseMasking"): - cellNoiseMaskingTool=result.addConfig(LArBadChannelMaskerCfg,configFlags, - problemsToMask=["highNoiseHG","highNoiseMG","highNoiseLG","deadReadout","deadPhys"], - ToolName="CellNoiseMask") - theNoiseMasker.MaskingTool=cellNoiseMaskingTool[0] + acc,cellNoiseMaskingTool= LArBadChannelMaskerCfg(configFlags,problemsToMask=["highNoiseHG","highNoiseMG","highNoiseLG","deadReadout","deadPhys"],ToolName="CellNoiseMask") + result.merge(acc) + theNoiseMasker.MaskingTool=cellNoiseMaskingTool pass if configFlags.get("LAr.doCellSporadicNoiseMasking"): - sporadicNoiseMaskingTool=result.addConfig(LArBadChannelMaskerCfg,configFlags, - problemsToMask=["sporadicBurstNoise",], - ToolName="SporadicNoiseMask") - theNoiseMasker.MaskingSporadicTool=sporadicNoiseMaskingTool[0] + acc,sporadicNoiseMaskingTool=LArBadChannelMaskerCfg(configFlags,problemsToMask=["sporadicBurstNoise",],ToolName="SporadicNoiseMask") + result.merge(acc) + theNoiseMasker.MaskingSporadicTool=sporadicNoiseMaskingTool pass correctionTools.append(theNoiseMasker) - pass - #Many more tools to be added, eg HV correction - return [result,]+correctionTools + return result,correctionTools diff --git a/LArCalorimeter/LArGeoModel/LArGeoAlgsNV/python/LArGMConfig.py b/LArCalorimeter/LArGeoModel/LArGeoAlgsNV/python/LArGMConfig.py index 2eff7b7a9c5..58bf13fd062 100644 --- a/LArCalorimeter/LArGeoModel/LArGeoAlgsNV/python/LArGMConfig.py +++ b/LArCalorimeter/LArGeoModel/LArGeoAlgsNV/python/LArGMConfig.py @@ -3,22 +3,23 @@ from AtlasGeoModel.GeoModelConfig import GeoModelCfg from IOVDbSvc.IOVDbSvcConfig import addFolders def LArGMCfg(configFlags): - result=ComponentAccumulator() - result.addConfig(GeoModelCfg,configFlags) + result,gms=GeoModelCfg(configFlags) doAlignment=configFlags("LAr.doAlign") from LArGeoAlgsNV.LArGeoAlgsNVConf import LArDetectorToolNV - result.getService("GeoModelSvc").DetectorTools += [ LArDetectorToolNV(ApplyAlignments=doAlignment) ] + gms.DetectorTools += [ LArDetectorToolNV(ApplyAlignments=doAlignment) ] + + result.addService(gms) if doAlignment: if configFlags.get("global.isMC"): #Monte Carlo case: - result.addConfig(addFolders,configFlags,["/LAR/Align","/LAR/LArCellPositionShift"],"LAR_OFL") + result.merge(addFolders(configFlags,["/LAR/Align","/LAR/LArCellPositionShift"],"LAR_OFL")[0]) else: #Regular offline data processing - result.addConfig(addFolders,configFlags,["/LAR/Align","/LAR/LArCellPositionShift"],"LAR_ONL") + result.merge(addFolders(configFlags,["/LAR/Align","/LAR/LArCellPositionShift"],"LAR_ONL")[0]) - + return result diff --git a/TileCalorimeter/TileGeoModel/python/TileGMConfig.py b/TileCalorimeter/TileGeoModel/python/TileGMConfig.py index 917e65ac7e6..51549323027 100644 --- a/TileCalorimeter/TileGeoModel/python/TileGMConfig.py +++ b/TileCalorimeter/TileGeoModel/python/TileGMConfig.py @@ -2,11 +2,11 @@ from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator from AtlasGeoModel.GeoModelConfig import GeoModelCfg def TileGMCfg(configFlags): - result=ComponentAccumulator() - - result.addConfig(GeoModelCfg,configFlags) + result,gms=GeoModelCfg(configFlags) from TileGeoModel.TileGeoModelConf import TileDetectorTool - result.getService("GeoModelSvc").DetectorTools += [ TileDetectorTool() ] + gms.DetectorTools += [ TileDetectorTool() ] + + result.addService(gms) return result diff --git a/Trigger/TrigSteer/L1Decoder/python/L1DecoderConfig.py b/Trigger/TrigSteer/L1Decoder/python/L1DecoderConfig.py index 83b99608522..b980719eed1 100644 --- a/Trigger/TrigSteer/L1Decoder/python/L1DecoderConfig.py +++ b/Trigger/TrigSteer/L1Decoder/python/L1DecoderConfig.py @@ -26,8 +26,8 @@ def L1DecoderCfg(flags): if flags.get("Trigger.L1Decoder.doMuon") == True: from L1Decoder.L1MuonConfig import RPCCablingConfig, TGCCablingConfig - acc.addConfig( TGCCablingConfig, flags ) - acc.addConfig( RPCCablingConfig, flags ) + acc.merge(TGCCablingConfig(flags )) + acc.merge(RPCCablingConfig(flags )) decoderAlg.roiUnpackers += [ MURoIsUnpackingTool( Decisions = "MURoIDecisions", OutputTrigRoIs = "MURoIs", MonTool = RoIsUnpackingMonitoring( prefix="MU", maxCount=20 ) ) ] @@ -37,16 +37,16 @@ def L1DecoderCfg(flags): for u in decoderAlg.roiUnpackers: u.OutputLevel=DEBUG - acc.addEventAlgo(decoderAlg) - - from TrigConfigSvc.TrigConfigSvcConfig import TrigConfigSvcCfg - acc.addConfig( TrigConfigSvcCfg, flags ) + from TrigConfigSvc.TrigConfigSvcConfig import TrigConfigSvcCfg + acc.merge(TrigConfigSvcCfg(flags )) - return acc + return acc,decoderAlg if __name__ == "__main__": + from AthenaCommon.Configurable import Configurable + Configurable.configurableRun3Behavior=1 from AthenaConfiguration.AllConfigFlags import ConfigFlags ConfigFlags.set("Trigger.L1Decoder.doCalo",True) @@ -54,7 +54,8 @@ if __name__ == "__main__": ConfigFlags.set("Trigger.L1Decoder.forceEnableAllChains",True) ConfigFlags.set('global.InputFiles',["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1",]) ConfigFlags.lock() - l1 = L1DecoderCfg( ConfigFlags ) + acc, alg = L1DecoderCfg( ConfigFlags ) + acc.addEventAlgo(alg) f=open("L1DecoderConf.pkl","w") - l1.store(f) + acc.store(f) f.close() diff --git a/Trigger/TrigSteer/L1Decoder/python/L1MuonConfig.py b/Trigger/TrigSteer/L1Decoder/python/L1MuonConfig.py index df81b82e01a..724b73bb2da 100644 --- a/Trigger/TrigSteer/L1Decoder/python/L1MuonConfig.py +++ b/Trigger/TrigSteer/L1Decoder/python/L1MuonConfig.py @@ -25,14 +25,17 @@ def RPCCablingConfig(flags): # get the GeoModelSvc and add MuonDetectorTool from AtlasGeoModel.GeoModelConfig import GeoModelCfg - acc.addConfig( GeoModelCfg, flags ) + gmsAcc,gms=GeoModelCfg(flags ) + acc.merge(gmsAcc) + from MuonGeoModel.MuonGeoModelConf import MuonDetectorTool detTool = MuonDetectorTool() detTool.UseConditionDb = 1 detTool.UseIlinesFromGM = 1 - acc.getService('GeoModelSvc').DetectorTools += [ detTool ] + gms.DetectorTools += [ detTool ] + acc.addService(gms) from MuonRPC_Cabling.MuonRPC_CablingConf import MuonRPC_CablingSvc rpcCablingSvc = MuonRPC_CablingSvc() @@ -47,10 +50,10 @@ def RPCCablingConfig(flags): #conddb.addFolderSplitMC('RPC','/RPC/TRIGGER/CM_THR_PHI', '/RPC/TRIGGER/CM_THR_PHI') - acc.addConfig( addFolders, flags, + acc.merge(addFolders(flags, [ '/RPC/TRIGGER/CM_THR_ETA', '/RPC/TRIGGER/CM_THR_ETA', '/RPC/TRIGGER/CM_THR_PHI', '/RPC/TRIGGER/CM_THR_PHI', '/RPC/CABLING/MAP_SCHEMA', '/RPC/CABLING/MAP_SCHEMA_CORR' ], - 'RPC' ) + 'RPC' )) # that should not be here??? acc.getService('IOVDbSvc').FoldersToMetaData += ['/GLOBAL/BField/Maps'] @@ -83,12 +86,14 @@ def TGCCablingConfig(flags): acc.addService( TGCCablingSvc ) from IOVDbSvc.IOVDbSvcConfig import addFolders - acc.addConfig( addFolders, flags, ['/TGC/CABLING/MAP_SCHEMA','/TGC/CABLING/MAP_SCHEMA'], 'TGC') + acc.merge(addFolders(flags, ['/TGC/CABLING/MAP_SCHEMA','/TGC/CABLING/MAP_SCHEMA'], 'TGC')) return acc # for reco running we need MDT and CSC, yet to come if __name__ == '__main__': + from AthenaCommon.Configurable import Configurable + Configurable.configurableRun3Behavior=1 from AthenaConfiguration.AllConfigFlags import ConfigFlags ConfigFlags.set('Trigger.L1Decoder.doCalo',True) @@ -100,8 +105,8 @@ if __name__ == '__main__': acc = ComponentAccumulator() - acc.addConfig( RPCCablingConfig, ConfigFlags ) - acc.addConfig( TGCCablingConfig, ConfigFlags ) + acc.merge(RPCCablingConfig(ConfigFlags )) + acc.merge(TGCCablingConfig(ConfigFlags )) f=open('MuonConfig.pkl','w') acc.store(f) diff --git a/Trigger/TrigValidation/TrigUpgradeTest/python/EgammaCaloMod.py b/Trigger/TrigValidation/TrigUpgradeTest/python/EgammaCaloMod.py index e712198d8c1..d844880c0cc 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/python/EgammaCaloMod.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/python/EgammaCaloMod.py @@ -18,11 +18,11 @@ def CaloLUMIBCIDTool( flags, name='CaloLumiBCIDToolDefault' ): if flags.get('global.isOnline'): - acc.addConfig(addFolders, flags, ['/LAR/LArPileup/LArPileupShape<key>LArShape32</key>', - '/LAR/LArPileup/LArPileupAverage'], 'LAR_ONL') + acc.merge(addFolders(flags, ['/LAR/LArPileup/LArPileupShape<key>LArShape32</key>', + '/LAR/LArPileup/LArPileupAverage'], 'LAR_ONL')) else: - acc.addConfig( addFolders, flags, ['/LAR/ElecCalibOfl/LArPileupShape<key>LArShape32</key>', - '/LAR/ElecCalibOfl/LArPileupAverage'], 'LAR_OFL') + acc.merge(addFolders(flags, ['/LAR/ElecCalibOfl/LArPileupShape<key>LArShape32</key>', + '/LAR/ElecCalibOfl/LArPileupAverage'], 'LAR_OFL')) theTool = CaloLumiBCIDTool(name, isMC=False, LumiTool=theLumiTool, keyShape='LArShape32') @@ -35,8 +35,8 @@ def CaloLUMIBCIDTool( flags, name='CaloLumiBCIDToolDefault' ): from TrigBunchCrossingTool.BunchCrossingTool import BunchCrossingTool theBunchCrossingTool = BunchCrossingTool() - acc.addConfig( addFolders, flags, ['/LAR/ElecCalibMC/Shape', - '/LAR/ElecCalibMC/LArPileupAverage'], 'LAR_OFL') + acc.merge(addFolders(flags, ['/LAR/ElecCalibMC/Shape', + '/LAR/ElecCalibMC/LArPileupAverage'], 'LAR_OFL')) theTool = CaloLumiBCIDTool(name, isMC=True, LArOFCTool = theOFCTool, BunchCrossingTool = theBunchCrossingTool) @@ -59,7 +59,7 @@ def TileCond( flags ): from IOVDbSvc.IOVDbSvcConfig import addFolders def __addFolder(f): - acc.addConfig( addFolders, flags, '%s <key>%s</key>' %(f,f), 'TILE_OFL' if '/OFL' in f else 'TILE_ONL') + acc.merge(addFolders(flags, '%s <key>%s</key>' %(f,f), 'TILE_OFL' if '/OFL' in f else 'TILE_ONL')) tool.ProxyOflCes = TileCondProxyCoolFlt('TileCondProxyCool_OflCes', Source = '/TILE/OFL02/CALIB/CES') tool.ProxyOflCisLin = TileCondProxyCoolFlt('TileCondProxyCool_OflCisLin', Source = '/TILE/OFL02/CALIB/CIS/LIN') @@ -109,8 +109,8 @@ def TileCond( flags ): __addFolder( '/TILE/ONL01/STATUS/ADC' ) acc.addPublicTool( badChanTool ) - acc.addConfig( addFolders, flags, ['/LAR/BadChannelsOfl/BadChannels <key>/LAR/BadChannels/BadChannels</key>', - '/LAR/BadChannelsOfl/MissingFEBs<key>/LAR/BadChannels/MissingFEBs</key>'], 'LAR_OFL') + acc.merge( addFolders(flags, ['/LAR/BadChannelsOfl/BadChannels <key>/LAR/BadChannels/BadChannels</key>', + '/LAR/BadChannelsOfl/MissingFEBs<key>/LAR/BadChannels/MissingFEBs</key>'], 'LAR_OFL')) @@ -133,21 +133,21 @@ def TrigCaloDataAccessConfig(flags): from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator from IOVDbSvc.IOVDbSvcConfig import addFolders acc = ComponentAccumulator() - acc.addConfig(CaloLUMIBCIDTool, flags) + acc.merge(CaloLUMIBCIDTool(flags)) from TrigT2CaloCommon.TrigT2CaloCommonConf import TrigDataAccess da = TrigDataAccess() # ??? so we do not need the tools (quick hack) da.ApplyOffsetCorrection = False # ??? does not realy belong here - acc.addConfig( addFolders, flags, ['/LAR/Identifier/OnOffIdMap', '/LAR/Identifier/CalibIdMap', - '/LAR/Identifier/LArTTCellMapAtlas'], 'LAR') + acc.merge(addFolders(flags, ['/LAR/Identifier/OnOffIdMap', '/LAR/Identifier/CalibIdMap', + '/LAR/Identifier/LArTTCellMapAtlas'], 'LAR')) - acc.addConfig( addFolders, flags, ['/CALO/Identifier/CaloTTOnOffIdMapAtlas', '/CALO/Identifier/CaloTTOnAttrIdMapAtlas', - '/CALO/Identifier/CaloTTPpmRxIdMapAtlas'], 'CALO') + acc.merge(addFolders(flags, ['/CALO/Identifier/CaloTTOnOffIdMapAtlas', '/CALO/Identifier/CaloTTOnAttrIdMapAtlas', + '/CALO/Identifier/CaloTTPpmRxIdMapAtlas'], 'CALO')) # ??? should be moved to tile domain - acc.addConfig( TileCond, flags ) + acc.merge(TileCond(flags )) acc.addPublicTool( da ) @@ -165,19 +165,20 @@ def EgammaCaloMod( flags ): # load Calo geometry from LArGeoAlgsNV.LArGMConfig import LArGMCfg from TileGeoModel.TileGMConfig import TileGMCfg - acc.addConfig( LArGMCfg, flags ) - acc.addConfig( TileGMCfg, flags ) + acc.merge(LArGMCfg( flags )) + acc.merge(TileGMCfg( flags )) acc.getService('GeoModelSvc').DetectorTools['TileDetectorTool'].GeometryConfig = 'RECO' # ??? likely should be elsewhere from IOVDbSvc.IOVDbSvcConfig import addFolders - acc.addConfig( addFolders, flags, ['/LAR/Identifier/FebRodMap'], 'LAR' ) + acc.merge( addFolders(flags, ['/LAR/Identifier/FebRodMap'], 'LAR' )) - acc.addConfig( TrigCaloDataAccessConfig, flags ) + acc.merge(TrigCaloDataAccessConfig(flags )) # setup algorithms - acc.addSequence( seqAND('L2CaloEgamma') ) + #acc.addSequence( seqAND('L2CaloEgamma'), parentName=parentSeq ) + mainSeq=seqAND('L2CaloEgamma') from DecisionHandling.DecisionHandlingConf import RoRSeqFilter filterL1RoIsAlg = RoRSeqFilter('filterL1RoIsAlg') filterL1RoIsAlg.Input = ['EMRoIDecisions'] @@ -186,7 +187,8 @@ def EgammaCaloMod( flags ): filterL1RoIsAlg.Chains = [ m.split(':')[1].strip() for m in MenuTest.EMThresholdToChainMapping ] filterL1RoIsAlg.OutputLevel = DEBUG - acc.addEventAlgo( filterL1RoIsAlg, sequence='L2CaloEgamma' ) + #acc.addEventAlgo( filterL1RoIsAlg, sequenceName='L2CaloEgamma' ) + mainSeq+=filterL1RoIsAlg inViewAlgsSeqName = 'fastCaloInViewAlgs' from ViewAlgs.ViewAlgsConf import EventViewCreatorAlgorithm @@ -201,8 +203,8 @@ def EgammaCaloMod( flags ): fastCaloViewsMaker.ViewNodeName = inViewAlgsSeqName - acc.addEventAlgo( fastCaloViewsMaker, sequence='L2CaloEgamma' ) - + #acc.addEventAlgo( fastCaloViewsMaker, sequenceName='L2CaloEgamma' ) + mainSeq+=fastCaloViewsMaker from TrigT2CaloEgamma.TrigT2CaloEgammaConfig import RingerFexConfig ringer = RingerFexConfig('RingsMaker') ringer.RingsKey='CaloRings' @@ -228,8 +230,9 @@ def EgammaCaloMod( flags ): __fex_tools = [ samp2, samp1, sampe, samph, ring ] - acc.addSequence( seqAND( inViewAlgsSeqName ), sequence='L2CaloEgamma' ) - + #acc.addSequence( seqAND( inViewAlgsSeqName ), parentName='L2CaloEgamma' ) + inViewSeq=seqAND( inViewAlgsSeqName ) + mainSeq+=inViewSeq from TrigT2CaloEgamma.TrigT2CaloEgammaConf import T2CaloEgammaFastAlgo fastCalo = T2CaloEgammaFastAlgo( 'FastCaloAlgo' ) @@ -245,6 +248,8 @@ def EgammaCaloMod( flags ): accessTool = TrigDataAccess() accessTool.ApplyOffsetCorrection = False + acc.addPublicTool(accessTool) + fastCalo.IAlgToolList = __fex_tools from TrigT2CaloCalibration.EgammaCalibrationConfig import EgammaHitsCalibrationBarrelConfig, EgammaHitsCalibrationEndcapConfig, EgammaGapCalibrationConfig from TrigT2CaloCalibration.EgammaCalibrationConfig import EgammaTransitionRegionsConfig @@ -267,8 +272,8 @@ def EgammaCaloMod( flags ): #[ acc.addAlgTool( t ) for t in __endcapTools ] fastCalo.CalibListEndcap= __endcapTools - acc.addEventAlgo( fastCalo, sequence=inViewAlgsSeqName ) - + #acc.addEventAlgo( fastCalo, sequenceName=inViewAlgsSeqName ) + inViewSeq+=fastCalo from TrigEgammaHypo.TrigEgammaHypoConf import TrigL2CaloHypoAlgMT hypo = TrigL2CaloHypoAlgMT( 'L2CaloHypo' ) hypo.HypoInputDecisions = fastCaloViewsMaker.InputMakerOutputDecisions[0] @@ -289,15 +294,20 @@ def EgammaCaloMod( flags ): from TrigEgammaHypo.TrigL2CaloHypoTool import TrigL2CaloHypoToolFromName hypo.HypoTools = [ TrigL2CaloHypoToolFromName( c ) for c in chains ] - acc.addEventAlgo( hypo, sequence = 'L2CaloEgamma' ) - return acc,accessTool + #acc.addEventAlgo( hypo, sequenceName = 'L2CaloEgamma' ) + mainSeq+=hypo + + return acc,mainSeq if __name__ == '__main__': + from AthenaCommon.Configurable import Configurable + Configurable.configurableRun3Behavior=1 from AthenaConfiguration.AllConfigFlags import ConfigFlags + ConfigFlags.set( "global.InputFiles", + ["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1"] ) ConfigFlags.lock() - - acc = EgammaCaloMod( ConfigFlags ) + acc,seq = EgammaCaloMod( ConfigFlags ) acc.printConfig() print 'All ok' diff --git a/Trigger/TrigValidation/TrigUpgradeTest/share/newJOtest.py b/Trigger/TrigValidation/TrigUpgradeTest/share/newJOtest.py index b77e663a970..f18b3ebaf04 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/share/newJOtest.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/share/newJOtest.py @@ -7,6 +7,10 @@ from AthenaConfiguration.AllConfigFlags import ConfigFlags from AthenaCommon.CFElements import parOR, seqOR, seqAND, stepSeq from AthenaCommon.AlgSequence import dumpMasterSequence from AthenaCommon.AppMgr import theApp + +from AthenaCommon.Configurable import Configurable +Configurable.configurableRun3Behavior=1 + #theApp.setup() flags = ConfigFlags @@ -42,24 +46,25 @@ inputLoader = SGInputLoader(DetStore = 'StoreGateSvc/DetectorStore', Load = [], NeededResources = []) -acc.addEventAlgo( inputLoader, sequence='AthAlgSeq' ) +acc.addEventAlgo( inputLoader) from ByteStreamCnvSvc.ByteStreamConfig import TrigBSReadCfg -acc.addConfig( TrigBSReadCfg, flags ) +acc.merge(TrigBSReadCfg(flags )) -from AtlasGeoModel.GeoModelConfig import GeoModelCfg -acc.addConfig( GeoModelCfg, flags ) +#from AtlasGeoModel.GeoModelConfig import GeoModelCfg +#acc.merge(GeoModelCfg(flags )) from TrigUpgradeTest.TriggerHistSvcConfig import TriggerHistSvcConfig -acc.addConfig( TriggerHistSvcConfig, flags ) +acc.merge(TriggerHistSvcConfig(flags )) acc.addSequence( seqOR( "hltTop") ) from L1Decoder.L1DecoderConfig import L1DecoderCfg -acc.addConfig( L1DecoderCfg, flags, sequence="hltTop" ) -l1 = acc.getEventAlgo( "L1Decoder" ) +accL1,l1=L1DecoderCfg(flags)# sequenc="hltTop" ) from TrigUpgradeTest.TestUtils import applyMenu applyMenu( l1 ) +acc.merge(accL1) +acc.addEventAlgo(l1,"hltTop") from EventInfoMgt.EventInfoMgtConf import TagInfoMgr tagInfoMgr = TagInfoMgr() @@ -77,17 +82,20 @@ athenaPoolSvcSvc.PoolAttributes = ["DEFAULT_SPLITLEVEL ='0'", "STREAM_MEMBER_WIS acc.addService( athenaPoolSvcSvc ) acc.getService("EventPersistencySvc").CnvServices += [ athenaPoolSvcSvc.getName() ] -acc.addSequence( seqAND("hltSteps"), sequence="hltTop" ) -acc.addSequence( parOR("hltStep_1"), sequence="hltSteps" ) -acc.addSequence( parOR("hltStep_2"), sequence="hltSteps" ) +acc.addSequence( seqAND("hltSteps"), parentName="hltTop" ) +acc.addSequence( parOR("hltStep_1"), parentName="hltSteps" ) +acc.addSequence( parOR("hltStep_2"), parentName="hltSteps" ) # setup algorithm sequences here, need few additional components from TrigUpgradeTest.RegSelConfig import RegSelConfig -acc.addConfig( RegSelConfig, flags ) +acc.merge(RegSelConfig(flags )) from TrigUpgradeTest.EgammaCaloMod import EgammaCaloMod -acc.addConfig( EgammaCaloMod, flags, sequence="hltStep_1" ) +accECM,seqECM=EgammaCaloMod(flags) + +acc.merge(accECM) +acc.addSequence(seqECM, parentName="hltStep_1" ) # adding calo requires more infrastructure than we actually have #from TrigUpgradeTest.EgammaCaloMod import EgammaCaloMod -- GitLab