diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/ChainMerging.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/ChainMerging.py index 5a87049bb3ebb808b540ba3c4bb964e4bbbc1c8a..be17ad84a14a497e88861554c66f4ab1b08026b2 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/ChainMerging.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/ChainMerging.py @@ -5,7 +5,7 @@ log = logging.getLogger( __name__ ) from TriggerMenuMT.HLTMenuConfig.Menu.MenuComponents import Chain, ChainStep, EmptyMenuSequence, RecoFragmentsPool -from TriggerMenuMT.HLTMenuConfig.Menu.MenuAlignmentTools import getAlignmentGroupOrdering +from TriggerMenuMT.HLTMenuConfig.Menu.MenuAlignmentTools import get_alignment_group_ordering as getAlignmentGroupOrdering from collections import OrderedDict from copy import deepcopy import re diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/DictFromChainName.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/DictFromChainName.py index 63dac829e4b8f1368bf2a709e629a1460a5c9626..0b03fa560f1eed361c666d2825532ae775b5d55e 100755 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/DictFromChainName.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/DictFromChainName.py @@ -223,7 +223,7 @@ def analyseChainName(chainName, L1thresholds, L1item): from .SignatureDicts import getSignatureNameFromToken, AllowedCosmicChainIdentifiers, \ AllowedCalibChainIdentifiers, AllowedMonitorChainIdentifiers, AllowedBeamspotChainIdentifiers - from .MenuAlignmentTools import getAlignmentGroupFromPattern + from .MenuAlignmentTools import get_alignment_group_from_pattern as getAlignmentGroupFromPattern def buildDict(signature, sigToken ): groupdict = {'signature': signature, 'threshold': '', 'multiplicity': '', diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/GenerateMenuMT.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/GenerateMenuMT.py index b2b5bf0947e04e12ebcf4349a61232204913a72c..f60279990189dfce42b2952e8bdb11ceb9b74a9e 100755 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/GenerateMenuMT.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/GenerateMenuMT.py @@ -16,17 +16,13 @@ from TriggerMenuMT.HLTMenuConfig.Menu.DictFromChainName import dictFromChainName from TriggerMenuMT.HLTMenuConfig.Menu.ChainDictTools import splitInterSignatureChainDict from TriggerMenuMT.HLTMenuConfig.Menu.MenuPrescaleConfig import MenuPrescaleConfig, applyHLTPrescale from TriggerMenuMT.HLTMenuConfig.Menu.ChainMerging import mergeChainDefs -from TriggerMenuMT.HLTMenuConfig.Menu.MenuAlignmentTools import analyseCombinations +from TriggerMenuMT.HLTMenuConfig.Menu.MenuAlignmentTools import MenuAlignment from TriggerMenuMT.HLTMenuConfig.CommonSequences import EventBuildingSequenceSetup from AthenaCommon.Logging import logging log = logging.getLogger( __name__ ) _func_to_modify_signatures = None - - - - class Singleton(type): _instances = {} @@ -54,10 +50,17 @@ class GenerateMenuMT(object): def __init__(self): - self.chains = [] - self.chainDefs = [] + self.chainsInMenu = [] self.listOfErrorChainDefs = [] self.selectChainsForTesting = [] + + self.allChainsForAlignment = [] + self.chainDicts = [] + self.combinationsInMenu = [] + self.combinationsInMenu = [] + self.alignmentGroupsToAlign = set() + self.configLengthDict = {} + self.signaturesOverwritten = False self.L1Prescales = None self.HLTPrescales = None @@ -67,7 +70,7 @@ class GenerateMenuMT(object): self.allSignatures = ['Egamma', 'Muon', 'Jet', 'Bjet', 'Bphysics', 'MET', 'Tau', 'HeavyIon', 'Beamspot', 'Cosmic', 'EnhancedBias', 'Monitor', 'Calib', 'Streaming', 'Combined', 'MinBias', 'UnconventionalTracking', 'Test'] #, AFP - self.calibCosmicMonSigs = ['Streaming','Monitor','Beamspot','Cosmic', 'Calib', 'EnhancedBias'] #others not implemented yet ['Beamspot', 'Cosmic', 'Monitor', 'Calib', 'Streaming'] + self.calibCosmicMonSigs = ['Streaming','Monitor','Beamspot','Cosmic', 'Calib', 'EnhancedBias'] # flags self.doEgammaChains = True @@ -81,7 +84,7 @@ class GenerateMenuMT(object): self.doMinBiasChains = True self.doHeavyIonChains = True self.doCosmicChains = True - self.doCalibChains = True + self.doCalibChains = True self.doStreamingChains = True self.doMonitorChains = True self.doBeamspotChains = True @@ -154,24 +157,11 @@ class GenerateMenuMT(object): else: log.info("Doing nothing with L1 menu configuration...") - @memoize - def generateAllChainConfigs(self): - """ - == Obtains chain configs for all chains in menu - """ - - # get all chain names from menu - log.debug ("getting chains from Menu") - chainsInMenu = self.getChainsFromMenu() - # decoding of the chain name + def getChainDicts(self): chainCounter = 0 - - all_chains = [] - combinations_in_menu = [] - alignmentGroups_to_align = set() - length_of_configs = {} - for chain in chainsInMenu: + all_chain_dicts = [] + for chain in self.chainsInMenu: log.debug("Now processing chain: %s ", chain) chainDict = dictFromChainName(chain) @@ -180,8 +170,62 @@ class GenerateMenuMT(object): #set default chain prescale chainDict['prescale'] = 1 + all_chain_dicts += [chainDict] + self.chainDicts = all_chain_dicts + + return + + def importSignaturesToGenerate(self): + # check if all the signature files can be imported files can be imported + # and then import them! + log.debug("[getSignaturesInMenu] signaturesToGenerate: %s", self.signaturesToGenerate) + + for sig in self.signaturesToGenerate: + log.debug("[getSignaturesInMenu] sig: %s", sig) + + try: + if eval('self.do' + sig + 'Chains'): + if sig == 'Egamma': + sigFolder = sig + subSigs = ['Electron', 'Photon'] + elif sig in self.calibCosmicMonSigs: + sigFolder = 'CalibCosmicMon' + #only import the CalibCosmicMon signatures that we need, not all of them! + subSigs = [sig] + else: + sigFolder = sig + subSigs = [sig] + for ss in subSigs: + if sigFolder == 'Combined': + continue + else: + #import the includes into the global namespace. Only import the signature we need! + #this is equivalent having this line at the beginning of the file: + #import TriggerMenuMT.HLTMenuConfig.[sig].Generate[sig]ChainDefs as Generate[sig]ChainDefs + import_module = 'TriggerMenuMT.HLTMenuConfig.' + sigFolder +'.Generate' + ss + 'ChainDefs' + globals()['Generate'+ss+'ChainDefs'] = __import__(import_module,fromlist=['Generate'+ss+'ChainDefs']) + if ss not in self.availableSignatures: + self.availableSignatures.append(ss) + + except ImportError: + log.exception('Problems when importing ChainDef generating code for %s', sig) + import traceback + traceback.print_exc() + + log.debug('Available signature(s) for chain generation: %s', self.availableSignatures) + return + + def generateChains(self): + + all_chains = [] + combinations_in_menu = [] + alignmentGroups_to_align = set() + length_of_configs = {} + + + for chainDict in self.chainDicts: - log.debug("Next: getting chain configuration for chain %s ", chain.name) + log.debug("Next: getting chain configuration for chain %s ", chainDict['chainName']) chainConfig,lengthOfChainConfigs = self.__generateChainConfig(chainDict) all_chains += [(chainDict,chainConfig,lengthOfChainConfigs)] @@ -202,18 +246,52 @@ class GenerateMenuMT(object): combinations_in_menu += [list(set(chainDict['alignmentGroups']))] for align_group in list(set(chainDict['alignmentGroups'])): alignmentGroups_to_align.update([align_group]) + + self.allChainsForAlignment = all_chains + self.combinationsInMenu = combinations_in_menu + self.alignmentGroupsToAlign = alignmentGroups_to_align + self.configLengthDict = length_of_configs + + return + + @memoize + def generateAllChainConfigs(self): + """ + == Obtains chain configs for all chains in menu + """ + + # get all chain names from menu + log.debug ("getting chains from Menu") + self.getChainsFromMenu() + + # decoding of the chain name + self.getChainDicts() + + #import the necessary signatures + self.importSignaturesToGenerate() + self.generateChains() + # align event building sequences - EventBuildingSequenceSetup.alignEventBuildingSteps(all_chains) + EventBuildingSequenceSetup.alignEventBuildingSteps(self.allChainsForAlignment) #dict of signature: set it belongs to #e.g. {'Electron': ['Electron','Muon','Photon']} - alignmentGroup_sets_to_align = analyseCombinations(combinations_in_menu, alignmentGroups_to_align) - - log.debug('Aligning the following signatures with sets: %s',sorted(alignmentGroup_sets_to_align)) - log.debug('Length of each of the alignment groups: %s',length_of_configs) + menuAlignment = MenuAlignment(self.combinationsInMenu, + self.alignmentGroupsToAlign, + self.configLengthDict) + menuAlignment.analyse_combinations() + + # alignmentGroups_to_align = menuAlignment.groupsToAlign + # lengthOfChainConfigs = self.configLengthDict + # combinationsInMenu = menuAlignment.combinationsInMenu + # alignmentGroup_sets_to_align = menuAlignment.setsToAlign + + log.debug('Aligning the following signatures with sets: %s',sorted(menuAlignment.sets_to_align)) + log.debug('Length of each of the alignment groups: %s',self.configLengthDict) + - for chainDict,chainConfig,lengthOfChainConfigs in all_chains: + for chainDict,chainConfig,lengthOfChainConfigs in self.allChainsForAlignment: # start by ordering electron, photon, muon by having e+mu, g+mu, e+g chains # desired ordering: electron, photon, muon, tau, jet, met, b-jet @@ -228,122 +306,23 @@ class GenerateMenuMT(object): #parallel-merged single-signature chains or single signature chains. Anything that needs no splitting! if len(set(alignmentGroups)) == 1: - if alignmentGroups[0] not in alignmentGroup_sets_to_align or len(alignmentGroup_sets_to_align[alignmentGroups[0]]) == 1: - # no need to align lonely alignment groups - # the latter condition should never happen, because we only put in alignment groups that are - # in combined chains, and thus will need *some* aligning. but good to check in any case. - log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) - chainConfig.numberAllSteps() - TriggerConfigHLT.registerChain( chainDict, chainConfig ) - continue - elif alignmentGroups[0] == alignmentGroup_sets_to_align[alignmentGroups[0]][0]: - # if it's the first chain in the set to be aligned, again - nothing to do here. - log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) - chainConfig.numberAllSteps() - TriggerConfigHLT.registerChain( chainDict, chainConfig ) - continue - else: - # now we know that empty steps are necessary before this chain. we can loop through and add accordingly - # but we want to do this in reverse - the_align_sigs = alignmentGroup_sets_to_align[alignmentGroups[0]][:alignmentGroup_sets_to_align[alignmentGroups[0]].index(alignmentGroups[0])] - the_align_sigs.reverse() - - for align_sig in the_align_sigs: - chainConfig.insertEmptySteps(chainDict,'Empty'+align_sig+'Align',length_of_configs[align_sig],0) - - log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) - chainConfig.numberAllSteps() - TriggerConfigHLT.registerChain( chainDict, chainConfig ) - + alignedChainConfig = menuAlignment.single_align(chainDict, chainConfig) + TriggerConfigHLT.registerChain( chainDict, alignedChainConfig ) + elif len(alignmentGroups) == 2: - #check for a few bad conditions first: - if(alignmentGroups[0] not in alignmentGroup_sets_to_align or alignmentGroups[1] not in alignmentGroup_sets_to_align): - log.error(" one of the alignmentGroups in %s is not available in the sets to align dictionary!", alignmentGroups) - elif alignmentGroup_sets_to_align[alignmentGroups[0]] != alignmentGroup_sets_to_align[alignmentGroups[1]]: - log.error(" the two alignmentGroups %s point to different sets in the sets to align dictionary. Set1: %s, set2: %s!", - alignmentGroups, alignmentGroup_sets_to_align[alignmentGroups[0]],alignmentGroup_sets_to_align[alignmentGroups[1]]) - - if len(alignmentGroup_sets_to_align[alignmentGroups[0]]) == 2: - - # if the pair is on its own, then we just make sure the first signature's number - # of steps is equal to the max in that signature (so the next signature starts at the right step) - - # not a dictionary because we could have a chain mu_jet_munoL1? Not sure. But don't want to - # overwrite duplicates yet. - # probably, at some point, will need to divide this beyond signature but instead as unique sequence within a signature. - # munoL1 is already one case... - length_firstgrp = 0 - max_length_firstgrp = length_of_configs[alignmentGroup_sets_to_align[alignmentGroups[0]][0]] - for config_length,config_grp in lengthOfChainConfigs: - if config_grp == alignmentGroup_sets_to_align[alignmentGroups[0]][0]: - length_firstgrp = config_length - if length_firstgrp < max_length_firstgrp: - #too short! need to add padding steps between two alignment groups... - needed_steps = max_length_firstgrp - length_firstgrp - chainConfig.insertEmptySteps(chainDict,'Empty'+alignmentGroup_sets_to_align[alignmentGroups[0]][0]+'Align',needed_steps,length_firstgrp) - - elif length_firstgrp > max_length_firstgrp: - log.error("%s first signature length %d is greater than the max calculated, %d",chainDict.name,length_firstgrp, max_length_firstgrp) - - log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) - chainConfig.numberAllSteps() - TriggerConfigHLT.registerChain( chainDict, chainConfig ) - - - #this should probably work for signatures > 2, but might be a few gotchas (and errors need updating) - elif len(alignmentGroup_sets_to_align[alignmentGroups[0]]) > 2: - if(alignmentGroups[0] not in alignmentGroup_sets_to_align or alignmentGroups[1] not in alignmentGroup_sets_to_align): - log.error(" one of the alignmentGroups in %s is not available in the sets to align dictionary!", alignmentGroups) - elif alignmentGroup_sets_to_align[alignmentGroups[0]] != alignmentGroup_sets_to_align[alignmentGroups[1]]: - log.error(" the two alignmentGroups %s point to different sets in the sets to align dictionary. Set1: %s, set2: %s!", - alignmentGroups, alignmentGroup_sets_to_align[alignmentGroups[0]], alignmentGroup_sets_to_align[alignmentGroups[1]]) - - # we need to know which alignmentGroups are in the chain in which order. Assume this is always stored correctly. - # (this should be true) - - # never need to align the last chain - it can end a different length, no problem. - # ignore any signatures after the end of those in this chain - - alignGroups_set = alignmentGroup_sets_to_align[alignmentGroups[1]][:alignmentGroup_sets_to_align[alignmentGroups[1]].index(alignmentGroups[1])] - alignGroups_set.reverse() - grp_masks = [x in alignmentGroups for x in alignGroups_set] - grp_lengths = [] - for align_grp,grp_in_chain in zip(alignGroups_set,grp_masks): - if grp_in_chain: - for config_length,config_grp in lengthOfChainConfigs: - if config_grp == align_grp: - grp_lengths += [config_length] - else: - grp_lengths += [0] - - for istep,(align_grp,grp_in_chain,length_in_chain) in enumerate(zip(alignGroups_set,grp_masks,grp_lengths)): - # We're working our way backwards through the chain - # need to know how many steps are already before us! - nSteps_before_grp = 0 - if istep < len(grp_lengths)-1: - nSteps_before_grp = sum(grp_lengths[istep+1:]) - max_length_grp = length_of_configs[align_grp] - if grp_in_chain: - if length_in_chain < max_length_grp: - #too short! gotta add padding steps between two alignmentGroups... - needed_steps = max_length_grp - length_in_chain - start_step = nSteps_before_grp + length_in_chain - chainConfig.insertEmptySteps(chainDict,'Empty'+align_grp+'Align',needed_steps,start_step) - else: - # this sig isn't in the chain, but we still will need empty steps for it - # always add them to the start, because we're running in reverse order - chainConfig.insertEmptySteps(chainDict,'Empty'+align_grp+'Align',length_of_configs[align_grp],nSteps_before_grp) - - log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) - chainConfig.numberAllSteps() - TriggerConfigHLT.registerChain( chainDict, chainConfig ) - + alignedChainConfig = menuAlignment.multi_align(chainDict, chainConfig, lengthOfChainConfigs) + TriggerConfigHLT.registerChain( chainDict, alignedChainConfig ) + else: log.error("Menu can't deal with combined chains with more than two alignmentGroups at the moment. oops...") + raise NotImplementedError("more than three alignment groups still needs implementing in ChainMerging.py, ATR-22206") if not TriggerConfigHLT.isChainRegistered(chainDict['chainName']): log.error("Chain %s has not been registered in the menu!", chainDict['chainName']) - + import pprint + pp = pprint.PrettyPrinter(indent=4, depth=8) + log.error('The chain dictionary is: %s', pp.pformat(chainDict)) + raise Exception("Please fix the menu or the chain.") return TriggerConfigHLT.configsList() @@ -391,49 +370,13 @@ class GenerateMenuMT(object): else: log.debug("The following chains were found in the menu: \n %s", '\n '.join(map(str,(chains) ))) - return chains + self.chainsInMenu = chains def __generateChainConfig(self, mainChainDict): """ # Assembles the chain configuration and returns a chain object with (name, L1see and list of ChainSteps) """ - # check if all the signature files can be imported files can be imported - log.debug("[__generateChainConfig] signaturesToGenerate: %s", self.signaturesToGenerate) - - for sig in self.signaturesToGenerate: - log.debug("[__generateChainConfig] sig: %s", sig) - - try: - if eval('self.do' + sig + 'Chains'): - if sig == 'Egamma': - sigFolder = sig - subSigs = ['Electron', 'Photon'] - elif sig in self.calibCosmicMonSigs: - sigFolder = 'CalibCosmicMon' - subSigs = self.calibCosmicMonSigs - else: - sigFolder = sig - subSigs = [sig] - for ss in subSigs: - if sigFolder == 'Combined': - continue - else: - exec('import TriggerMenuMT.HLTMenuConfig.' + sigFolder + '.Generate' + ss + 'ChainDefs') - if ss not in self.availableSignatures: - self.availableSignatures.append(ss) - - except ImportError: - log.exception('Problems when importing ChainDef generating code for %s', sig) - import traceback - traceback.print_exc() - - log.debug('Available signature(s) for chain generation: %s', self.availableSignatures) - - import pprint - pp = pprint.PrettyPrinter(indent=4, depth=8) - - # split the the chainDictionaries for each chain and print them in a pretty way chainDicts = splitInterSignatureChainDict(mainChainDict) @@ -467,8 +410,8 @@ class GenerateMenuMT(object): if currentSig in self.availableSignatures and currentSig != 'Combined': try: - log.debug("Trying to get chain config for %s in folder %s", currentSig, sigFolder) - functionToCall ='TriggerMenuMT.HLTMenuConfig.' + sigFolder + '.Generate' + currentSig + 'ChainDefs.generateChainConfigs(chainPartDict)' + functionToCall ='Generate'+currentSig+'ChainDefs.generateChainConfigs(chainPartDict)' + log.debug("Trying to get chain config for %s in folder %s using %s", currentSig, sigFolder, functionToCall) chainPartConfig = eval(functionToCall) except RuntimeError: log.exception( 'Problems creating ChainDef for chain\n %s ', chainName) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/MenuAlignmentTools.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/MenuAlignmentTools.py index 4d2873baf4b75c886256b359a20475a4c145649e..42510b3ed349078447219644c1122fb18bcd7eab 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/MenuAlignmentTools.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/MenuAlignmentTools.py @@ -2,6 +2,7 @@ import numpy as np from collections import OrderedDict from AthenaCommon.Logging import logging + log = logging.getLogger( __name__ ) # Warning: If the grouping of signatures changes in *one* of these functions, @@ -24,50 +25,78 @@ the_signature_grouping = OrderedDict([ ('UnconventionalTracking','UnconventionalTracking'), ]) -def getAlignmentGroupOrdering(): +def get_alignment_group_ordering(): seen = set() - return [v for _,v in the_signature_grouping.items() if not (v in seen or seen.add(v))] + return [v for v in the_signature_grouping.values() if not (v in seen or seen.add(v))] -def getAlignmentGroupFromPattern(sName, extra): - signature_for_alignment = sName + extra +def get_alignment_group_from_pattern(signature, extra): + signature_for_alignment = signature + extra if signature_for_alignment in the_signature_grouping.keys(): return the_signature_grouping[signature_for_alignment] - elif sName in the_signature_grouping.keys(): - return the_signature_grouping[sName] + elif signature in the_signature_grouping.keys(): + return the_signature_grouping[signature] else: - log.info("No dedicated alignment grouping for signature %s (%s)",sName,extra) - return sName - -# Here, we use a list of all the signature combinations in the menu and the signature -# groupings/orderings we've defined in the_signature_grouping to calculate which signatures -# need to be ordered compared to which others. -# The function returns a dictionary of an ordered list of signatures each signature belongs to -# e.g. 'Muon' : ['Egamma','Muon'] -def analyseCombinations(combinations_in_menu, alignmentGroups_in_combinations): + log.info("No dedicated alignment grouping for signature %s (%s)",signature,extra) + return signature + +class MenuAlignment(): + """ Class to hold/calculate chain alignment """ + def __init__(self, combinations_in_menu, groups_to_align, length_of_configs): - # need to find out of if an alignment group, or anything in combination with that - # aligment group, is in combination with any other alignment group. + self.combinations_in_menu = combinations_in_menu + self.length_of_configs = length_of_configs + self.groups_to_align = groups_to_align + + self.signature_dict = {} + self.inverse_sig_dict ={} + self.sets_to_align = {} + # first we make a dictionary of the ordering, based on the_signature_grouping # e.g. 1 electron+photon, 2 muon, 3 tau, 4 jet/met/b-jet, 5 noL1 muons - sig_dict = {} igrp = 0 - for _,v in the_signature_grouping.items(): - if v not in sig_dict: - sig_dict[v] = igrp + for value in the_signature_grouping.values(): + if value not in self.signature_dict: + self.signature_dict[value] = igrp igrp += 1 - inv_sig_dict = {v: k for k, v in sig_dict.items()} + self.inverse_sig_dict = {v: k for k, v in self.signature_dict.items()} + + # Chains can be grouped together, e.g. e chains, mu chains, gamma chains. + # Chains are grouped if we want their steps to run in parallel, e.g. if they have shared + # sequences, like e+gamma chains use the same calo clustering step. + # Each group is called alignment group. These can also split signatures: muons have + # two alignment groups, Muon and MuonnoL1 (chainDict['signature']+chainDict['extra']) + + # With combined chains, e.g. e+mu, we can define in the_signature_grouping if we want + # the two legs to run in parallel (put them in the same alignment group, e.g. EgammaMuon), + # or in series (to save CPU - don't run slow muons until the electron decision has been made) + + # The alignment groups have a global ordering defined in the_signature_grouping + # But we only want to make all these triggers run in series if the combinations exist + # in the menu - i.e. do not put Tau after Egamma and Muon unless there are combined + # Tau chains that necessitate this ordering. + + # Here, we use a list of all the combined chain signature combinations in the menu and the signature + # groupings/orderings we've defined in the_signature_grouping to calculate which signatures + # need to be ordered compared to which others. + # The function returns a dictionary of an ordered list of signatures each signature belongs to + # e.g. 'Muon' : ['Egamma','Muon'] + + def analyse_combinations(self): #combinations_in_menu, alignmentGroups_in_combinations): + + # need to find out of if an alignment group, or anything in combination with that + # aligment group, is in combination with any other alignment group. - the_matrix = np.eye((len(sig_dict))) + the_matrix = np.eye((len(self.signature_dict))) - for comb in combinations_in_menu: + for comb in self.combinations_in_menu: if len(comb) > 2: log.error("Not setup for chains with more than two signatures yet!") else: - the_matrix[sig_dict[comb[0]]][sig_dict[comb[1]]] = 1 - the_matrix[sig_dict[comb[1]]][sig_dict[comb[0]]] = 1 + the_matrix[self.signature_dict[comb[0]]][self.signature_dict[comb[1]]] = 1 + the_matrix[self.signature_dict[comb[1]]][self.signature_dict[comb[0]]] = 1 _,eigenvecs = np.linalg.eig(the_matrix) # eigenvecs: The normalized (unit length) eigenvectors, such that the column v[:,i] @@ -100,30 +129,168 @@ def analyseCombinations(combinations_in_menu, alignmentGroups_in_combinations): unique_sets +=[aset] # convert the indices back to actual signature names - unique_by_sig = [[ inv_sig_dict[sig_int] for sig_int in setlist ] for setlist in unique_sets] + unique_by_sig = [[ self.inverse_sig_dict[sig_int] for sig_int in setlist ] for setlist in unique_sets] sig_to_set = {} - for sig in alignmentGroups_in_combinations: + for sig in self.groups_to_align: for aset in unique_by_sig: if sig in aset: sig_to_set[sig] = aset - return sig_to_set - -# takes the list of signatures in a chain and changes it to be the post-grouping signatures -# e.g. ['Electron','Muon'] ==> ['Egamma','Muon] -def setChainSignatures(signatures, lengthOfChainConfigs): - - for i, asig in enumerate(signatures): - if asig in the_signature_grouping: - signatures[i] = the_signature_grouping[asig] - - # this will be funny for e.g. Electron-Photon chains, or others grouped together - # but we won't use this for these chains so it's ok - for iCC,(config_length,asig) in enumerate(lengthOfChainConfigs): - if asig in the_signature_grouping: - lengthOfChainConfigs[iCC] = (config_length,the_signature_grouping[asig]) - # else it'll be a signature that doesn't need aligning, like beamspot or something. - # maybe they should all be added to the_signature_grouping, even if they never - # need to be merged together - return signatures, lengthOfChainConfigs + self.sets_to_align = sig_to_set + + return + + def alignment_group_is_alone(self, alignment_grp): + # no need to align lonely alignment groups + # the latter condition should never happen, because we only put in alignment groups that are + # in combined chains, and thus will need *some* aligning. but good to check in any case. + if alignment_grp not in self.sets_to_align: + return True + elif len(self.sets_to_align[alignment_grp]) == 1: + return True + return False + + def alignment_group_is_first(self, alignment_grp): + + # if it's the first chain in the set to be aligned, again - nothing to do here. + if alignment_grp == self.sets_to_align[alignment_grp][0]: + return True + return False + + def get_groups_to_align(self, alignment_grp): + + all_groups_to_align = self.sets_to_align[alignment_grp] + index_of_chain_group = self.sets_to_align[alignment_grp].index(alignment_grp) + + # we don't need to add empty steps for alignment groups that happen *after* the + # one our chain belongs to - so just cut off at the chain's alignment group here. + return all_groups_to_align[:index_of_chain_group] + + + def single_align(self, chainDict, chainConfig): + + if len(set(chainDict['alignmentGroups'])) != 1: + log.error("Cannot call single_align on chain {} with alignment groups {}".format( + chainDict['chainName'], ",".join(chainDict['alignmentGroups']))) + raise Exception("Will not proceed, the chain is not suitable for single alignment.") + + alignment_grp = chainDict['alignmentGroups'][0] + + if self.alignment_group_is_alone(alignment_grp): + log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) + chainConfig.numberAllSteps() + elif self.alignment_group_is_first(alignment_grp): + # if it's the first chain in the set to be aligned, again - nothing to do here, + # since this is single_align + log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) + chainConfig.numberAllSteps() + else: + # now we know that empty steps are necessary before this chain. we can loop through and add accordingly + aligngroups_set = self.get_groups_to_align(alignment_grp) + # but we want to do this in reverse + aligngroups_set.reverse() + + for align_grp_to_align in aligngroups_set: + chainConfig.insertEmptySteps(chainDict,'Empty'+align_grp_to_align+'Align',self.length_of_configs[align_grp_to_align],0) + log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) + chainConfig.numberAllSteps() + + return chainConfig + + def multi_align(self, chainDict, chainConfig, lengthOfChainConfigs): + + alignment_grps = chainDict['alignmentGroups'] + + #check for a few bad conditions first: + if not set(alignment_grps).issubset(self.sets_to_align): + log.error(" one of the alignmentGroups in %s is not available in the sets to align dictionary!", alignment_grps) + raise Exception("MenuAlignment.analyse_combinations() needs checking, this should never happen.") + elif len(set([tuple(self.sets_to_align[x]) for x in alignment_grps])) != 1: + log.error(" the alignmentGroups %s point to different sets in the sets to align dictionary", alignment_grps) + for x in alignment_grps: + log.error(" Set: %s, group %s", self.sets_to_align[x] , x) + raise Exception("MenuAlignment.analyse_combinations() needs checking, this should never happen.") + + #now we know that all alignmentGroups points to the same set, so just use the first entry + if len(self.sets_to_align[alignment_grps[0]]) == 2: + + # if the pair is on its own, then we just make sure the first signature's number + # of steps is equal to the max in that signature (so the next signature starts at the right step) + + # not a dictionary because we could have a chain mu_jet_munoL1? Not sure. But don't want to + # overwrite duplicates yet. + # probably, at some point, will need to divide this beyond signature but instead as unique sequence within a signature. + # munoL1 is already one case... + length_firstgrp = 0 + max_length_firstgrp = self.length_of_configs[self.sets_to_align[alignment_grps[0]][0]] + + for config_length,config_grp in lengthOfChainConfigs: + if config_grp == self.sets_to_align[alignment_grps[0]][0]: + length_firstgrp = config_length + + if length_firstgrp < max_length_firstgrp: + #too short! need to add padding steps between two alignment groups... + needed_steps = max_length_firstgrp - length_firstgrp + chainConfig.insertEmptySteps(chainDict,'Empty'+self.sets_to_align[alignment_grps[0]][0]+'Align',needed_steps,length_firstgrp) + + elif length_firstgrp > max_length_firstgrp: + log.error("%s first signature length %d is greater than the max calculated, %d",chainDict.name,length_firstgrp, max_length_firstgrp) + raise Exception("Probably something went wrong in GenerateMenuMT.generateChains()!") + + #this should probably work for signatures > 2, but might be a few gotchas (and errors need updating) + #Can't properly test until ATR-22206 is resolved. + elif len(self.sets_to_align[alignment_grps[0]]) > 2: + if not set(alignment_grps).issubset(self.sets_to_align): + log.error(" one of the alignmentGroups in %s is not available in the sets to align dictionary!", alignment_grps) + raise Exception("MenuAlignment.analyse_combinations() needs checking, this should never happen.") + elif len(set([tuple(self.sets_to_align[x]) for x in alignment_grps])) != 1: + log.error(" the alignmentGroups %s point to different sets in the sets to align dictionary", alignment_grps) + for x in alignment_grps: + log.error(" Set: %s, group %s", self.sets_to_align[x] , x) + raise Exception("MenuAlignment.analyse_combinations() needs checking, this should never happen.") + + # we need to know which alignment_grps are in the chain in which order. Assume this is always stored correctly. + # (this should be true) + # never need to align the last chain - it can end a different length, no problem. + # ignore any signatures after the end of those in this chain + aligngroups_set = self.get_groups_to_align(alignment_grps[-1]) + aligngroups_set.reverse() + grp_masks = [x in alignment_grps for x in aligngroups_set] + grp_lengths = [] + for align_grp,grp_in_chain in zip(aligngroups_set,grp_masks): + if grp_in_chain: + for config_length,config_grp in lengthOfChainConfigs: + if config_grp == align_grp: + grp_lengths += [config_length] + else: + grp_lengths += [0] + + for istep,(align_grp,grp_in_chain,length_in_chain) in enumerate(zip(aligngroups_set,grp_masks,grp_lengths)): + # We're working our way backwards through the chain + # need to know how many steps are already before us! + n_steps_before_grp = 0 + if istep < len(grp_lengths)-1: + n_steps_before_grp = sum(grp_lengths[istep+1:]) + max_length_grp = self.length_of_configs[align_grp] + if grp_in_chain: + if length_in_chain < max_length_grp: + #too short! gotta add padding steps between two alignmentGroups... + needed_steps = max_length_grp - length_in_chain + start_step = n_steps_before_grp + length_in_chain + chainConfig.insertEmptySteps(chainDict,'Empty'+align_grp+'Align',needed_steps,start_step) + else: + # this sig isn't in the chain, but we still will need empty steps for it + # always add them to the start, because we're running in reverse order + chainConfig.insertEmptySteps(chainDict,'Empty'+align_grp+'Align',self.length_of_configs[align_grp],n_steps_before_grp) + else: + log.error("Should never reach this point. alignmentGroups: %s, sets_to_align: %s",alignment_grps,self.sets_to_align) + raise Exception("MenuAlignment.multi_align() needs checking, this should never happen.") + + log.debug("Finished with retrieving chain configuration for chain %s", chainDict['chainName']) + chainConfig.numberAllSteps() + + return chainConfig + + + \ No newline at end of file diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/SignatureDicts.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/SignatureDicts.py index 404b0c8dfe05058fb9945fef69a05e76b4e1ecd8..4a521185d8d9d447e2a61268da621b6e9bc5623b 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/SignatureDicts.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/SignatureDicts.py @@ -31,7 +31,7 @@ SliceIDDict = { 'Test' : 'TestChain', } -AllowedSignatures = ["jet", "bjet", "electron", "photon", "egamma", +AllowedSignatures = ["jet", "bjet", "electron", "photon", "egamma", "muon", "met", "tau", @@ -57,6 +57,8 @@ ChainDictTemplate = { 'EBstep' : '', 'chainParts' : [], 'topoStartFrom' : False, + 'sigFolder' : '', + 'subSigs' : [] } #========================================================== @@ -73,6 +75,8 @@ TestChainParts = { 'trigType' : ['TestChain'], 'threshold' : '', 'addInfo' : [''], + 'sigFolder' : 'Test', + 'subSigs' : ['Test'] } # ---- Test Dictionary of default Values ---- @@ -84,6 +88,8 @@ TestChainParts_Default = { 'trigType' : '', 'threshold' : '', 'addInfo' : [], + 'sigFolder' : 'Test', + 'subSigs' : ['Test'] } #========================================================== @@ -135,6 +141,8 @@ JetChainParts = { 'aggSEP500htSEP30etSEP0eta320', 'aggSEP100htSEP10etSEP0eta320',], 'smc' : ['30smcINF', '35smcINF', '40smcINF', '50smcINF', '60smcINF', 'nosmc'], + 'sigFolder' : 'Jet', + 'subSigs' : ['Jet'] } # ---- Jet Dictionary of default Values ---- @@ -166,6 +174,8 @@ JetChainParts_Default = { 'trkopt' : 'notrk', 'hypoScenario' : 'simple', 'smc' : 'nosmc', + 'sigFolder' : 'Jet', + 'subSigs' : ['Jet'] } # ---- bJet Dictionary of default Values that are different to the ones for normal jet chains ---- @@ -195,6 +205,8 @@ MuonChainParts = { 'addInfo' : ['1step','idperf','3layersEC','cosmic',"muonqual"], 'topo' : AllowedTopos_mu, 'flavour' : [], + 'sigFolder' : 'Muon', + 'subSigs' : ['Muon'] } # ---- MuonDictionary of default Values ---- MuonChainParts_Default = { @@ -212,6 +224,8 @@ MuonChainParts_Default = { 'invMassInfo' : '', 'topo' : [], 'flavour' : '', + 'sigFolder' : 'Muon', + 'subSigs' : ['Muon'] } #========================================================== @@ -222,13 +236,18 @@ AllowedTopos_Bphysics = ['bJpsimumu','bUpsimumu','bBmumu','bDimu','bDimu2700','b # ---- Bphysics Dictionary of all allowed Values ---- BphysicsChainParts = deepcopy(MuonChainParts) BphysicsChainParts['signature'] = ['Bphysics'] +BphysicsChainParts['subFolder'] = 'Bphysics' +BphysicsChainParts['subSigs'] = ['Bphysics'] BphysicsChainParts['topo'] = AllowedTopos_Bphysics # ---- Bphysics Dictionary of default Values ---- BphysicsChainParts_Default = deepcopy(MuonChainParts_Default) BphysicsChainParts_Default['signature'] = ['Bphysics'] +BphysicsChainParts_Default['subFolder'] = 'Bphysics' +BphysicsChainParts_Default['subSigs'] = ['Bphysics'] BphysicsChainParts_Default['topo'] = [] + #========================================================== # Taus #========================================================== @@ -251,6 +270,8 @@ TauChainParts = { 'calib' : '', 'addInfo' : ['IdTest'], 'topo' : AllowedTopos_tau, + 'sigFolder' : 'Tau', + 'subSigs' : ['Tau'] } TauChainParts_Default = { 'signature' : ['Tau'], @@ -268,6 +289,8 @@ TauChainParts_Default = { 'calib' : '', 'addInfo' : '', 'topo' : [], + 'sigFolder' : 'Tau', + 'subSigs' : ['Tau'] } #========================================================== @@ -293,6 +316,8 @@ METChainParts = { 'L2muonCorr' : [], 'EFmuonCorr' : [], 'addInfo' : ['FStracks'], + 'sigFolder' : 'MET', + 'subSigs' : ['MET'] } # ---- MetDictionary of default Values ---- METChainParts_Default = { @@ -310,6 +335,8 @@ METChainParts_Default = { 'EFmuonCorr' : '', 'addInfo' : '', 'jetDataType' : 'tc', + 'sigFolder' : 'MET', + 'subSigs' : ['MET'] } #========================================================== @@ -360,7 +387,10 @@ ElectronChainParts = { 'lhInfo' : [], 'L2IDAlg' : ['noringer'], 'addInfo' : [ 'etcut', 'etcut1step',"v2","v3"], + 'sigFolder' : 'Egamma', + 'subSigs' : ['Electron','Photon'] } + # ---- Egamma Dictionary of default Values ---- ElectronChainParts_Default = { 'signature' : ['Electron'], @@ -383,6 +413,8 @@ ElectronChainParts_Default = { 'recoAlg' : '', 'FSinfo' : '', 'addInfo' : [], + 'sigFolder' : 'Egamma', + 'subSigs' : ['Electron','Photon'] } #========================================================== @@ -407,6 +439,9 @@ PhotonChainParts = { 'recoAlg' : [], 'FSinfo' : [], 'addInfo' : ['etcut',], + 'sigFolder' : 'Egamma', + 'subSigs' : ['Electron','Photon'] + } # ---- Photon Dictionary of default Values ---- @@ -427,6 +462,8 @@ PhotonChainParts_Default = { 'recoAlg' : '', 'FSinfo' : '', 'addInfo' : [], + 'sigFolder' : 'Egamma', + 'subSigs' : ['Electron','Photon'] } #========================================================== @@ -457,6 +494,8 @@ MinBiasChainParts = { 'hypoEFsumEtInfo': ['sumet40', 'sumet50', 'sumet60', 'sumet70', 'sumet80', 'sumet90', 'sumet110', 'sumet150',], 'recoAlg' : ['mbts', 'sptrk', 'sp', 'noalg', 'perf', 'hmt', 'hmtperf', 'idperf', 'zdcperf'], 'addInfo' : ['peb'], + 'sigFolder' : 'MinBias', + 'subSigs' : ['MinBias'] } # ---- MinBiasDictionary of default Values ---- MinBiasChainParts_Default = { @@ -477,6 +516,8 @@ MinBiasChainParts_Default = { 'hypoEFsumEtInfo': '', 'recoAlg' : [], 'addInfo' : [], + 'sigFolder' : 'MinBias', + 'subSigs' : ['MinBias'] } #========================================================== @@ -503,6 +544,8 @@ HeavyIonChainParts = { 'recoAlg' : [], 'addInfo' : [], 'gap' : [], + 'sigFolder' : 'HeavyIon', + 'subSigs' : ['HeavyIon'] } # ---- HeavyIonDictionary of default Values ---- @@ -525,7 +568,9 @@ HeavyIonChainParts_Default = { 'hypoEFsumEtInfo': '', 'recoAlg' : [], 'addInfo' : [], - 'gap' : '' + 'gap' : '', + 'sigFolder' : 'HeavyIon', + 'subSigs' : ['HeavyIon'] } #========================================================== @@ -549,6 +594,8 @@ CosmicChainParts = { 'multiplicity' : '', 'trigType' : 'cosmic', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Cosmic'] } # ---- Cosmic Chain Default Dictionary of all allowed Values ---- @@ -565,7 +612,8 @@ CosmicChainParts_Default = { 'multiplicity' : '', 'trigType' : '', 'extra' : '', - + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Cosmic'] } #========================================================== @@ -588,7 +636,9 @@ StreamingChainParts = { 'trigType' : 'streamer', 'extra' : '', 'streamType' : AllowedStreamingChainIdentifiers, - 'algo' : ['NoAlg'] + 'algo' : ['NoAlg'], + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Streaming'] } # ---- Cosmic Chain Default Dictionary of all allowed Values ---- @@ -604,6 +654,8 @@ StreamingChainParts_Default = { 'extra' : '', 'streamType' : '', 'algo' : [], + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Streaming'] } #========================================================== @@ -634,8 +686,11 @@ CalibChainParts = { 'multiplicity' : '', 'trigType' : ['trk'], 'extra' : ['rerun','bs',''], + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Calib'] } + # ---- Calib Chain Default Dictionary of all allowed Values ---- CalibChainParts_Default = { 'signature' : ['Calib'], @@ -652,6 +707,8 @@ CalibChainParts_Default = { 'location' : '', 'trigType' : '', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Calib'] } #========================================================== @@ -676,6 +733,8 @@ MonitorChainParts = { 'multiplicity' : '', 'trigType' : 'mon', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Monitor'] } # ---- Monitor Chain Default Dictionary of all allowed Values ---- @@ -690,7 +749,8 @@ MonitorChainParts_Default = { 'multiplicity' : '', 'trigType' : '', 'extra' : '', - + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Monitor'] } #========================================================== @@ -709,6 +769,8 @@ EnhancedBiasChainParts = { 'multiplicity' : '', 'trigType' : '', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['EnhancedBias'] } # ---- EnhancedBias Chain Default Dictionary of all allowed Values ---- @@ -722,6 +784,8 @@ EnhancedBiasChainParts_Default = { 'multiplicity' : '', 'trigType' : '', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['EnhancedBias'] } #========================================================== @@ -742,6 +806,8 @@ BeamspotChainParts = { 'multiplicity' : '', 'trigType' : 'beamspot', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Beamspot'] } # ---- Beamspot Chain Default Dictionary of all allowed Values ---- @@ -759,6 +825,8 @@ BeamspotChainParts_Default = { 'location' : 'vtx', 'trigType' : 'beamspot', 'extra' : '', + 'sigFolder' : 'CalibCosmicMon', + 'subSigs' : ['Beamspot'] } #========================================================== @@ -775,6 +843,8 @@ UnconventionalTrackingChainParts = { 'threshold' : '', 'extra' : '', 'addInfo' : [], + 'sigFolder' : 'UnconventionalTracking', + 'subSigs' : ['UnconventionalTracking'] } # ---- Unconventional Tracking Dictionary of default Values ---- UnconventionalTrackingChainParts_Default = { @@ -787,6 +857,8 @@ UnconventionalTrackingChainParts_Default = { 'threshold' : '', 'extra' : '', 'addInfo' : [], + 'sigFolder' : 'UnconventionalTracking', + 'subSigs' : ['UnconventionalTracking'] } #==========================================================