From a1277434aa477b473dfdcef48140fd97d707939f Mon Sep 17 00:00:00 2001 From: Francesca Pastore <francesca.pastore@cern.ch> Date: Thu, 9 Jan 2025 21:05:06 +0100 Subject: [PATCH 1/4] version passing ctests and 0mult chains, still with comments, to clean --- .../HLT/Config/ChainConfigurationBase.py | 2 +- .../python/HLT/Config/MenuComponents.py | 39 ++- .../python/HLT/Config/Utility/ChainMerging.py | 233 +++++++++++++----- .../HLT/Config/Utility/MenuAlignmentTools.py | 1 + .../python/HLT/Menu/Physics_pp_run3_v1.py | 18 ++ .../python/HLT/Test/TestChainConfiguration.py | 12 +- 6 files changed, 231 insertions(+), 74 deletions(-) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py index 91b83c717469..07cc9fe10573 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py @@ -47,7 +47,7 @@ class ChainConfigurationBase(metaclass=abc.ABCMeta): log.debug("Configuring step %s with %d chainParts", stepName, len(self.dict['chainParts'])) # do not generate Menu Sequences, just store the functions that can do that - seqArray = [functools.partial(gen, flags, **stepArgs) for gen in sequenceCfgArray] + seqArray = [functools.partial(gen, flags, **stepArgs) for gen in sequenceCfgArray] if (len(seqArray)>0): if inspect.signature(comboHypoCfg).parameters and all(inspect.signature(comboTool).parameters for comboTool in comboTools): diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py index d2f395ffda72..002b57b851d5 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py @@ -18,6 +18,7 @@ from collections.abc import MutableSequence import functools import inspect import re +import types from AthenaCommon.Logging import logging log = logging.getLogger( __name__ ) @@ -363,12 +364,24 @@ class EmptyMenuSequence: return "MenuSequence::%s \n Hypo::%s \n Maker::%s \n Sequence::%s \n HypoTool::%s\n"\ %(self.name, "Empty", self.maker.Alg.getName(), self.sequence.Alg.getName(), "None") +def createEmptyMenuSequenceCfg(flags, name): + """ creates generator functions named as the empty sequence""" + def create_sequence(name): + return EmptyMenuSequence(name) + # this allows to create the function with the same name of the sequence + create_sequence.__name__ = name + globals()[name] = create_sequence + return globals()[name] + + + def EmptyMenuSequenceCfg(flags, name): """Function to create a EmptyMenuSequence (used in the functools.partial)""" return EmptyMenuSequence(name) def isEmptySequenceCfg(o): - return o.func.__name__ == "EmptyMenuSequenceCfg" + return 'Empty' in o.func.__name__ +# return o.func.__name__ == "EmptyMenuSequenceCfg" class MenuSequence: """Class to group reco sequences with the Hypo. @@ -526,6 +539,17 @@ class Chain(object): elif re.search('^Step[0-9]{2}_', step_name): step_name = step_name[7:] step.name = 'Step%d_'%(stepID+1)+step_name + + # also modify the empty sequence names to follow the step name change + for iseq, seq in enumerate(step.sequenceGens): + if isEmptySequenceCfg(seq): + name = seq.func.__name__ + if re.search('Seq[0-9]_',name): + newname = re.sub('Seq[0-9]_', 'Seq%d_'%(stepID+1), name) + #replace the empty sequence + thisEmpty = createEmptyMenuSequenceCfg(None, newname) + step.sequenceGens[iseq]=functools.partial(thisEmpty, name=newname) + return @@ -569,7 +593,7 @@ class Chain(object): for stepID in range(1,n_new_steps+1): new_step_name = prev_step_name+'_'+empty_step_name+'%d_'%stepID+next_step_name - log.debug("Configuring empty step %s", new_step_name) + log.debug("Adding empty step %s", new_step_name) steps_to_add += [ChainStep(new_step_name, chainDicts=prev_chain_dict, comboHypoCfg=ComboHypoCfg, isEmpty=True)] self.steps = chain_steps_pre_split + steps_to_add + chain_steps_post_split @@ -603,6 +627,10 @@ class Chain(object): log.debug("Adding topo configurator %s for %s to %s", topoPair[0].__qualname__, topoPair[1], "step " + stepname) self.topoMap[step] = topoPair + def __str__(self): + return "\n-*- Chain %s -*- \n + Seeds: %s, Steps: %s, AlignmentGroups: %s "%(\ + self.name, ' '.join(map(str, self.L1decisions)), self.nSteps, self.alignmentGroups) + def __repr__(self): return "\n-*- Chain %s -*- \n + Seeds: %s, Steps: %s, AlignmentGroups: %s \n + Steps: \n %s \n"%(\ self.name, ' '.join(map(str, self.L1decisions)), self.nSteps, self.alignmentGroups, '\n '.join(map(str, self.steps))) @@ -754,7 +782,8 @@ class ChainStep(object): self.comboToolConfs.append(tool) def getComboHypoFncName(self): - return self.comboHypoCfg.func.__name__ if isinstance(self.comboHypoCfg, functools.partial) else self.comboHypoCfg + return self.comboHypoCfg.__name__ if isinstance(self.comboHypoCfg, types.FunctionType) else self.comboHypoCfg + def makeCombo(self): """ Configure the Combo Hypo Alg and generate the corresponding function, without instantiation which is done in createSequences() """ @@ -775,10 +804,6 @@ class ChainStep(object): self.combo = _ComboHypoPool[key] log.debug("Created combo %s with name %s, step comboName %s, key %s", funcName, self.combo.name, comboNameFromStep,key) - - - - def createComboHypoTools(self, flags, chainName): chainDict = HLTMenuConfig.getChainDictFromChainName(chainName) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py index 1b12e827dba6..0c17eaa3424f 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py @@ -1,24 +1,49 @@ # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration from TriggerMenuMT.HLT.Config.Utility.MenuAlignmentTools import get_alignment_group_ordering as getAlignmentGroupOrdering -from TriggerMenuMT.HLT.Config.MenuComponents import Chain, ChainStep, EmptyMenuSequenceCfg, isEmptySequenceCfg +from TriggerMenuMT.HLT.Config.MenuComponents import Chain, ChainStep, EmptyMenuSequenceCfg, isEmptySequenceCfg, createEmptyMenuSequenceCfg from AthenaCommon.Logging import logging from DecisionHandling.DecisionHandlingConfig import ComboHypoCfg from TrigCompositeUtils.TrigCompositeUtils import legName import functools +from itertools import repeat from copy import deepcopy import re log = logging.getLogger( __name__ ) + +def getMergedEmptyStepName(alignmentGroup, stepNumber, multiplicity, signature): + currentStepName = 'Empty' + alignmentGroup +'Align'+str(stepNumber)+'_'+ str(multiplicity) + signature + return currentStepName + +def getChainAlinmentOrdering(listOfChainDefs): + # not sure we need it + ordering = getAlignmentGroupOrdering() + merging_dict = {key: [] for key in ordering} + + for ich,cConfig in enumerate(listOfChainDefs): + chain_ag = cConfig.alignmentGroups[0] + if chain_ag not in merging_dict: + log.error("[getChainAlinmentOrdering] Alignment group %s can't be auto-merged because it's not in the grouping list!",chain_ag) + continue + merging_dict[chain_ag].append(ich) + + keys_to_remove = [key for key, value in merging_dict.items() if not value] + for key in keys_to_remove: + del merging_dict[key] + return merging_dict + + def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = None): - #chainDefList is a list of Chain() objects - #one for each part in the chain + """ function to merge leg chians, used also by signature code + chainDefList is a list of Chain() objects + one for each part in the chain """ strategy = chainDict["mergingStrategy"] offset = chainDict["mergingOffset"] - log.debug("[mergeChainDefs] %s: Combine by using %s merging", chainDict['chainName'], strategy) + log.debug("[mergeChainDefs] %s: Combine by using %s merging with %d chain defs (legs)", chainDict['chainName'], strategy, len(listOfChainDefs)) leg_numbering = [] @@ -31,9 +56,11 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non return mergeSerial(listOfChainDefs) elif strategy=="auto": - ordering = getAlignmentGroupOrdering() - merging_dict = {} - for ich,cConfig in enumerate(listOfChainDefs): + # old ###################################### + print(f"FRANCESCA CHAIN {listOfChainDefs[0].name}, legs to merge: {len(listOfChainDefs)}, signatures: {chainDict['signatures']}") + ordering = getAlignmentGroupOrdering() + merging_dict = {} # maps the chain ag with the alignemnt order dict + for ich,cConfig in enumerate(listOfChainDefs): chain_ag = cConfig.alignmentGroups[0] if chain_ag not in ordering: log.error("[mergeChainDefs] Alignment group %s can't be auto-merged because it's not in the grouping list!",chain_ag) @@ -41,7 +68,7 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non merging_dict[chain_ag] += [ich] else: merging_dict[chain_ag] = [ich] - + tmp_merged = [] tmp_merged_ordering = [] for ag in merging_dict: @@ -54,17 +81,67 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non log.debug("[mergeChainDefs] don't need to parallel merge") tmp_merged += [listOfChainDefs[merging_dict[ag][0]]] tmp_merged_ordering += [ordering.index(ag)] - + + print(f"FRANCESCA OLD tmp_merged_ordering={tmp_merged_ordering}") + print(f"FRANCESCA OLD tmp_merged multi={ [len(leg.steps[0].multiplicity) for leg in tmp_merged]}") + print(f"FRANCESCA tmp_merged ag={ [leg.alignmentGroups for leg in tmp_merged]}") #reset the ordering to index from zero (padding comes later!) merged_ordering = [-1]*len(tmp_merged_ordering) - copy_ordering = tmp_merged_ordering.copy() + copy_ordering = tmp_merged_ordering.copy() tmp_val = 0 while len(copy_ordering) > 0: min_index = tmp_merged_ordering.index(min(copy_ordering)) copy_ordering.pop(copy_ordering.index(min(copy_ordering))) merged_ordering[min_index] = tmp_val tmp_val += 1 + print(f"FRANCESCA OLD merged_ordering={merged_ordering}") # this is correct + + + + if 1: + # NEW ##################### +# THIS IS FRANCESCA CHANGE + merging_dict_new = getChainAlinmentOrdering(listOfChainDefs) + + tmp_merged_new = [] + tmp_merged_new_ordering = [] + log.debug(f"FRANCESCA NEW [mergeChainDefs] parallel merging with this dictionary: {merging_dict_new}") + index =0 + for ag in merging_dict_new: + print(f"FRANCESCA3 NEW : {merging_dict_new[ag]} for index {index}") + if len(merging_dict_new[ag]) > 1: + log.debug(f"[mergeChainDefs] parallel merging for alignment group {ag} and order {merging_dict_new[ag]}") + new_chain_defs, perSig_lengthOfChainConfigs = mergeParallel(list( listOfChainDefs[i] for i in merging_dict_new[ag] ), offset, leg_numbering, perSig_lengthOfChainConfigs) + tmp_merged_new += [new_chain_defs] + #tmp_merged_new_ordering += [ordering.index(ag)] + tmp_merged_new_ordering += [merging_dict_new[ag]] + else: + log.debug(f"[mergeChainDefs] don't need to parallel merge for alignemnt group {ag} and leg {merging_dict_new[ag]}") + tmp_merged_new += [listOfChainDefs[merging_dict_new[ag][0]]] + # tmp_merged_new += [listOfChainDefs[index]] # do not invert here, because the serial merging is doing it + # TODO remove the inversion in the serial merging, leave all order manipulation here + tmp_merged_new_ordering += [merging_dict_new[ag]] + # tmp_merged_new_ordering += [ordering.index(ag)] + index +=1 + print(f"FRANCESCA NEW tmp_merged_ordering={tmp_merged_new_ordering}") + print(f"FRANCESCA NEW tmp_merged multi={ [len(leg.steps[0].multiplicity) for leg in tmp_merged_new]}") + print(f"FRANCESCA NEW tmp_merged ag={ [leg.alignmentGroups for leg in tmp_merged_new]}") + #reset the ordering to index from zero (padding comes later!) + merged_ordering_new = [-1]*len(tmp_merged_new_ordering) + copy_ordering_new = tmp_merged_new_ordering.copy() + tmp_val_new = 0 + while len(copy_ordering_new) > 0: + min_index = tmp_merged_new_ordering.index(min(copy_ordering_new)) + copy_ordering_new.pop(copy_ordering_new.index(min(copy_ordering_new))) + merged_ordering_new[min_index] = tmp_val_new + tmp_val_new += 1 + print(f"FRANCESCA NEW merged_ordering={merged_ordering_new}") # this is correct + if merged_ordering_new != merged_ordering: + log.error("FRANCESCA ERRORRRR") + ### END OF NEW + + # only serial merge if necessary if len(tmp_merged) == 1: if perSig_lengthOfChainConfigs is None: @@ -114,9 +191,9 @@ def check_leg_lengths(perSig_lengthOfChainConfigs): return mismatched_ag, max_length - + def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfChainConfigs = None): - + ''' Performs merging of steps wihtin the same step number ''' # default mutable values must be initialized to None if leg_numbering is None: leg_numbering = [] @@ -124,6 +201,7 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha log.error("[mergeParallel] Offset for parallel merging not implemented.") raise Exception("[mergeParallel] Cannot merge this chain, exiting.") + log.debug(f"[mergeParallel] Parallel merging for chain alignments {[cConfig.alignmentGroups for cConfig in chainDefList]}") allSteps = [] allStepsMult = [] nSteps = [] @@ -173,7 +251,8 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha sigNames += [stepDict['chainParts'][0]['signature'] + is_fs_string] seqMultName = '_'.join([sigName for sigName in sigNames]) - seqStepName = 'Empty' + align_grp_to_lengthen + 'Align' + str(current_leg_ag_length+i) + '_' + seqMultName + + seqStepName = getMergedEmptyStepName(align_grp_to_lengthen, current_leg_ag_length+i, 1, seqMultName) seqNames = [getEmptySeqName(previous_step_dicts[iSeq]['signature'], current_leg_ag_length+i, align_grp_to_lengthen) for iSeq in range(len(sigNames))] emptySequences = build_empty_sequences(previous_step_dicts, step_mult, 'mergeParallel', cConfig.L1decisions, seqNames, chainName) @@ -188,7 +267,7 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha perSig_lengthOfChainConfigs[iConfig][0][index_modified_leg] = max_length else: log.debug("[mergeParallel] Alignment groups are empty for this combined chain") - + allSteps.append(cConfig.steps) allStepsMult.append(len(cConfig.steps[0].multiplicity)) nSteps.append(len(cConfig.steps)) @@ -196,15 +275,24 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha # Use zip_longest_parallel so that we get None in case one chain has more steps than the other orderedSteps = list(zip_longest_parallel(allSteps, allStepsMult)) - + if perSig_lengthOfChainConfigs is not None and len(perSig_lengthOfChainConfigs) > 0: in_chain_ag_lengths = {} ag_ordering = getAlignmentGroupOrdering() + for ag in ag_ordering: - for ag_lengths,sig_ags in perSig_lengthOfChainConfigs: + for ag_lengths, sig_ags in perSig_lengthOfChainConfigs: for ag_length, sig_ag in zip(ag_lengths, sig_ags): - if (sig_ag in in_chain_ag_lengths and in_chain_ag_lengths[sig_ag] < ag_length) or sig_ag not in in_chain_ag_lengths: - in_chain_ag_lengths[sig_ag] = ag_length + old = True + if old: + if (sig_ag in in_chain_ag_lengths and in_chain_ag_lengths[sig_ag] < ag_length) or sig_ag not in in_chain_ag_lengths: + in_chain_ag_lengths[sig_ag] = ag_length + else: + # THIS IS FRANCESCA CHANGE + if sig_ag == ag: + if (sig_ag in in_chain_ag_lengths and in_chain_ag_lengths[sig_ag] < ag_length) or sig_ag not in in_chain_ag_lengths: + in_chain_ag_lengths[ag] = ag_length + for ag, ag_length in in_chain_ag_lengths.items(): vertical_alignment_groups += [ag]*ag_length else: @@ -220,12 +308,31 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha for chain_index in range(len(chainDefList)): log.debug('[mergeParallel] Chain object to merge (i.e. chainDef) %s', chainDefList[chain_index]) - for step_index, (steps, step_ag) in enumerate(zip(orderedSteps,vertical_alignment_groups)): - mySteps = list(steps) - log.debug("[mergeParallel] Merging step counter %d", step_index+1) - - combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) - combChainSteps.append(combStep) + old =False + if old: + for step_index, (steps, step_ag) in enumerate(zip(orderedSteps,vertical_alignment_groups)): + mySteps = list(steps) + log.debug(f"[mergeParallel] Merging step {step_index+1} with ag ={step_ag}") + combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) + combChainSteps.append(combStep) + else: + #THIS IS FRANCESCA CHANGE + # find ag by steps for this alignemtn group, maintainign order + step_ags = [ag for ag in vertical_alignment_groups if ag in alignmentGroups] + step_ag = alignmentGroups[0] + unique_elements = list(set(step_ags)) + if len(unique_elements) >1: + log.error("mergeParallel must be done within the same alignment group") + for step_index, steps in enumerate(orderedSteps): + mySteps = list(steps) + print(f"FRANCESCA NEW steps {len(mySteps)} ag {step_ag}") + #if step_ag not in alignmentGroups: #skip steps due to serial alignement + # continue + log.debug(f"[mergeParallel] Merging step {step_index+1} with ag ={step_ag}") + combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) + combChainSteps.append(combStep) + + combinedChainDef = Chain(chainName, ChainSteps=combChainSteps, L1decisions=l1Decisions, nSteps = nSteps, alignmentGroups = alignmentGroups) @@ -304,7 +411,7 @@ def serial_zip(allSteps, chainName, chainDefList, legOrdering): if step_index == 0: prev_ag_step_index = step_index previousAG = getCurrentAG(step) - log.debug('[serial_zip] chain_index: %s step_index: %s, alignment group: %s', chain_index, step_index, previousAG) + log.debug('[serial_zip] chain_index: %s step_index: %s, alignment group: %s', chain_index, step_index+1, previousAG) # create list of correct length (chainSteps in parallel) stepList = [None]*n_parts @@ -354,13 +461,13 @@ def serial_zip(allSteps, chainName, chainDefList, legOrdering): previousAG = currentAG prev_ag_step_index = 1 - seqStepName = 'Empty' + currentAG +'Align'+str(ag_step_index)+'_'+seqMultName + seqStepName = getMergedEmptyStepName(currentAG, ag_step_index, 1, seqMultName) seqNames = [getEmptySeqName(emptyChainDicts[iSeq]['signature'], ag_step_index, currentAG) for iSeq in range(nLegs)] log.verbose("[serial_zip] step name for this leg: %s", seqStepName) log.verbose("[serial_zip] created empty sequence(s): %s", seqNames) - log.verbose("[serial_zip] L1decisions %s ", chainDefList[stepPlacement2].L1decisions) + log.verbose("[serial_zip] L1decisions %s ", chainDefList[stepPlacement2].L1decisions) emptySequences = build_empty_sequences(emptyChainDicts, step_mult, 'serial_zip', chainDefList[stepPlacement2].L1decisions, seqNames, chainName) @@ -435,16 +542,15 @@ def checkStepContent(parallel_steps): return False def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, currentChainSteps = None, leg_numbering = None, alignment_group = ""): - # default mutable values must be initialized to None if allSteps is None: allSteps = [] if currentChainSteps is None: currentChainSteps = [] if leg_numbering is None: leg_numbering =[] stepName = 'merged' #we will renumber all steps after chains are aligned #Step' + str(stepNumber) - stepSeq = [] - stepMult = [] - log.debug("[makeCombinedStep] stepNumber %d, steps %s ", stepNumber, parallel_steps) + + log.debug("[makeCombinedStep] stepNumber %d, alignment_group %s, %d steps: [%s], %d chain list: [%s], alignemnt groups: [%s]", stepNumber, alignment_group, len(parallel_steps), ', '.join([step.name if step is not None else "EMPTY" for step in parallel_steps ]), len(chainDefList), ', '.join([chain.name for chain in chainDefList]), ', '.join([chain.alignmentGroups[0] for chain in chainDefList])) + stepDicts = [] comboHypoTools = [] comboHypo = None @@ -469,7 +575,7 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, # every step is empty but some might have empty sequences and some might not if step is None or step.isEmpty: new_stepDicts = deepcopy(chainDefList[chain_index].steps[-1].stepDicts) - currentStepName = 'Empty' + chainDefList[chain_index].alignmentGroups[0]+'Align'+str(stepNumber)+'_'+new_stepDicts[0]['chainParts'][0]['multiplicity']+new_stepDicts[0]['signature'] + currentStepName = getMergedEmptyStepName(chainDefList[chain_index].alignmentGroups[0], stepNumber, 1, new_stepDicts[0]['signature']) log.debug('[makeCombinedStep] step has no sequences, making empty step %s', currentStepName) # we need a chain dict here, use the one corresponding to this leg of the chain @@ -510,14 +616,17 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, log.debug("[makeCombinedStep] Merged empty step: \n %s", theChainStep) return theChainStep - for chain_index, step in enumerate(parallel_steps): #this is a horizontal merge! -#TODO hasNonEmptyStep is already true here, + stepSeq = [] + chainIndex = [] + for num, chain in enumerate(chainDefList): + chainIndex.extend(list(repeat(num, len(chain.alignmentGroups)))) + + for index, step in enumerate(parallel_steps): #this is a horizontal merge! + chain_index = chainIndex[index] + if step is None or (hasNonEmptyStep and step.isEmpty): # this happens for merging chains with different numbers of steps, we need to "pad" out with empty sequences to propogate the decisions # all other chain parts' steps should contain an empty sequence - - if chain_index+1 > len(chainDefList): - chain_index-=chain_index if alignment_group == "": alignment_group = chainDefList[0].alignmentGroups[0] @@ -526,22 +635,19 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, seqName = getEmptySeqName(new_stepDict['signature'], stepNumber, alignment_group) if isFullScanRoI(chainDefList[chain_index].L1decisions[0]): - stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName+"FS")) - currentStepName = 'Empty' + alignment_group +'Align'+str(stepNumber)+'_'+new_stepDict['chainParts'][0]['multiplicity']+new_stepDict['signature']+'FS' + thisEmpty = createEmptyMenuSequenceCfg(None, seqName+"FS") + stepSeq.append(functools.partial(thisEmpty, name=seqName+"FS")) + #stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName+"FS")) + currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, new_stepDict['chainParts'][0]['multiplicity'], new_stepDict['signature']+'FS') + else: - stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName)) - currentStepName = 'Empty' + alignment_group +'Align'+str(stepNumber)+'_'+new_stepDict['chainParts'][0]['multiplicity']+new_stepDict['signature'] + thisEmpty = createEmptyMenuSequenceCfg(None, seqName) + stepSeq.append(functools.partial(thisEmpty, name=seqName)) + #stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName)) + currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, new_stepDict['chainParts'][0]['multiplicity'], new_stepDict['signature']) - log.debug("[makeCombinedStep] found empty step, step number %d chain_index: %s, step name: %s, made new empty sequence name: %s", stepNumber, chain_index, currentStepName, seqName) - - #stepNumber is indexed from 1, need the previous step indexed from 0, so do - 2 - prev_step_mult = -1 - if stepNumber > 1 and len(currentChainSteps[stepNumber-2].multiplicity) >0: - prev_step_mult = int(currentChainSteps[stepNumber-2].multiplicity[chain_index]) - else: - #get the step multiplicity from the step dict. This should be - prev_step_mult = int(new_stepDict['chainParts'][0]['multiplicity']) - stepMult.append(prev_step_mult) + log.debug("[makeCombinedStep] found empty step to be merged, step number: %d chain_index: %s, step name: %s, made new empty sequence name: %s", stepNumber, chain_index, currentStepName, seqName) + # we need a chain dict here, use the one corresponding to this leg of the chain oldLegName = new_stepDict['chainName'] if re.search('^leg[0-9]{3}_',oldLegName): @@ -551,7 +657,7 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, leg_counter += 1 else: # Standard step, append it to the combined step - log.debug("[makeCombinedStep] step %s, multiplicity = %s", step.name, str(step.multiplicity)) + log.debug("[makeCombinedStep] step %s, multiplicity = %s", step.name, str(step.multiplicity)) if len(step.sequenceGens): log.debug("[makeCombinedStep] with sequences = %s", ' '.join(map(str, [seq.func.__name__ for seq in step.sequenceGens]))) @@ -566,12 +672,7 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, #remove redundant instances of StepN_ and merged_ (happens when merging already merged chains) if currentStepName.startswith('merged_'): currentStepName = currentStepName[7:] - stepSeq.extend(step.sequenceGens) - # set the multiplicity of all the legs - if len(step.multiplicity) == 0: - stepMult.append(0) - else: - stepMult.extend(step.multiplicity) + stepSeq.extend(step.sequenceGens) comboHypoTools.extend(step.comboToolConfs) # update the chain dict list for the combined step with the chain dict from this step log.debug('[makeCombinedStep] adding step dictionaries %s',step.stepDicts) @@ -596,14 +697,13 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, comboHypoTools = list(set(comboHypoTools)) theChainStep = ChainStep(stepName, SequenceGens = stepSeq, chainDicts = stepDicts, comboHypoCfg = comboHypo, comboToolConfs = comboHypoTools) - log.debug("[makeCombinedStep] Merged step: \n %s", theChainStep) + log.debug("[makeCombinedStep] Merged step index %d: \n %s", stepNumber, theChainStep) return theChainStep # modified version of zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-, which takes into account the multiplicity of the steps def zip_longest_parallel(AllSteps, multiplicity, fillvalue=None): - from itertools import repeat iterators = [iter(it) for it in AllSteps] inactives =set() @@ -630,15 +730,20 @@ def zip_longest_parallel(AllSteps, multiplicity, fillvalue=None): yield tuple(values) + def build_empty_sequences(emptyChainDicts, step_mult, caller, L1decisions, seqNames, chainName): emptySequences = [] for ileg in range(len(L1decisions)): if isFullScanRoI(L1decisions[ileg]): - log.debug("[%s] adding FS empty sequence", caller) - emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg]+"FS")] + log.debug("[%s] adding FS empty sequenc with name %s", caller, seqNames[ileg]+"FS") + thisEmpty = createEmptyMenuSequenceCfg(None, seqNames[ileg]+"FS") + emptySequences += [functools.partial(thisEmpty, name=seqNames[ileg]+"FS")] + #emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg]+"FS")] else: - log.debug("[%s] adding non-FS empty sequence", caller) - emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg])] + log.debug("[%s] adding non-FS empty sequence with name %s", caller, seqNames[ileg]) + thisEmpty = createEmptyMenuSequenceCfg(None, seqNames[ileg]) + emptySequences += [functools.partial(thisEmpty, name=seqNames[ileg])] + #emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg])] log.verbose("[%s] emptyChainDicts %s", caller, emptyChainDicts) log.debug("[%s] %s has number of empty sequences %d and empty legs in stepDicts %d", diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/MenuAlignmentTools.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/MenuAlignmentTools.py index 56823eb166f5..45a7b2a38add 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/MenuAlignmentTools.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/MenuAlignmentTools.py @@ -35,6 +35,7 @@ the_signature_grouping = OrderedDict([ def get_alignment_group_ordering(): + # TODO use this: ordered_ag = list(dict.fromkeys(ordering.values())) seen = set() return [v for v in the_signature_grouping.values() if not (v in seen or seen.add(v))] diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py index 56c065a17e36..d187ec7d39ed 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py @@ -789,6 +789,14 @@ def setupMenu(): ChainProp(name='HLT_2g22_tight_ringer_L12eEM18M', groups=PrimaryPhIGroup+MultiPhotonGroup, monGroups=['egammaMon:shifter']), ChainProp(name='HLT_g35_medium_g25_medium_ringer_L12eEM24L', groups=PrimaryPhIGroup+SinglePhotonGroup, monGroups=['egammaMon:shifter']), ChainProp(name='HLT_2g50_loose_ringer_L12eEM24L', groups=PrimaryPhIGroup+MultiPhotonGroup, monGroups=['egammaMon:shifter']), + + + + # FRANCESCA + + #ChainProp(name='HLT_0e5_nopid_L1All', l1SeedThresholds=['eEM5'], groups=PrimaryPhIGroup+SingleElectronGroup), + #ChainProp(name='HLT_0g10_loose_L1All', l1SeedThresholds=['eEM5'], groups=PrimaryPhIGroup+SinglePhotonGroup), + #ChainProp(name='HLT_0e5_nopid_0g10_loose_L1All', l1SeedThresholds=['eEM5','eEM5'], groups=PrimaryPhIGroup+SinglePhotonGroup), ] chains['MET'] += [ @@ -1839,6 +1847,16 @@ def setupMenu(): chains['Combined'] += [ + # FRANCESCA + #ChainProp(name='HLT_0e5_nopid_0g5_loose_0tau5_ptonly_j10_xe5_pfopufit_L1All', l1SeedThresholds=['FSNOSEED','eEM5','eTAU12','FSNOSEED','FSNOSEED'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), + + #TEST ChainProp(name='HLT_0e5_nopid_0g5_loose_j10_xe5_pfopufit_0tau5_ptonly_L1All', l1SeedThresholds=['eEM5','eEM5','FSNOSEED','FSNOSEED','eTAU12'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), + + + #ChainProp(name='HLT_0e5_nopid_0g10_loose_0tau10_ptonly_mu5noL1_L1All', l1SeedThresholds=['FSNOSEED','eEM5','eEM5','eTAU12'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), + + #ChainProp(name='HLT_mu5noL1_0e5_nopid_0g10_loose_0tau10_ptonly_L1All', l1SeedThresholds=['FSNOSEED','eEM5','eEM5','eTAU12'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), + # AFP+dijet backup chains, discussed in ATR-24813 ChainProp(name='HLT_2j120_mb_afprec_afpdijet_L1AFP_A_AND_C', l1SeedThresholds=['FSNOSEED']*2, stream=[PhysicsStream], groups=MinBiasGroup+SupportGroup, monGroups=['mbMon:shifter']), ChainProp(name='HLT_2j175_mb_afprec_afpdijet_L1AFP_A_AND_C', l1SeedThresholds=['FSNOSEED']*2, stream=[PhysicsStream], groups=MinBiasGroup+SupportGroup, monGroups=['mbMon:t0']), diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py index 854acadd5b7e..30ec67c84bc9 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py @@ -4,12 +4,12 @@ from AthenaCommon.Logging import logging logging.getLogger().info("Importing %s",__name__) log = logging.getLogger(__name__) - +import functools from ..Config.ChainConfigurationBase import ChainConfigurationBase from TriggerMenuMT.CFtest.HLTSignatureConfig import muMenuSequence, elMenuSequence, gamMenuSequence from TriggerMenuMT.CFtest.HLTSignatureHypoTools import dimuDrComboHypoTool -from TriggerMenuMT.HLT.Config.MenuComponents import EmptyMenuSequenceCfg +from TriggerMenuMT.HLT.Config.MenuComponents import createEmptyMenuSequenceCfg, EmptyMenuSequenceCfg #-------------------------------------------------------- # fragments generating config will be functions in new JO @@ -154,8 +154,16 @@ class TestChainConfiguration(ChainConfigurationBase): def Step_empty2(self, flags): return self.getEmptyStep(2,'empty') + def Step_empty3_new(self, flags): + thisEmpty = createEmptyMenuSequenceCfg(None, name='EmptySequence') + print(f'FRAFRA {thisEmpty.__name__}') + emptySequence = functools.partial(thisEmpty, name='EmptySequence') + print(emptySequence) + return self.getStep(flags,'emptySeq', [emptySequence]) #, name="EmptySequence") + def Step_empty3(self, flags): return self.getStep(flags,'emptySeq', [EmptyMenuSequenceCfg], name="EmptySequence") + # Electrons -- GitLab From d47ab7b06ba45f3dc742d65d5d90b87779c67093 Mon Sep 17 00:00:00 2001 From: Francesca Pastore <francesca.pastore@cern.ch> Date: Mon, 13 Jan 2025 11:48:29 +0100 Subject: [PATCH 2/4] cleaned --- .../HLT/Config/ChainConfigurationBase.py | 2 +- .../python/HLT/Config/MenuComponents.py | 19 +-- .../python/HLT/Config/Utility/ChainMerging.py | 159 +++--------------- .../python/HLT/Menu/Physics_pp_run3_v1.py | 18 -- .../python/HLT/Test/TestChainConfiguration.py | 12 +- 5 files changed, 32 insertions(+), 178 deletions(-) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py index 07cc9fe10573..b57251c39032 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/ChainConfigurationBase.py @@ -47,7 +47,7 @@ class ChainConfigurationBase(metaclass=abc.ABCMeta): log.debug("Configuring step %s with %d chainParts", stepName, len(self.dict['chainParts'])) # do not generate Menu Sequences, just store the functions that can do that - seqArray = [functools.partial(gen, flags, **stepArgs) for gen in sequenceCfgArray] + seqArray = [functools.partial(gen, flags, **stepArgs) for gen in sequenceCfgArray] if (len(seqArray)>0): if inspect.signature(comboHypoCfg).parameters and all(inspect.signature(comboTool).parameters for comboTool in comboTools): diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py index 002b57b851d5..e41c5e173416 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py @@ -364,24 +364,12 @@ class EmptyMenuSequence: return "MenuSequence::%s \n Hypo::%s \n Maker::%s \n Sequence::%s \n HypoTool::%s\n"\ %(self.name, "Empty", self.maker.Alg.getName(), self.sequence.Alg.getName(), "None") -def createEmptyMenuSequenceCfg(flags, name): - """ creates generator functions named as the empty sequence""" - def create_sequence(name): - return EmptyMenuSequence(name) - # this allows to create the function with the same name of the sequence - create_sequence.__name__ = name - globals()[name] = create_sequence - return globals()[name] - - - def EmptyMenuSequenceCfg(flags, name): """Function to create a EmptyMenuSequence (used in the functools.partial)""" return EmptyMenuSequence(name) def isEmptySequenceCfg(o): - return 'Empty' in o.func.__name__ -# return o.func.__name__ == "EmptyMenuSequenceCfg" + return o.func.__name__ == "EmptyMenuSequenceCfg" class MenuSequence: """Class to group reco sequences with the Hypo. @@ -546,9 +534,8 @@ class Chain(object): name = seq.func.__name__ if re.search('Seq[0-9]_',name): newname = re.sub('Seq[0-9]_', 'Seq%d_'%(stepID+1), name) - #replace the empty sequence - thisEmpty = createEmptyMenuSequenceCfg(None, newname) - step.sequenceGens[iseq]=functools.partial(thisEmpty, name=newname) + #replace the empty sequence + step.sequenceGens[iseq]=functools.partial(EmptyMenuSequenceCfg, None, name=newname) return diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py index 0c17eaa3424f..cf491e769388 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py @@ -1,7 +1,7 @@ # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration from TriggerMenuMT.HLT.Config.Utility.MenuAlignmentTools import get_alignment_group_ordering as getAlignmentGroupOrdering -from TriggerMenuMT.HLT.Config.MenuComponents import Chain, ChainStep, EmptyMenuSequenceCfg, isEmptySequenceCfg, createEmptyMenuSequenceCfg +from TriggerMenuMT.HLT.Config.MenuComponents import Chain, ChainStep, EmptyMenuSequenceCfg, isEmptySequenceCfg from AthenaCommon.Logging import logging from DecisionHandling.DecisionHandlingConfig import ComboHypoCfg @@ -13,29 +13,6 @@ import re log = logging.getLogger( __name__ ) - -def getMergedEmptyStepName(alignmentGroup, stepNumber, multiplicity, signature): - currentStepName = 'Empty' + alignmentGroup +'Align'+str(stepNumber)+'_'+ str(multiplicity) + signature - return currentStepName - -def getChainAlinmentOrdering(listOfChainDefs): - # not sure we need it - ordering = getAlignmentGroupOrdering() - merging_dict = {key: [] for key in ordering} - - for ich,cConfig in enumerate(listOfChainDefs): - chain_ag = cConfig.alignmentGroups[0] - if chain_ag not in merging_dict: - log.error("[getChainAlinmentOrdering] Alignment group %s can't be auto-merged because it's not in the grouping list!",chain_ag) - continue - merging_dict[chain_ag].append(ich) - - keys_to_remove = [key for key, value in merging_dict.items() if not value] - for key in keys_to_remove: - del merging_dict[key] - return merging_dict - - def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = None): """ function to merge leg chians, used also by signature code chainDefList is a list of Chain() objects @@ -56,11 +33,9 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non return mergeSerial(listOfChainDefs) elif strategy=="auto": - # old ###################################### - print(f"FRANCESCA CHAIN {listOfChainDefs[0].name}, legs to merge: {len(listOfChainDefs)}, signatures: {chainDict['signatures']}") - ordering = getAlignmentGroupOrdering() + ordering = getAlignmentGroupOrdering() merging_dict = {} # maps the chain ag with the alignemnt order dict - for ich,cConfig in enumerate(listOfChainDefs): + for ich,cConfig in enumerate(listOfChainDefs): chain_ag = cConfig.alignmentGroups[0] if chain_ag not in ordering: log.error("[mergeChainDefs] Alignment group %s can't be auto-merged because it's not in the grouping list!",chain_ag) @@ -68,7 +43,7 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non merging_dict[chain_ag] += [ich] else: merging_dict[chain_ag] = [ich] - + tmp_merged = [] tmp_merged_ordering = [] for ag in merging_dict: @@ -81,66 +56,16 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non log.debug("[mergeChainDefs] don't need to parallel merge") tmp_merged += [listOfChainDefs[merging_dict[ag][0]]] tmp_merged_ordering += [ordering.index(ag)] - - print(f"FRANCESCA OLD tmp_merged_ordering={tmp_merged_ordering}") - print(f"FRANCESCA OLD tmp_merged multi={ [len(leg.steps[0].multiplicity) for leg in tmp_merged]}") - print(f"FRANCESCA tmp_merged ag={ [leg.alignmentGroups for leg in tmp_merged]}") + #reset the ordering to index from zero (padding comes later!) merged_ordering = [-1]*len(tmp_merged_ordering) - copy_ordering = tmp_merged_ordering.copy() + copy_ordering = tmp_merged_ordering.copy() tmp_val = 0 while len(copy_ordering) > 0: min_index = tmp_merged_ordering.index(min(copy_ordering)) copy_ordering.pop(copy_ordering.index(min(copy_ordering))) merged_ordering[min_index] = tmp_val tmp_val += 1 - print(f"FRANCESCA OLD merged_ordering={merged_ordering}") # this is correct - - - - if 1: - # NEW ##################### -# THIS IS FRANCESCA CHANGE - merging_dict_new = getChainAlinmentOrdering(listOfChainDefs) - - tmp_merged_new = [] - tmp_merged_new_ordering = [] - log.debug(f"FRANCESCA NEW [mergeChainDefs] parallel merging with this dictionary: {merging_dict_new}") - index =0 - for ag in merging_dict_new: - print(f"FRANCESCA3 NEW : {merging_dict_new[ag]} for index {index}") - if len(merging_dict_new[ag]) > 1: - log.debug(f"[mergeChainDefs] parallel merging for alignment group {ag} and order {merging_dict_new[ag]}") - new_chain_defs, perSig_lengthOfChainConfigs = mergeParallel(list( listOfChainDefs[i] for i in merging_dict_new[ag] ), offset, leg_numbering, perSig_lengthOfChainConfigs) - tmp_merged_new += [new_chain_defs] - #tmp_merged_new_ordering += [ordering.index(ag)] - tmp_merged_new_ordering += [merging_dict_new[ag]] - else: - log.debug(f"[mergeChainDefs] don't need to parallel merge for alignemnt group {ag} and leg {merging_dict_new[ag]}") - tmp_merged_new += [listOfChainDefs[merging_dict_new[ag][0]]] - # tmp_merged_new += [listOfChainDefs[index]] # do not invert here, because the serial merging is doing it - # TODO remove the inversion in the serial merging, leave all order manipulation here - tmp_merged_new_ordering += [merging_dict_new[ag]] - # tmp_merged_new_ordering += [ordering.index(ag)] - index +=1 - - print(f"FRANCESCA NEW tmp_merged_ordering={tmp_merged_new_ordering}") - print(f"FRANCESCA NEW tmp_merged multi={ [len(leg.steps[0].multiplicity) for leg in tmp_merged_new]}") - print(f"FRANCESCA NEW tmp_merged ag={ [leg.alignmentGroups for leg in tmp_merged_new]}") - #reset the ordering to index from zero (padding comes later!) - merged_ordering_new = [-1]*len(tmp_merged_new_ordering) - copy_ordering_new = tmp_merged_new_ordering.copy() - tmp_val_new = 0 - while len(copy_ordering_new) > 0: - min_index = tmp_merged_new_ordering.index(min(copy_ordering_new)) - copy_ordering_new.pop(copy_ordering_new.index(min(copy_ordering_new))) - merged_ordering_new[min_index] = tmp_val_new - tmp_val_new += 1 - print(f"FRANCESCA NEW merged_ordering={merged_ordering_new}") # this is correct - if merged_ordering_new != merged_ordering: - log.error("FRANCESCA ERRORRRR") - ### END OF NEW - # only serial merge if necessary if len(tmp_merged) == 1: @@ -251,7 +176,6 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha sigNames += [stepDict['chainParts'][0]['signature'] + is_fs_string] seqMultName = '_'.join([sigName for sigName in sigNames]) - seqStepName = getMergedEmptyStepName(align_grp_to_lengthen, current_leg_ag_length+i, 1, seqMultName) seqNames = [getEmptySeqName(previous_step_dicts[iSeq]['signature'], current_leg_ag_length+i, align_grp_to_lengthen) for iSeq in range(len(sigNames))] @@ -267,7 +191,7 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha perSig_lengthOfChainConfigs[iConfig][0][index_modified_leg] = max_length else: log.debug("[mergeParallel] Alignment groups are empty for this combined chain") - + allSteps.append(cConfig.steps) allStepsMult.append(len(cConfig.steps[0].multiplicity)) nSteps.append(len(cConfig.steps)) @@ -275,24 +199,16 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha # Use zip_longest_parallel so that we get None in case one chain has more steps than the other orderedSteps = list(zip_longest_parallel(allSteps, allStepsMult)) - + if perSig_lengthOfChainConfigs is not None and len(perSig_lengthOfChainConfigs) > 0: in_chain_ag_lengths = {} ag_ordering = getAlignmentGroupOrdering() for ag in ag_ordering: - for ag_lengths, sig_ags in perSig_lengthOfChainConfigs: + for ag_lengths, sig_ags in perSig_lengthOfChainConfigs: for ag_length, sig_ag in zip(ag_lengths, sig_ags): - old = True - if old: - if (sig_ag in in_chain_ag_lengths and in_chain_ag_lengths[sig_ag] < ag_length) or sig_ag not in in_chain_ag_lengths: - in_chain_ag_lengths[sig_ag] = ag_length - else: - # THIS IS FRANCESCA CHANGE - if sig_ag == ag: - if (sig_ag in in_chain_ag_lengths and in_chain_ag_lengths[sig_ag] < ag_length) or sig_ag not in in_chain_ag_lengths: - in_chain_ag_lengths[ag] = ag_length - + if (sig_ag in in_chain_ag_lengths and in_chain_ag_lengths[sig_ag] < ag_length) or sig_ag not in in_chain_ag_lengths: + in_chain_ag_lengths[sig_ag] = ag_length for ag, ag_length in in_chain_ag_lengths.items(): vertical_alignment_groups += [ag]*ag_length else: @@ -308,31 +224,13 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha for chain_index in range(len(chainDefList)): log.debug('[mergeParallel] Chain object to merge (i.e. chainDef) %s', chainDefList[chain_index]) - old =False - if old: - for step_index, (steps, step_ag) in enumerate(zip(orderedSteps,vertical_alignment_groups)): - mySteps = list(steps) - log.debug(f"[mergeParallel] Merging step {step_index+1} with ag ={step_ag}") - combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) - combChainSteps.append(combStep) - else: - #THIS IS FRANCESCA CHANGE - # find ag by steps for this alignemtn group, maintainign order - step_ags = [ag for ag in vertical_alignment_groups if ag in alignmentGroups] - step_ag = alignmentGroups[0] - unique_elements = list(set(step_ags)) - if len(unique_elements) >1: - log.error("mergeParallel must be done within the same alignment group") - for step_index, steps in enumerate(orderedSteps): - mySteps = list(steps) - print(f"FRANCESCA NEW steps {len(mySteps)} ag {step_ag}") - #if step_ag not in alignmentGroups: #skip steps due to serial alignement - # continue - log.debug(f"[mergeParallel] Merging step {step_index+1} with ag ={step_ag}") - combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) - combChainSteps.append(combStep) - - + for step_index, (steps, step_ag) in enumerate(zip(orderedSteps,vertical_alignment_groups)): + mySteps = list(steps) + log.debug(f"[mergeParallel] Merging step {step_index+1} with ag ={step_ag}") + + combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) + combChainSteps.append(combStep) + combinedChainDef = Chain(chainName, ChainSteps=combChainSteps, L1decisions=l1Decisions, nSteps = nSteps, alignmentGroups = alignmentGroups) @@ -343,6 +241,9 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha return combinedChainDef, perSig_lengthOfChainConfigs +def getMergedEmptyStepName(alignmentGroup, stepNumber, multiplicity, signature): + currentStepName = 'Empty' + alignmentGroup +'Align'+str(stepNumber)+'_'+ str(multiplicity) + signature + return currentStepName def getEmptySeqName(stepName, step_number, alignGroup): #remove redundant instances of StepN @@ -467,7 +368,7 @@ def serial_zip(allSteps, chainName, chainDefList, legOrdering): log.verbose("[serial_zip] step name for this leg: %s", seqStepName) log.verbose("[serial_zip] created empty sequence(s): %s", seqNames) - log.verbose("[serial_zip] L1decisions %s ", chainDefList[stepPlacement2].L1decisions) + log.verbose("[serial_zip] L1decisions %s ", chainDefList[stepPlacement2].L1decisions) emptySequences = build_empty_sequences(emptyChainDicts, step_mult, 'serial_zip', chainDefList[stepPlacement2].L1decisions, seqNames, chainName) @@ -635,15 +536,11 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, seqName = getEmptySeqName(new_stepDict['signature'], stepNumber, alignment_group) if isFullScanRoI(chainDefList[chain_index].L1decisions[0]): - thisEmpty = createEmptyMenuSequenceCfg(None, seqName+"FS") - stepSeq.append(functools.partial(thisEmpty, name=seqName+"FS")) - #stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName+"FS")) + stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName+"FS")) currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, new_stepDict['chainParts'][0]['multiplicity'], new_stepDict['signature']+'FS') else: - thisEmpty = createEmptyMenuSequenceCfg(None, seqName) - stepSeq.append(functools.partial(thisEmpty, name=seqName)) - #stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName)) + stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName)) currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, new_stepDict['chainParts'][0]['multiplicity'], new_stepDict['signature']) log.debug("[makeCombinedStep] found empty step to be merged, step number: %d chain_index: %s, step name: %s, made new empty sequence name: %s", stepNumber, chain_index, currentStepName, seqName) @@ -736,14 +633,10 @@ def build_empty_sequences(emptyChainDicts, step_mult, caller, L1decisions, seqNa for ileg in range(len(L1decisions)): if isFullScanRoI(L1decisions[ileg]): log.debug("[%s] adding FS empty sequenc with name %s", caller, seqNames[ileg]+"FS") - thisEmpty = createEmptyMenuSequenceCfg(None, seqNames[ileg]+"FS") - emptySequences += [functools.partial(thisEmpty, name=seqNames[ileg]+"FS")] - #emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg]+"FS")] + emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg]+"FS")] else: log.debug("[%s] adding non-FS empty sequence with name %s", caller, seqNames[ileg]) - thisEmpty = createEmptyMenuSequenceCfg(None, seqNames[ileg]) - emptySequences += [functools.partial(thisEmpty, name=seqNames[ileg])] - #emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg])] + emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg])] log.verbose("[%s] emptyChainDicts %s", caller, emptyChainDicts) log.debug("[%s] %s has number of empty sequences %d and empty legs in stepDicts %d", diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py index d187ec7d39ed..56c065a17e36 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Menu/Physics_pp_run3_v1.py @@ -789,14 +789,6 @@ def setupMenu(): ChainProp(name='HLT_2g22_tight_ringer_L12eEM18M', groups=PrimaryPhIGroup+MultiPhotonGroup, monGroups=['egammaMon:shifter']), ChainProp(name='HLT_g35_medium_g25_medium_ringer_L12eEM24L', groups=PrimaryPhIGroup+SinglePhotonGroup, monGroups=['egammaMon:shifter']), ChainProp(name='HLT_2g50_loose_ringer_L12eEM24L', groups=PrimaryPhIGroup+MultiPhotonGroup, monGroups=['egammaMon:shifter']), - - - - # FRANCESCA - - #ChainProp(name='HLT_0e5_nopid_L1All', l1SeedThresholds=['eEM5'], groups=PrimaryPhIGroup+SingleElectronGroup), - #ChainProp(name='HLT_0g10_loose_L1All', l1SeedThresholds=['eEM5'], groups=PrimaryPhIGroup+SinglePhotonGroup), - #ChainProp(name='HLT_0e5_nopid_0g10_loose_L1All', l1SeedThresholds=['eEM5','eEM5'], groups=PrimaryPhIGroup+SinglePhotonGroup), ] chains['MET'] += [ @@ -1847,16 +1839,6 @@ def setupMenu(): chains['Combined'] += [ - # FRANCESCA - #ChainProp(name='HLT_0e5_nopid_0g5_loose_0tau5_ptonly_j10_xe5_pfopufit_L1All', l1SeedThresholds=['FSNOSEED','eEM5','eTAU12','FSNOSEED','FSNOSEED'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), - - #TEST ChainProp(name='HLT_0e5_nopid_0g5_loose_j10_xe5_pfopufit_0tau5_ptonly_L1All', l1SeedThresholds=['eEM5','eEM5','FSNOSEED','FSNOSEED','eTAU12'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), - - - #ChainProp(name='HLT_0e5_nopid_0g10_loose_0tau10_ptonly_mu5noL1_L1All', l1SeedThresholds=['FSNOSEED','eEM5','eEM5','eTAU12'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), - - #ChainProp(name='HLT_mu5noL1_0e5_nopid_0g10_loose_0tau10_ptonly_L1All', l1SeedThresholds=['FSNOSEED','eEM5','eEM5','eTAU12'], groups=PrimaryPhIGroup+SingleElectronGroup+SinglePhotonGroup), - # AFP+dijet backup chains, discussed in ATR-24813 ChainProp(name='HLT_2j120_mb_afprec_afpdijet_L1AFP_A_AND_C', l1SeedThresholds=['FSNOSEED']*2, stream=[PhysicsStream], groups=MinBiasGroup+SupportGroup, monGroups=['mbMon:shifter']), ChainProp(name='HLT_2j175_mb_afprec_afpdijet_L1AFP_A_AND_C', l1SeedThresholds=['FSNOSEED']*2, stream=[PhysicsStream], groups=MinBiasGroup+SupportGroup, monGroups=['mbMon:t0']), diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py index 30ec67c84bc9..854acadd5b7e 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Test/TestChainConfiguration.py @@ -4,12 +4,12 @@ from AthenaCommon.Logging import logging logging.getLogger().info("Importing %s",__name__) log = logging.getLogger(__name__) -import functools + from ..Config.ChainConfigurationBase import ChainConfigurationBase from TriggerMenuMT.CFtest.HLTSignatureConfig import muMenuSequence, elMenuSequence, gamMenuSequence from TriggerMenuMT.CFtest.HLTSignatureHypoTools import dimuDrComboHypoTool -from TriggerMenuMT.HLT.Config.MenuComponents import createEmptyMenuSequenceCfg, EmptyMenuSequenceCfg +from TriggerMenuMT.HLT.Config.MenuComponents import EmptyMenuSequenceCfg #-------------------------------------------------------- # fragments generating config will be functions in new JO @@ -154,16 +154,8 @@ class TestChainConfiguration(ChainConfigurationBase): def Step_empty2(self, flags): return self.getEmptyStep(2,'empty') - def Step_empty3_new(self, flags): - thisEmpty = createEmptyMenuSequenceCfg(None, name='EmptySequence') - print(f'FRAFRA {thisEmpty.__name__}') - emptySequence = functools.partial(thisEmpty, name='EmptySequence') - print(emptySequence) - return self.getStep(flags,'emptySeq', [emptySequence]) #, name="EmptySequence") - def Step_empty3(self, flags): return self.getStep(flags,'emptySeq', [EmptyMenuSequenceCfg], name="EmptySequence") - # Electrons -- GitLab From eac73cb545ff4a4e3e8c37e5556957f959da6945 Mon Sep 17 00:00:00 2001 From: Francesca Pastore <francesca.pastore@cern.ch> Date: Mon, 13 Jan 2025 17:39:37 +0100 Subject: [PATCH 3/4] added TJ comments --- .../python/HLT/Config/Utility/ChainMerging.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py index cf491e769388..ab95a4d73784 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py @@ -14,9 +14,9 @@ import re log = logging.getLogger( __name__ ) def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = None): - """ function to merge leg chians, used also by signature code + """ function to merge chain definitions for all legs, used also by signature code chainDefList is a list of Chain() objects - one for each part in the chain """ + one for each part (leg) in the chain """ strategy = chainDict["mergingStrategy"] offset = chainDict["mergingOffset"] @@ -34,7 +34,7 @@ def mergeChainDefs(listOfChainDefs, chainDict, perSig_lengthOfChainConfigs = Non elif strategy=="auto": ordering = getAlignmentGroupOrdering() - merging_dict = {} # maps the chain ag with the alignemnt order dict + merging_dict = {} # maps the chain alignment group with the alignment order dict for ich,cConfig in enumerate(listOfChainDefs): chain_ag = cConfig.alignmentGroups[0] if chain_ag not in ordering: @@ -118,7 +118,7 @@ def check_leg_lengths(perSig_lengthOfChainConfigs): def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfChainConfigs = None): - ''' Performs merging of steps wihtin the same step number ''' + ''' Performs merging of steps with the same step number ''' # default mutable values must be initialized to None if leg_numbering is None: leg_numbering = [] @@ -226,7 +226,7 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha for step_index, (steps, step_ag) in enumerate(zip(orderedSteps,vertical_alignment_groups)): mySteps = list(steps) - log.debug(f"[mergeParallel] Merging step {step_index+1} with ag ={step_ag}") + log.debug(f"[mergeParallel] Merging step {step_index+1} with alignment group = {step_ag}") combStep = makeCombinedStep(mySteps, step_index+1, chainDefList, orderedSteps, combChainSteps, leg_numbering, step_ag) combChainSteps.append(combStep) @@ -362,7 +362,7 @@ def serial_zip(allSteps, chainName, chainDefList, legOrdering): previousAG = currentAG prev_ag_step_index = 1 - seqStepName = getMergedEmptyStepName(currentAG, ag_step_index, 1, seqMultName) + seqStepName = getMergedEmptyStepName(currentAG, ag_step_index, nLegs, seqMultName) seqNames = [getEmptySeqName(emptyChainDicts[iSeq]['signature'], ag_step_index, currentAG) for iSeq in range(nLegs)] @@ -449,8 +449,9 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, if leg_numbering is None: leg_numbering =[] stepName = 'merged' #we will renumber all steps after chains are aligned #Step' + str(stepNumber) + + log.debug("[makeCombinedStep] stepNumber %d, alignment_group %s, %d steps: [%s], %d chain list: [%s], alignment groups: [%s]", stepNumber, alignment_group, len(parallel_steps), ', '.join([step.name if step is not None else "EMPTY" for step in parallel_steps ]), len(chainDefList), ', '.join([chain.name for chain in chainDefList]), ', '.join([chain.alignmentGroups[0] for chain in chainDefList])) - log.debug("[makeCombinedStep] stepNumber %d, alignment_group %s, %d steps: [%s], %d chain list: [%s], alignemnt groups: [%s]", stepNumber, alignment_group, len(parallel_steps), ', '.join([step.name if step is not None else "EMPTY" for step in parallel_steps ]), len(chainDefList), ', '.join([chain.name for chain in chainDefList]), ', '.join([chain.alignmentGroups[0] for chain in chainDefList])) stepDicts = [] comboHypoTools = [] @@ -476,7 +477,7 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, # every step is empty but some might have empty sequences and some might not if step is None or step.isEmpty: new_stepDicts = deepcopy(chainDefList[chain_index].steps[-1].stepDicts) - currentStepName = getMergedEmptyStepName(chainDefList[chain_index].alignmentGroups[0], stepNumber, 1, new_stepDicts[0]['signature']) + currentStepName = getMergedEmptyStepName(chainDefList[chain_index].alignmentGroups[0], stepNumber, chainDefList[chain_index].steps[-1].multiplicity, new_stepDicts[0]['signature']) log.debug('[makeCombinedStep] step has no sequences, making empty step %s', currentStepName) # we need a chain dict here, use the one corresponding to this leg of the chain @@ -518,12 +519,13 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, return theChainStep stepSeq = [] - chainIndex = [] + chain_indices = [] + # create the list of leg indices that takes into account the multiplicity of the sub-legs + # (for example if a leg is already the result of a merging of other sub-legs, the leg index is repeated as many times as the number of sublegs. alignmentGroups are stored preserving this order, so it can be used here) for num, chain in enumerate(chainDefList): - chainIndex.extend(list(repeat(num, len(chain.alignmentGroups)))) + chain_indices.extend(list(repeat(num, len(chain.alignmentGroups)))) - for index, step in enumerate(parallel_steps): #this is a horizontal merge! - chain_index = chainIndex[index] + for chain_index, step in zip(chain_indices, parallel_steps): #this is a horizontal merge! if step is None or (hasNonEmptyStep and step.isEmpty): # this happens for merging chains with different numbers of steps, we need to "pad" out with empty sequences to propogate the decisions -- GitLab From d7f98afbad02e4c5fc333113003d27da3a0006dd Mon Sep 17 00:00:00 2001 From: Francesca Pastore <francesca.pastore@cern.ch> Date: Thu, 16 Jan 2025 11:56:21 +0100 Subject: [PATCH 4/4] fix empty sequence and emtpy step names to avoid duplications, runs also with AD chain --- .../python/HLT/Config/MenuComponents.py | 17 ++- .../python/HLT/Config/Utility/ChainMerging.py | 108 +++++++++--------- 2 files changed, 69 insertions(+), 56 deletions(-) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py index e41c5e173416..1a1057734aca 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/MenuComponents.py @@ -364,12 +364,22 @@ class EmptyMenuSequence: return "MenuSequence::%s \n Hypo::%s \n Maker::%s \n Sequence::%s \n HypoTool::%s\n"\ %(self.name, "Empty", self.maker.Alg.getName(), self.sequence.Alg.getName(), "None") +def createEmptyMenuSequenceCfg(flags, name): + """ creates the generator function named as the empty sequence""" + def create_sequence(name): + return EmptyMenuSequence(name) + # this allows to create the function with the same name as the sequence + #TODO need to extend it to also use it instead of EmptyMenuSequenceCfg inside custom steps + create_sequence.__name__ = name + globals()[name] = create_sequence + return globals()[name] + def EmptyMenuSequenceCfg(flags, name): """Function to create a EmptyMenuSequence (used in the functools.partial)""" return EmptyMenuSequence(name) def isEmptySequenceCfg(o): - return o.func.__name__ == "EmptyMenuSequenceCfg" + return 'Empty' in o.func.__name__ class MenuSequence: """Class to group reco sequences with the Hypo. @@ -527,7 +537,6 @@ class Chain(object): elif re.search('^Step[0-9]{2}_', step_name): step_name = step_name[7:] step.name = 'Step%d_'%(stepID+1)+step_name - # also modify the empty sequence names to follow the step name change for iseq, seq in enumerate(step.sequenceGens): if isEmptySequenceCfg(seq): @@ -535,8 +544,8 @@ class Chain(object): if re.search('Seq[0-9]_',name): newname = re.sub('Seq[0-9]_', 'Seq%d_'%(stepID+1), name) #replace the empty sequence - step.sequenceGens[iseq]=functools.partial(EmptyMenuSequenceCfg, None, name=newname) - + thisEmpty = createEmptyMenuSequenceCfg(None, newname) + step.sequenceGens[iseq]=functools.partial(thisEmpty, name=newname) return diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py index ab95a4d73784..ce134ef7c44c 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLT/Config/Utility/ChainMerging.py @@ -1,7 +1,7 @@ # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration from TriggerMenuMT.HLT.Config.Utility.MenuAlignmentTools import get_alignment_group_ordering as getAlignmentGroupOrdering -from TriggerMenuMT.HLT.Config.MenuComponents import Chain, ChainStep, EmptyMenuSequenceCfg, isEmptySequenceCfg +from TriggerMenuMT.HLT.Config.MenuComponents import Chain, ChainStep, EmptyMenuSequenceCfg, isEmptySequenceCfg, createEmptyMenuSequenceCfg from AthenaCommon.Logging import logging from DecisionHandling.DecisionHandlingConfig import ComboHypoCfg @@ -176,8 +176,9 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha sigNames += [stepDict['chainParts'][0]['signature'] + is_fs_string] seqMultName = '_'.join([sigName for sigName in sigNames]) - seqStepName = getMergedEmptyStepName(align_grp_to_lengthen, current_leg_ag_length+i, 1, seqMultName) - seqNames = [getEmptySeqName(previous_step_dicts[iSeq]['signature'], current_leg_ag_length+i, align_grp_to_lengthen) for iSeq in range(len(sigNames))] + nLegs = 1 # TODO, make it follow the real multiplicity of the step + seqStepName = getMergedEmptyStepName(align_grp_to_lengthen, current_leg_ag_length+i, nLegs, seqMultName) + seqNames = [getEmptySeqName(previous_step_dicts[iSeq]['signature'], current_leg_ag_length+i, align_grp_to_lengthen,i) for iSeq in range(len(sigNames))] emptySequences = build_empty_sequences(previous_step_dicts, step_mult, 'mergeParallel', cConfig.L1decisions, seqNames, chainName) # insert a step with an empty sequence @@ -193,7 +194,8 @@ def mergeParallel(chainDefList, offset, leg_numbering = None, perSig_lengthOfCha log.debug("[mergeParallel] Alignment groups are empty for this combined chain") allSteps.append(cConfig.steps) - allStepsMult.append(len(cConfig.steps[0].multiplicity)) + #TODO: instead of the real step multiplicy (len(cConfig.steps[0].multiplicity))), we set allStepsMult=[1] because the zip doesn't need it: when a step is missing in one leg, one None step is added, not multiple steps. I think we can remove the allStepsMult in the zip_longest below + allStepsMult.append(1) nSteps.append(len(cConfig.steps)) l1Decisions.extend(cConfig.L1decisions) @@ -245,17 +247,11 @@ def getMergedEmptyStepName(alignmentGroup, stepNumber, multiplicity, signature): currentStepName = 'Empty' + alignmentGroup +'Align'+str(stepNumber)+'_'+ str(multiplicity) + signature return currentStepName -def getEmptySeqName(stepName, step_number, alignGroup): - #remove redundant instances of StepN - if re.search('^Step[0-9]_',stepName): - stepName = stepName[6:] - elif re.search('^Step[0-9]{2}_', stepName): - stepName = stepName[7:] - seqName = 'Empty'+ alignGroup +'Seq'+str(step_number)+ '_'+ stepName +def getEmptySeqName(signature, step_number, alignGroup,order): + seqName = 'Empty'+ alignGroup +'Seq'+str(step_number)+ '_'+ str(order) + signature return seqName - def isFullScanRoI(inputL1Nav): fsRoIList = ['HLTNav_L1FSNOSEED','HLTNav_L1MET','HLTNav_L1J'] @@ -339,10 +335,9 @@ def serial_zip(allSteps, chainName, chainDefList, legOrdering): log.debug("[serial_zip] nLegs: %s, len(emptyChainDicts): %s, len(L1decisions): %s", nLegs, len(emptyChainDicts), len(chainDefList[stepPlacement2].L1decisions)) sigNames = [] for ileg,(emptyChainDict,_) in enumerate(zip(emptyChainDicts,chainDefList[stepPlacement2].L1decisions)): - if isFullScanRoI(chainDefList[stepPlacement2].L1decisions[ileg]): - sigNames +=[emptyChainDict['chainParts'][0]['signature']+'FS'] - else: - sigNames +=[emptyChainDict['chainParts'][0]['signature']] + is_fs_string = 'FS' if isFullScanRoI(chainDefList[stepPlacement2].L1decisions[ileg]) else '' + sigNames +=[emptyChainDict['chainParts'][0]['signature']+is_fs_string] + seqMultName = '_'.join([sigName for sigName in sigNames]) currentAG = '' @@ -364,7 +359,7 @@ def serial_zip(allSteps, chainName, chainDefList, legOrdering): seqStepName = getMergedEmptyStepName(currentAG, ag_step_index, nLegs, seqMultName) - seqNames = [getEmptySeqName(emptyChainDicts[iSeq]['signature'], ag_step_index, currentAG) for iSeq in range(nLegs)] + seqNames = [getEmptySeqName(emptyChainDicts[iSeq]['signature'], ag_step_index, currentAG,iSeq) for iSeq in range(nLegs)] log.verbose("[serial_zip] step name for this leg: %s", seqStepName) log.verbose("[serial_zip] created empty sequence(s): %s", seqNames) @@ -465,7 +460,7 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, log.debug("hasNonEmptyStep %d", hasNonEmptyStep) if not hasNonEmptyStep: - + # only empty steps here if len(parallel_steps)>=len(chainDefList) and all(step is None for step in parallel_steps[len(chainDefList):]): # We need to remove manually here the None steps exceeding the len of chainDefList. The right solution # would be to make sure that these cases don't happen upstream, but I am not confident enough with this @@ -477,7 +472,8 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, # every step is empty but some might have empty sequences and some might not if step is None or step.isEmpty: new_stepDicts = deepcopy(chainDefList[chain_index].steps[-1].stepDicts) - currentStepName = getMergedEmptyStepName(chainDefList[chain_index].alignmentGroups[0], stepNumber, chainDefList[chain_index].steps[-1].multiplicity, new_stepDicts[0]['signature']) + nLegs = len(chainDefList[chain_index].steps[-1].multiplicity) + currentStepName = getMergedEmptyStepName(chainDefList[chain_index].alignmentGroups[0], stepNumber, nLegs, new_stepDicts[0]['signature']) log.debug('[makeCombinedStep] step has no sequences, making empty step %s', currentStepName) # we need a chain dict here, use the one corresponding to this leg of the chain @@ -518,42 +514,50 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, log.debug("[makeCombinedStep] Merged empty step: \n %s", theChainStep) return theChainStep - stepSeq = [] - chain_indices = [] - # create the list of leg indices that takes into account the multiplicity of the sub-legs - # (for example if a leg is already the result of a merging of other sub-legs, the leg index is repeated as many times as the number of sublegs. alignmentGroups are stored preserving this order, so it can be used here) + stepSeq = [] + legsInStep = [] + # count the number of legs inside this chain part/step (inner legs) + # this happens if the step is already the result of a merging, due to the alignemnt, and can have more than one leg + # use the alignmentGroups here, which is stored by grouping the legs per alignemnt group + # TODO: can be extracted from stepDict['chainParts'][0]['multiplicity']? for num, chain in enumerate(chainDefList): - chain_indices.extend(list(repeat(num, len(chain.alignmentGroups)))) - - for chain_index, step in zip(chain_indices, parallel_steps): #this is a horizontal merge! - + legsInStep.append(len(chain.alignmentGroups)) + assert(len(legsInStep) == len(parallel_steps)) + + for chain_index, step in enumerate(parallel_steps): #this is a horizontal merge! + if step is None or (hasNonEmptyStep and step.isEmpty): # this happens for merging chains with different numbers of steps, we need to "pad" out with empty sequences to propogate the decisions # all other chain parts' steps should contain an empty sequence - + + log.debug("[makeCombinedStep] step %s is Empty and has %d legs", step.name if step is not None else "None", legsInStep[chain_index]) if alignment_group == "": alignment_group = chainDefList[0].alignmentGroups[0] - new_stepDict = deepcopy(chainDefList[chain_index].steps[-1].stepDicts[-1]) - seqName = getEmptySeqName(new_stepDict['signature'], stepNumber, alignment_group) + # loop over the inner legs of this sub-chain and create one empty sequence per each inner leg + for innerLeg in range(legsInStep[chain_index]): + new_stepDict = deepcopy(chainDefList[chain_index].steps[-1].stepDicts[-1]) + seqName = getEmptySeqName( new_stepDict['signature'], stepNumber, alignment_group, innerLeg) + log.debug("[makeCombinedStep] creating Empty sequence %s", seqName) + signature=new_stepDict['signature'] + is_fs_string = 'FS' if isFullScanRoI(chainDefList[chain_index].L1decisions[0]) else '' + seqName=seqName+is_fs_string + signature=new_stepDict['signature']+is_fs_string + thisEmpty = createEmptyMenuSequenceCfg(None, seqName) + stepSeq.append(functools.partial(thisEmpty, name=seqName)) + oldLegName = new_stepDict['chainName'] + if re.search('^leg[0-9]{3}_',oldLegName): + oldLegName = oldLegName[7:] + new_stepDict['chainName'] = legName(oldLegName,leg_counter) + stepDicts.append(new_stepDict) + leg_counter += 1 - if isFullScanRoI(chainDefList[chain_index].L1decisions[0]): - stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName+"FS")) - currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, new_stepDict['chainParts'][0]['multiplicity'], new_stepDict['signature']+'FS') - - else: - stepSeq.append(functools.partial(EmptyMenuSequenceCfg, None, name=seqName)) - currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, new_stepDict['chainParts'][0]['multiplicity'], new_stepDict['signature']) + nLegs = legsInStep[chain_index] + currentStepName = getMergedEmptyStepName(alignment_group, stepNumber, nLegs, signature) log.debug("[makeCombinedStep] found empty step to be merged, step number: %d chain_index: %s, step name: %s, made new empty sequence name: %s", stepNumber, chain_index, currentStepName, seqName) - # we need a chain dict here, use the one corresponding to this leg of the chain - oldLegName = new_stepDict['chainName'] - if re.search('^leg[0-9]{3}_',oldLegName): - oldLegName = oldLegName[7:] - new_stepDict['chainName'] = legName(oldLegName,leg_counter) - stepDicts.append(new_stepDict) - leg_counter += 1 + else: # Standard step, append it to the combined step log.debug("[makeCombinedStep] step %s, multiplicity = %s", step.name, str(step.multiplicity)) @@ -590,7 +594,7 @@ def makeCombinedStep(parallel_steps, stepNumber, chainDefList, allSteps = None, # the step naming for combined chains needs to be revisted!! stepName += '_' + currentStepName - log.debug('[makeCombinedStep] current step name %s',stepName) + log.debug('[makeCombinedStep] current step name %s, with %d sequences',stepName, len(stepSeq)) # for merged steps, we need to update the name to add the leg name comboHypoTools = list(set(comboHypoTools)) @@ -632,13 +636,13 @@ def zip_longest_parallel(AllSteps, multiplicity, fillvalue=None): def build_empty_sequences(emptyChainDicts, step_mult, caller, L1decisions, seqNames, chainName): emptySequences = [] - for ileg in range(len(L1decisions)): - if isFullScanRoI(L1decisions[ileg]): - log.debug("[%s] adding FS empty sequenc with name %s", caller, seqNames[ileg]+"FS") - emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg]+"FS")] - else: - log.debug("[%s] adding non-FS empty sequence with name %s", caller, seqNames[ileg]) - emptySequences += [functools.partial(EmptyMenuSequenceCfg, None, name=seqNames[ileg])] + for ileg in range(len(L1decisions)): + is_fs_string = 'FS' if isFullScanRoI(L1decisions[ileg]) else '' + sname = seqNames[ileg]+is_fs_string + log.debug("[%s] adding %s empty sequenc with name %s", caller, is_fs_string, sname) + thisEmpty = createEmptyMenuSequenceCfg(None, sname) + emptySequences += [functools.partial(thisEmpty, name=sname)] + log.verbose("[%s] emptyChainDicts %s", caller, emptyChainDicts) log.debug("[%s] %s has number of empty sequences %d and empty legs in stepDicts %d", -- GitLab