From d442983ee921f77bd19c33eb3160ed8da5e8d25e Mon Sep 17 00:00:00 2001 From: Francesca Pastore <francesca.pastore@cern.ch> Date: Wed, 18 Oct 2017 16:09:55 +0200 Subject: [PATCH] moved algorithm creation in HLTSignatureConfig; added EmuStepProcessingTest in ctest --- .../TrigUpgradeTest/CMakeLists.txt | 1 + .../TrigUpgradeTest/python/HLTCFConfig.py | 403 +++++++++++++----- .../TrigUpgradeTest/python/MenuComponents.py | 78 ++++ .../share/EmuStepProcessingTest.py | 155 ++----- .../TrigUpgradeTest/share/HLTCF.py | 286 ++----------- .../share/HLTSignatureConfig.py | 96 +++-- .../TrigUpgradeTest/share/L1CF.py | 1 + .../TrigUpgradeTest/src/TestInputMaker.cxx | 2 +- .../test/test_emu_step_processing.sh | 5 + 9 files changed, 534 insertions(+), 493 deletions(-) create mode 100644 Trigger/TrigValidation/TrigUpgradeTest/python/MenuComponents.py create mode 100755 Trigger/TrigValidation/TrigUpgradeTest/test/test_emu_step_processing.sh diff --git a/Trigger/TrigValidation/TrigUpgradeTest/CMakeLists.txt b/Trigger/TrigValidation/TrigUpgradeTest/CMakeLists.txt index f6359210802..e7970d7fbc6 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/CMakeLists.txt +++ b/Trigger/TrigValidation/TrigUpgradeTest/CMakeLists.txt @@ -74,6 +74,7 @@ atlas_add_test( idCaloRunData SCRIPT test/test_id_calo_run_data.sh ) atlas_add_test( EmuL1Decoding SCRIPT test/test_emu_l1_decoding.sh) +atlas_add_test( EmuStepProcessing SCRIPT test/test_emu_step_processing.sh) atlas_install_joboptions( share/*.py ) atlas_install_python_modules( python/*.py ) diff --git a/Trigger/TrigValidation/TrigUpgradeTest/python/HLTCFConfig.py b/Trigger/TrigValidation/TrigUpgradeTest/python/HLTCFConfig.py index 17955202797..97bb7d2383f 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/python/HLTCFConfig.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/python/HLTCFConfig.py @@ -1,127 +1,332 @@ -# Classes to configure the CF graph, via strings +# Classes to configure the CF graph, via Nodes +from AthenaCommon.CFElements import parOR, seqAND, stepSeq -class Alg: - def __init__(self, name): - self.name = name - self.inputs = [] - self.outputs = [] +allAlgs={} +def useExisting(name): + global allAlgs + return allAlgs[name] + +def AlgExisting(name): + global allAlgs + return name in allAlgs + + +def remember(name, instance): + global allAlgs + allAlgs[name] = instance + return instance - def __str__(self): - return "%s with inputs [%s] and outputs [%s]"%(self.name,', '.join(map(str, self.inputs)),', '.join(map(str, self.outputs))) - def addInput(self, Input): - self.inputs.append(Input) - self.inputs=list(set(self.inputs)) +class AlgNode(): + def __init__(self, Alg, inputProp, outputProp): + self.name = ("%s_%s_%s")%( Alg.name(), inputProp, outputProp) + self.algname = Alg.name() + self.outputProp=outputProp + self.inputProp=inputProp + self.configureAlg(Alg) + + def configureAlg(self, Alg): + alg_name = Alg.getName() + if AlgExisting(alg_name): + print "Found already Alg %s"%alg_name + self.Alg = useExisting(alg_name) + else: + print "Add new Alg %s"%alg_name + self.Alg=Alg + remember(alg_name, self.Alg) + + if self.Alg is None: + return - def addOutput(self, Output): - self.outputs.append(Output) + self.addDefaultOutput() + + def addDefaultOutput(self): + self.setOutput(("%s_out"%(self.name))) + + def setPar(self, prop, name): + cval = self.Alg.getProperties()[prop] + try: + if type(cval) == type(list()): + cval.append(name) + setattr(self.Alg, prop, cval) + else: + setattr(self.Alg, prop, name) + except: + pass + def getPar(self, prop): + try: + return getattr(self.Alg, prop) + except: + return self.Alg.getDefaultProperty(prop) + raise "Error in reading property " + prop + " from " + self.Alg -class RecoAlg(Alg): - def __init__(self, name, FileName="noreco.dat"): - Alg.__init__(self,name) - self.filename=FileName -class HypoAlg(Alg): - def __init__(self, name): - Alg.__init__(self, name) + def setOutput(self, name): + return self.setPar(self.outputProp,name) + + def getOutput(self): + return self.getPar(self.outputProp) + + def getOutputList(self): + outputs = [] + cval = self.getOutput() + if type(cval) == type(list()): + outputs.extend(cval) + else: + outputs.append(cval) + return outputs + + def setInput(self, name): + return self.setPar(self.inputProp,name) + + def getInput(self): + return self.getPar(self.inputProp) + + def getInputList(self): + inputs = [] + cval = self.getInput() + if type(cval) == type(list()): + inputs.extend(cval) + else: + inputs.append(cval) + return inputs + + def __str__(self): + return "Alg::%s [%s] -> [%s]"%(self.name,' '.join(map(str, self.getInputList())), ' '.join(map(str, self.getOutputList()))) + + + ## class TestRecoAlgNode(AlgNode): + ## def __init__(self, Alg, inputProp, outputProp): + ## ## self.outputProp='Output' + ## ## self.inputProp='Input' + ## self.name = "R_"+ name + ## AlgNode.__init__(self, name, Alg, inputProp, outputProp) + + +class HypoAlgNode(AlgNode): + def __init__(self, Alg, inputProp, outputProp): + AlgNode.__init__(self, Alg, inputProp, outputProp) self.tools = [] def addHypoTool(self, hypotool): - self.tools.append(hypotool) + if "Comb" in self.name: ###TMP combo, only one threshold + import re + thresholds = map(int, re.findall(r'\d+', hypotool)) + self.setPar('Threshold1', thresholds[0]) + self.setPar('Threshold2', thresholds[1]) + self.setPar('DecisionLabel', hypotool) + else: + self.tools.append(hypotool) + return self.setPar('HypoTools',hypotool) -class HypoTool: + +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestRoRSeqFilter +class SequenceFilterNode(AlgNode): def __init__(self, name): - self.name=name + Alg= HLTTest__TestRoRSeqFilter(name, OutputLevel = 2) + inputProp='Inputs' + outputProp='Outputs' + AlgNode.__init__(self, Alg, inputProp, outputProp) + if "Step1" in self.name: # so that we see events running through, will be gone once L1 emulation is included + self.Alg.AlwaysPass = True + + def addDefaultOutput(self): + return #do nothing + + def setChains(self, name): + return self.setPar("Chains", name) -class RecoSequence: - def __init__(self, name, Seed,Reuse=False,Algs=[]): - self.name = name - self.algs = Algs - self.seed=Seed - self.reuse=Reuse - def __str__(self): - return "RecoSequence %s with algs [%s] and seed [%s]"%(self.name,', '.join(map(str, self.algs)), self.seed) + def getChains(self): + return self.getPar("Chains") - -class Sequence: - def __init__(self, name, Hypo, RecoSeq=[]): - self.name = name - self.recoSeq=RecoSeq - self.algs = [] - self.algs.extend(seq.algs for seq in RecoSeq) - self.hypo = Hypo - -class CFRecoSequence: - def __init__(self, name, RecoSequence, InputMaker): - self.name = name - self.recoSequence=RecoSequence - self.inputMaker=InputMaker - self.outputs=self.recoSequence.algs[-1].outputs # last alg output - def __str__(self): - return "CFRecoSequence %s is %d, with RecoSequence [%s] and inputMaker [%s]"%(self.name,self.recoSequence.reuse,self.recoSequence, self.inputMaker) -class SequenceThreshold: - def __init__(self, Sequence, Threshold): - self.sequence = Sequence - self.threshold = Threshold -class ChainStep: - def __init__(self, name, SequenceThresholds=[]): - self.name = name - self.sequenceThresholds = SequenceThresholds +#### Here functions to create the CF tree from CF configuration objects +#from TrigUpgradeTest.HLTCFConfig import * +#include("TrigUpgradeTest/HLTSignatureConfig.py") +import re +def create_CFSequence(CFseq): + print "\n * Create CFSequence %s"%(CFseq.name) + filterAlg=CFseq.filter.Alg + + nodes = CFseq.getAllNodes() + algos = [] + for node in nodes: + algos.append(node.Alg) -class Step: - def __init__(self, name, FilterAlg, Hypo, CFRecoSequences=[]): - self.name = name - self.filter = FilterAlg - self.cfRecoSequences = CFRecoSequences - self.hypo=Hypo - def __str__(self): - return "--- STEP %s ---\n + Filter: %s \n + CFRecoSequence: %s \n + Hypo: %s \n"%(self.name,\ - self.filter, \ - '\n '.join(map(str, self.cfRecoSequences)),\ - self.hypo) - - def getAllAlgs(self): - algs = [] - algs.append(self.filter) - algs.extend(seq.recoSequence.algs for seq in self.cfRecoSequences) - algs.append(self.hypo) - return algs + algos=list(set(algos)) + + step_seq = stepSeq( CFseq.name, filterAlg=filterAlg, rest=algos) + return step_seq + + + +def find_stepCF_algs(step_list): + algos = [] + for step in step_list: + #print "Finding algos of step %s"%step + step_algs = [] + step_algs.append(step.filter) + for seq in step.menuSeq.recoSeqList: + step_algs.extend(seq.algs ) + step_algs.append(seq.hypo) + #print step_algs + algos.extend(step_algs) + return algos + + +from AthenaCommon.AlgSequence import dumpSequence +def create_StepCF(name, seq_list, dump=0): + print "There are %d steps in this CFStep %s"%(len(seq_list), name) + stepCF = parOR(name) + for seq in seq_list: + step_seq = create_CFSequence(seq) + print "Add this step %s to %s"%(seq.name,name) + stepCF += step_seq + + if dump: dumpSequence (stepCF, indent=0) + return stepCF -class Chain: - def __init__(self, name, Seed, ChainSteps=[]): - self.name = name - self.seed=Seed - self.steps=ChainSteps - - seeds = Seed.strip().split("_") - seeds.pop(0) #remove first L1 - print seeds - self.group_seed = ["L1"+filter(lambda x: x.isalpha(), stri) for stri in seeds] - - -class SequenceFilter(Alg): - def __init__(self, name): - Alg.__init__(self,name) - self.chains=[] - - def addChain(self, Chain): - self.chains.append(Chain) - def __str__(self): - return "%s with inputs [%s] and outputs [%s], chains = [%s]"%(self.name,', '.join(map(str, self.inputs)),', '.join(map(str, self.outputs)),', '.join(map(str, self.chains))) +###### Here some graphical methods -class InputMaker(Alg): - def __init__(self, name): - Alg.__init__(self, name) - +from ViewAlgsTest.connectAlgorithmsIO import connectAlgorithmsIO, graph_generator +from AthenaCommon.AlgSequence import AthSequencer + + +def algColor(alg): + if "Hypo" in alg: return "darkorchid1" + if "Filter" in alg: return "chartreuse3" + if "InputMaker" in alg: return "cyan3" + return "cadetblue1" + +def stepCF_ControlFlow_to_dot(stepCF): + def _dump (seq, indent): + o = list() + for c in seq.getChildren(): + if "AthSequencer" in c.getFullName(): + o.append( ("%s[color=%s, shape=circle, width=.5, fixedsize=true ,style=filled]\n"%(c.name(),_seqColor(c)), indent) ) + else: + o.append( ("%s[fillcolor=%s,style=filled]\n"%(c.name(),algColor(c.name())), indent) ) + o.append( ("%s -> %s"%(seq.name(), c.name()), indent)) + o.extend( _dump (c, indent+1) ) + return o + + def _parOR (seq): + if seq.ModeOR is True: + if seq.Sequential is False: + if seq.StopOverride is True: + return True + return False + + def _seqAND(seq): + if seq.ModeOR is False: + if seq.Sequential is True: + if seq.StopOverride is False: + return True + return False + + def _seqColor(seq): + if _parOR(seq): return "red" + if _seqAND(seq): return "blue" + return "black" + + + + # to visualize: dot -T pdf Step1.dot > Step1.pdf + file = open( '%s.CF.dot'%stepCF.name(), mode="wt" ) + #strict + file.write( 'digraph step {\n'\ + +'\n'\ + +' node [ shape=polygon, fontname=Helvetica ]\n'\ + +' edge [ fontname=Helvetica ]\n' + +' %s [shape=Mdiamond]\n'%stepCF.name()) + + indent=0 + out = [("%s[color=%s shape=circle]\n"%(stepCF.name(),_seqColor(stepCF)), indent)] + + out.extend( _dump( stepCF, indent=indent+1 ) ) + for n,i in out: + line= " "*i+ n + file.write(line) + + file.write( '}\n') + + + + + +def stepCF_DataFlow_to_dot(name, cfseq_list): + # to visualize: dot -T pdf Step1.dot > Step1.pdf + file = open( '%s.DF.dot'%name, mode="wt" ) + #strict + file.write( 'digraph step {\n'\ + +'\n'\ + +' node [ shape=polygon, fontname=Helvetica ]\n'\ + +' edge [ fontname=Helvetica ]\n' + +' %s [shape=Mdiamond]\n'%name) + + + for cfseq in cfseq_list: + file.write(" %s[fillcolor=%s style=filled]\n"%(cfseq.filter.algname,algColor(cfseq.filter.algname))) + for inp in cfseq.filter.getInputList(): + file.write(addConnection(name, cfseq.filter.algname, inp)) + for seq in cfseq.menuSeq.recoSeqList: + for inp in seq.inputMaker.getInputList(): + file.write(addConnection(cfseq.filter.algname,seq.inputMaker.algname,inp)) + + # file.write('%s -> %s [label=%s]'%(name,cfseq.filter.algname,cfseq.filter.inputs[0])) + file.write( '\n subgraph cluster_%s {\n'%(cfseq.name)\ + +' node [color=white style=filled]\n'\ + +' style=filled\n'\ + +' color=lightgrey\n'\ + +' fontname=Helvetica\n'\ + +' label = %s\n'%(cfseq.name)) + + cfseq_algs = [] + cfseq_algs.append(cfseq.filter) + # cfseq_algs.append(cfseq.hypo) + + for seq in cfseq.menuSeq.recoSeqList: + cfseq_algs.append(seq.inputMaker) + cfseq_algs.extend(seq.algs ) + if seq.reuse==False: + file.write(" %s[fillcolor=%s]\n"%(seq.inputMaker.algname, algColor(seq.inputMaker.algname))) + for alg in seq.algs: file.write(" %s[fillcolor=%s]\n"%(alg.algname, algColor(alg.algname))) + seq.reuse=True + cfseq_algs.append(seq.hypo) + + file.write(" %s[color=%s]\n"%(seq.hypo.algname, algColor(seq.hypo.algname))) + #file.write(" %s[color=%s]\n"%(cfseq.hypo.algname, algColor(cfseq.hypo.algname))) + file.write(' }\n') + file.write(findConnections(cfseq_algs)) + file.write('\n') + + file.write( '}') + file.close() + + +def findConnections(alg_list): + lineconnect='' + for nodeA in alg_list: + for nodeB in alg_list: + if nodeA is nodeB: + continue + dataIntersection = list(set(nodeA.getOutputList()) & set(nodeB.getInputList())) + if len(dataIntersection) > 0: + lineconnect+=addConnection(nodeA.algname,nodeB.algname, dataIntersection[0]) + print 'Data connections between %s and %s: %s'%(nodeA.algname, nodeB.algname, dataIntersection) + return lineconnect + +def addConnection(nodeA, nodeB, label): + line = " %s -> %s [label=%s]\n"%(nodeA,nodeB,label) + return line diff --git a/Trigger/TrigValidation/TrigUpgradeTest/python/MenuComponents.py b/Trigger/TrigValidation/TrigUpgradeTest/python/MenuComponents.py new file mode 100644 index 00000000000..e37ada66626 --- /dev/null +++ b/Trigger/TrigValidation/TrigUpgradeTest/python/MenuComponents.py @@ -0,0 +1,78 @@ +########################################################## + + +class MenuRecoSequence(): + def __init__(self, name, InputMaker, Algs, Hypo, Seed): + self.name = name + self.inputMaker=InputMaker + self.algs = Algs + self.hypo = Hypo + self.seed = Seed + self.reuse = 0 + self.connect() + + + def connect(self): + alg_input = self.inputMaker.getOutput() + for Alg in self.algs: + Alg.setInput(alg_input) + alg_input = Alg.getOutput() + self.hypo.setInput(Alg.getOutput()) + print "connect: %s"%self + + def __str__(self): + return "MenuRecoSequence %s with \n Seed::%s \n InputMaker::%s \n %s \n Hypo::%s"%(self.name, self.seed, self.inputMaker, ', \n'.join(map(str, self.algs)), self.hypo) + +class MenuSequence(): + def __init__(self, name, recoSeqList): + self.name = name + self.recoSeqList=recoSeqList + + def __str__(self): + return "MenuSequence::%s \n %s"%(self.name,',\n '.join(map(str, self.recoSeqList))) + + +class CFSeq: + def __init__(self, name, FilterAlg, MenuSequence): + self.name = name + self.filter = FilterAlg + self.menuSeq = MenuSequence + + def __str__(self): + return "--- CFSeq %s ---\n + Filter: %s \n + %s \n "%(self.name,\ + self.filter, \ + self.menuSeq) + + def getAllNodes(self): + algs = [] + for seq in self.menuSeq.recoSeqList: + algs.append(seq.inputMaker) + algs.extend(seq.algs) + algs.append(seq.hypo) + return algs + + +class SequenceThreshold: + def __init__(self, Sequence, Threshold): + self.sequence = Sequence + self.threshold = Threshold + + +class ChainStep: + def __init__(self, name, SequenceThresholds=[]): + self.name = name + self.sequenceThresholds = SequenceThresholds + + +class Chain: + def __init__(self, name, Seed, ChainSteps=[]): + self.name = name + self.seed=Seed + self.steps=ChainSteps + + seeds = Seed.strip().split("_") + seeds.pop(0) #remove first L1 + print seeds + self.group_seed = ["L1"+filter(lambda x: x.isalpha(), stri) for stri in seeds] + + diff --git a/Trigger/TrigValidation/TrigUpgradeTest/share/EmuStepProcessingTest.py b/Trigger/TrigValidation/TrigUpgradeTest/share/EmuStepProcessingTest.py index fe4073c9645..faf8d62cbd3 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/share/EmuStepProcessingTest.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/share/EmuStepProcessingTest.py @@ -93,10 +93,14 @@ include("TrigUpgradeTest/HLTCF.py") from TrigUpgradeTest.HLTCFConfig import * +from TrigUpgradeTest.MenuComponents import * + # muon chains -muRecoSeq1 = RecoSequence("muRecoSeq1", Seed="MU", Algs = [ RecoAlg("muMSRecAlg", FileName="msmu.dat") ]) -muStep1 = Sequence("Step1MuHypo", Hypo = HypoAlg("Step1MuHypo"), RecoSeq=[muRecoSeq1] ) + +include("TrigUpgradeTest/HLTSignatureConfig.py") + +muStep1 = muStep1Sequence() MuChains = [ Chain(name='HLT_mu20', Seed="L1_MU10", ChainSteps=[ChainStep("Step1_mu20", [SequenceThreshold(muStep1,"mu20")])]), Chain(name='HLT_mu8', Seed="L1_MU6", ChainSteps=[ChainStep("Step1_mu8", [SequenceThreshold(muStep1,"mu8")] )]) @@ -104,24 +108,19 @@ MuChains = [ #electron chains -elRecoSeq1 = RecoSequence("elRecoSeq1", Seed="EM", Algs=[RecoAlg("CaloClustering", FileName="emclusters.dat")] ) -elStep1 = Sequence("Step1ElGamHypo", Hypo=HypoAlg("Step1ElGamHypo"), RecoSeq=[elRecoSeq1]) - +elStep1 = elStep1Sequence() ElChains = [ Chain(name='HLT_e20' , Seed="L1_EM10", ChainSteps=[ChainStep("Step1_e20", [SequenceThreshold(elStep1,"e20")])]) ] # combined chain -muelStep1 = Sequence("Step1MuEHypo", Hypo=HypoAlg("Step1ComboMuEHypo"),RecoSeq=[muRecoSeq1, elRecoSeq1]) - +muelStep1 = combStep1Sequence() CombChains =[ Chain(name='HLT_mu8_e8' , Seed="L1_EM6_MU6", ChainSteps=[ChainStep("Step1_mu8_e8", [SequenceThreshold(muelStep1,"mu8_e8")])]) ] - - group_of_chains = MuChains + ElChains + CombChains @@ -133,22 +132,21 @@ NSTEPS=1 count_steps=0 for nstep in range(0, NSTEPS): stepCF_name = "Step%i"%(nstep+1) - step_list = [] + CFseq_list = [] for chain in group_of_chains: chain_step_name= "%s:%s"%(stepCF_name, chain.name) - print "\n*******Filling stepCF %s for chain %s"%(stepCF_name, chain.name) + print "\n*******Filling step %s for chain %s"%(stepCF_name, chain.name) chain_step=chain.steps[count_steps] sequenceThresholds=chain_step.sequenceThresholds - step_cfseq=[] for st in sequenceThresholds: sequence=st.sequence threshold=st.threshold - step_name= sequence.name - hypotool=HypoTool(threshold) + cfseq_name= sequence.name + hypotool=threshold - print "Going through sequence %s with threhold %s"%(sequence.name, threshold) + print "Going through sequence %s with threshold %s"%(sequence.name, threshold) #define sequence input if count_steps == 0: # seeding print chain.group_seed @@ -159,126 +157,56 @@ for nstep in range(0, NSTEPS): previous_sequence=chain.steps[count_steps-1].sequence sequence_input=previous_sequence.outputs - - #was this sequence/hypo already used in other steps? Find if the step exists - # the step already exista if there are chains that differ only for thresholds - step_already_exists=False - foundStep= [step for step in step_list if step.name in step_name] - if len(foundStep): - step_already_exists=True - - if step_already_exists: - sfilter= foundStep[0].filter - sfilter.addChain(chain.name) - #get the hypo - hypoAlg= foundStep[0].hypo + # add hypotools + for recoSeq in sequence.recoSeqList: + hypoAlg= recoSeq.hypo print "Adding %s to %s"%(threshold,hypoAlg.name) hypoAlg.addHypoTool(hypotool) - continue - - print "Creating new step (sequence+IM+filter) %s"%sequence.name - #make the hypo - hypoAlg=sequence.hypo - - print "Adding %s to %s"%(threshold,hypoAlg.name) - hypoAlg.addHypoTool(hypotool) - + #### FILTER # one filter per previous sequence at the start of the sequence: check if it exists or create a new one # if the previous hypo has more than one output, try to get all of them - + # one filter per previous sequence: 1 input/previous seq, 1 output/next seq filter_name="Filter_%s%s"%(stepCF_name,previous_sequence) filter_out=["%s_%s_out"%(filter_name,i) for i in sequence_input] filter_already_exists=False - findFilter= [step.filter for step in step_list if filter_name in step.filter.name] + findFilter= [cfseq.filter for cfseq in CFseq_list if filter_name in cfseq.filter.name] if len(findFilter): print "Filter %s already exists"%(filter_name) filter_already_exists=True sfilter=findFilter[0] + print "Adding chain %s to %s"%(chain.name,sfilter.name) + sfilter.setChains(chain.name) + continue else: - sfilter = SequenceFilter( filter_name) - for o in filter_out: sfilter.addOutput(o) - for i in sequence_input: sfilter.addInput(i) - print "Filter Done: %s"%(sfilter) - - sfilter.addChain(chain.name) - - - # loop over RecoSequences of this sequence - for recoSeq in sequence.recoSeq: - seed=recoSeq.seed - #### INPUT MAKER - input_maker_name = "IM_"+recoSeq.name - print "Search %s"%input_maker_name - #TMP: now run InputMaker with sequence input, until InputMaker has inplicit DH + sfilter = SequenceFilterNode(name=filter_name) + for o in filter_out: sfilter.setOutput(o) + for i in sequence_input: sfilter.setInput(i) + sfilter.setChains(chain.name) + print "Filter Done: %s"%(sfilter) + + #loop over RecoSequences of this sequence to add inputs to InputMaker + for recoSeq in sequence.recoSeqList: + seed=recoSeq.seed + #TMP: now run InputMaker with sequence input, until InputMaker has inplicit DH + input_maker_name = recoSeq.inputMaker.name input_maker_input= (inp for inp in sequence_input if seed in inp) - input_maker_exists=False - for step in step_list: - for seq in step.cfRecoSequences: - if input_maker_name in seq.inputMaker.name: - print "IM alg %s already exist: add inputs only"%(input_maker_name) - for i in input_maker_input: seq.inputMaker.addInput(i) - input_maker_exists=True - break - - # check if the RecoSequence already exists: for example e+mu use the same Recoseuqneces - recoseq_already_exists=False - cfRecoSequence_name = "CF"+recoSeq.name - findRecoSeq=[seq for step in step_list for seq in step.cfRecoSequences if cfRecoSequence_name in seq.name] - if len(findRecoSeq): - if len(findRecoSeq) == 1: - cfRecoSequence=findRecoSeq[0] - print " This RecoSequence %s already exists, add hypo inputs and outputs only"%(cfRecoSequence.name) - for i in cfRecoSequence.outputs: hypoAlg.addInput(i) - for i in cfRecoSequence.outputs: hypoAlg.addOutput("Sequence%s_%s_out"%(cfRecoSequence.name,i)) - if not input_maker_exists: - print "ERROR! Input maker %s not found for this sequence: %s"%(input_maker_name, recoSeq.name) - else: - print "ERROR: found more than one reco sequences with name %s"%cfRecoSequence_name - else: - print "Making new reco sequence %s"%(recoSeq.name) - algs=recoSeq.algs - first_alg = algs[0] - first_alg_input= first_alg.name+"_in" - if input_maker_exists: - print "ERROR: why the recoseq exists and the IM not??" - else: - input_maker = InputMaker(input_maker_name) - print input_maker - for i in input_maker_input: input_maker.addInput(i) - input_maker.addOutput(first_alg_input) - - last_output = first_alg_input - for alg in recoSeq.algs: - input=last_output - output="%s_out"%(alg.name) - alg.addInput(input) - alg.addOutput(output) - print " Added connections for %s in sequence %s"%(alg,sequence.name) - last_output= output - cfRecoSequence=CFRecoSequence(cfRecoSequence_name, recoSeq, input_maker) - hypoAlg.addInput(last_output) - sequence_out = "Sequence%s_out"%recoSeq.name - hypoAlg.addOutput(sequence_out) - - # append cfrecosequence to the list - step_cfseq.append(cfRecoSequence) - - - #end of reco sequence loop + for i in input_maker_input: recoSeq.inputMaker.setInput(i) + #end of reco sequence loop + #make the decision step - step_seq = Step( step_name, FilterAlg=sfilter, CFRecoSequences=step_cfseq, Hypo=hypoAlg) - print step_seq - step_list.append(step_seq) + CF_seq = CFSeq( cfseq_name, FilterAlg=sfilter, MenuSequence=sequence) + print CF_seq + CFseq_list.append(CF_seq) #end of sequence/threshold #end of this step configuration, now implement CF: print "\n******** instantiate algorithms in CF" - stepCF = create_StepCF(stepCF_name, step_list, dump=1) - stepCF_DataFlow_to_dot(stepCF_name, step_list) + stepCF = create_StepCF(stepCF_name, CFseq_list, dump=1) + stepCF_DataFlow_to_dot(stepCF_name, CFseq_list) stepCF_ControlFlow_to_dot(stepCF) @@ -296,5 +224,6 @@ theApp.EvtMax = 3 + # is this needed?? #step +=seqFilter diff --git a/Trigger/TrigValidation/TrigUpgradeTest/share/HLTCF.py b/Trigger/TrigValidation/TrigUpgradeTest/share/HLTCF.py index 6c6e131619b..799526a5126 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/share/HLTCF.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/share/HLTCF.py @@ -1,20 +1,7 @@ -from AthenaCommon.CFElements import parOR, seqAND, stepSeq - - -allAlgs={} -def useExisting(name): - global allAlgs - return allAlgs[name] - -def AlgExisting(name): - global allAlgs - return name in allAlgs - -def remember(name, instance): - global allAlgs - allAlgs[name] = instance - return instance +##### obsolete +from AthenaCommon.CFElements import parOR, seqAND, stepSeq +from TrigUpgradeTest.HLTCFConfig import * from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestRoRSeqFilter def seqFilter(name, Inputs=[], Outputs=[], Chains=[]): @@ -29,15 +16,20 @@ def seqFilter(name, Inputs=[], Outputs=[], Chains=[]): allAlgs[name] = f return f -from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestInputMaker -def seqInputMaker(name, Inputs=[], Outputs=[]): - global allAlgs - input_list = list(set(Inputs)) - output_list = list (set(Outputs)) - fname="IM_"+name - f = HLTTest__TestInputMaker(fname, OutputLevel = DEBUG, Input=Inputs, Output=Outputs, LinkName="initialRoI") - allAlgs[name] = f - return f +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestRecoAlg +def TestRecoAlg(name, Output, FileName="noreco.dat", Input=""): + fname = "R_"+name + a = HLTTest__TestRecoAlg(fname, OutputLevel = DEBUG, FileName=FileName, Output=Output, Input=Input) + allAlgs[name] = a + return a + + +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestHypoAlg +def TestHypoAlg(name, Input, Output): + fname = "H_"+name + h = HLTTest__TestHypoAlg(fname, OutputLevel = DEBUG, Input=Input, Output=Output) + allAlgs[name] = h + return h from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestMerger @@ -49,237 +41,41 @@ def addSteps(s): -#### Here functions to create the CF tree from CF configuration objects - -from TrigUpgradeTest.HLTCFConfig import * -include("TrigUpgradeTest/HLTSignatureConfig.py") - -import re -def create_stepSequence(step): - print "\n Create Step %s"%(step.name) - filt=step.filter - if AlgExisting(filt.name): - print "Found already seqFilter %s"%filt - filterAlg = useExisting(filt.name) - else: - print "Create seqFilter %s"%filt - filterAlg=seqFilter(filt.name, Inputs=filt.inputs, Outputs=filt.outputs, Chains=filt.chains) - algos = [] - for seq in step.cfRecoSequences: - input_maker=seq.inputMaker - if AlgExisting(input_maker.name): - print "Found already InputMaker %s"%input_maker - inputMakerAlg = useExisting(input_maker.name) - else: - print "Create seqInputMaker %s"%input_maker - inputMakerAlg= seqInputMaker(input_maker.name, Inputs=input_maker.inputs[0], Outputs=input_maker.outputs[0]) - algos.append(inputMakerAlg) - #make algos: - for alg in seq.recoSequence.algs: - if AlgExisting(alg.name): - print "Found already RecoAlg %s"%alg - algAlg = useExisting(alg.name) - else: - print "Create Algo %s"%alg - theAlg = globals()[alg.name] - algAlg=theAlg(alg.name, FileName=alg.filename, Output=alg.outputs[0], Input=alg.inputs[0]) - algos.append(algAlg) - - # make Hypo - hypo=step.hypo - if AlgExisting(hypo.name): - print "Found already HypoAlg %s"%hypo - hypoAlg = useExisting(hypo.name) - else: - print "Crate HypoAlgo %s"%hypo - theAlg = globals()[hypo.name] - - tools=hypo.tools - if "Comb" in hypo.name: ###TMP combo, only one threshold - hypoAlg=theAlg(hypo.name, Input=hypo.inputs, Output=hypo.outputs) - thresholds = map(int, re.findall(r'\d+', tools[0].name)) - hypoAlg.Threshold1= thresholds[0] - hypoAlg.Threshold2= thresholds[1] - hypoAlg.DecisionLabel=tools[0].name - else: - hypoAlg=theAlg(hypo.name, Input=hypo.inputs[0], Output=hypo.outputs[0]) - htool_list = [] - for tool in tools: - print "Crate HypoTool %s"%tool.name - theAlg = globals()[tool.name] - hypotoolTool=theAlg(tool.name) - htool_list.append(hypotoolTool) - hypoAlg.HypoTools = htool_list - algos.append(hypoAlg) +##### new - print "Crate StepSeq %s"%step.name - step_seq = stepSeq( step.name, filterAlg=filterAlg, rest=algos) - return step_seq +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestRecoAlg +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestHypoAlg +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestHypoTool +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestComboHypoAlg -def find_stepCF_algs(step_list): - algos = [] - for step in step_list: - #print "Finding algos of step %s"%step - step_algs = [] - step_algs.append(step.filter) - for seq in step.cfRecoSequences: - step_algs.extend(seq.recoSequence.algs ) - step_algs.append(step.hypo) - #print step_algs - algos.extend(step_algs) - return algos - - -from AthenaCommon.AlgSequence import dumpSequence -def create_StepCF(name, step_list, dump=0): - print "There are %d steps in this CFStep %s"%(len(step_list), name) - stepCF = parOR(name) - for step in step_list: - step_seq = create_stepSequence(step) - print "Add this step %s to %s"%(step.name,name) - stepCF += step_seq - - - if dump: dumpSequence (stepCF, indent=0) - - return stepCF - - - -###### Here some graphical methods - - -from ViewAlgsTest.connectAlgorithmsIO import connectAlgorithmsIO, graph_generator -from AthenaCommon.AlgSequence import AthSequencer - - -def algColor(alg): - if "Hypo" in alg: return "darkorchid1" - if "F_" in alg: return "chartreuse3" - if "Filter" in alg: return "chartreuse3" - if "IM_" in alg: return "cyan3" - return "cadetblue1" - -def stepCF_ControlFlow_to_dot(stepCF): - def _dump (seq, indent): - o = list() - for c in seq.getChildren(): - if "AthSequencer" in c.getFullName(): - o.append( ("%s[color=%s, shape=circle, width=.5, fixedsize=true ,style=filled]\n"%(c.name(),_seqColor(c)), indent) ) - else: - o.append( ("%s[fillcolor=%s,style=filled]\n"%(c.name(),algColor(c.name())), indent) ) - o.append( ("%s -> %s"%(seq.name(), c.name()), indent)) - o.extend( _dump (c, indent+1) ) - return o - - def _parOR (seq): - if seq.ModeOR is True: - if seq.Sequential is False: - if seq.StopOverride is True: - return True - return False - - def _seqAND(seq): - if seq.ModeOR is False: - if seq.Sequential is True: - if seq.StopOverride is False: - return True - return False - - def _seqColor(seq): - if _parOR(seq): return "red" - if _seqAND(seq): return "blue" - return "black" - - - - # to visualize: dot -T pdf Step1.dot > Step1.pdf - file = open( '%s.CF.dot'%stepCF.name(), mode="wt" ) - #strict - file.write( 'digraph step {\n'\ - +'\n'\ - +' node [ shape=polygon, fontname=Helvetica ]\n'\ - +' edge [ fontname=Helvetica ]\n' - +' %s [shape=Mdiamond]\n'%stepCF.name()) - - indent=0 - out = [("%s[color=%s shape=circle]\n"%(stepCF.name(),_seqColor(stepCF)), indent)] - - out.extend( _dump( stepCF, indent=indent+1 ) ) - for n,i in out: - line= " "*i+ n - file.write(line) - - file.write( '}\n') - - - - - -def stepCF_DataFlow_to_dot(name, step_list): - # to visualize: dot -T pdf Step1.dot > Step1.pdf - file = open( '%s.DF.dot'%name, mode="wt" ) - #strict - file.write( 'digraph step {\n'\ - +'\n'\ - +' node [ shape=polygon, fontname=Helvetica ]\n'\ - +' edge [ fontname=Helvetica ]\n' - +' %s [shape=Mdiamond]\n'%name) +def TestHypoTool(name, prop): + threshold_value=''.join(filter(lambda x: x.isdigit(), name)) + value = int(threshold_value)*1000 + h = HLTTest__TestHypoTool(name, OutputLevel=DEBUG, Threshold=value, Property=prop) + return h +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestInputMaker +def InputMakerAlg(name): + return HLTTest__TestInputMaker(name, OutputLevel = DEBUG, LinkName="initialRoI") - for step in step_list: - file.write(" %s[fillcolor=%s style=filled]\n"%(step.filter.name,algColor(step.filter.name))) - for inp in step.filter.inputs: - file.write(addConnection(name, step.filter.name, inp)) - for seq in step.cfRecoSequences: - for inp in seq.inputMaker.inputs: - file.write(addConnection(step.filter.name,seq.inputMaker.name,inp)) - # file.write('%s -> %s [label=%s]'%(name,step.filter.name,step.filter.inputs[0])) - file.write( '\n subgraph cluster_%s {\n'%(step.name)\ - +' node [color=white style=filled]\n'\ - +' style=filled\n'\ - +' color=lightgrey\n'\ - +' fontname=Helvetica\n'\ - +' label = %s\n'%(step.name)) +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestRoRSeqFilter +def SeqFilterNew(name): + return HLTTest__TestRoRSeqFilter(name, OutputLevel = DEBUG) - step_algs = [] - step_algs.append(step.filter) - step_algs.append(step.hypo) - for seq in step.cfRecoSequences: - step_algs.append(seq.inputMaker) - step_algs.extend(seq.recoSequence.algs ) - if seq.recoSequence.reuse==False: - file.write(" %s[fillcolor=%s]\n"%(seq.inputMaker.name, algColor(seq.inputMaker.name))) - for alg in seq.recoSequence.algs: file.write(" %s[fillcolor=%s]\n"%(alg.name, algColor(alg.name))) - seq.recoSequence.reuse=True - - file.write(" %s[color=%s]\n"%(step.hypo.name, algColor(step.hypo.name))) - file.write(' }\n') - file.write(findConnections(step_algs)) - file.write('\n') - - file.write( '}') - file.close() +def TestRecAlgNew(name, FileName="noreco.dat"): + return HLTTest__TestRecoAlg(name=name, FileName=FileName, OutputLevel = DEBUG) +def TestHypoAlgNew(name): + return HLTTest__TestHypoAlg(name=name, OutputLevel = DEBUG) -def findConnections(alg_list): - lineconnect='' - for nodeA in alg_list: - for nodeB in alg_list: - if nodeA is nodeB: - continue - dataIntersection = list(set(nodeA.outputs) & set(nodeB.inputs)) - if len(dataIntersection) > 0: - lineconnect+=addConnection(nodeA.name,nodeB.name, dataIntersection[0]) - print 'Data connections between %s and %s: %s'%(nodeA.name, nodeB.name, dataIntersection) - return lineconnect +def muTestHypoTool(name): + return TestHypoTool(name, "pt") -def addConnection(nodeA, nodeB, label): - line = " %s -> %s [label=%s]\n"%(nodeA,nodeB,label) - return line +def elTestHypoTool(name): + return TestHypoTool(name, "et") diff --git a/Trigger/TrigValidation/TrigUpgradeTest/share/HLTSignatureConfig.py b/Trigger/TrigValidation/TrigUpgradeTest/share/HLTSignatureConfig.py index 2582f057348..2ce98e394b6 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/share/HLTSignatureConfig.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/share/HLTSignatureConfig.py @@ -1,46 +1,33 @@ - from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestRecoAlg -def TestRecoAlg(name, Output, FileName="noreco.dat", Input=""): - fname = "R_"+name - a = HLTTest__TestRecoAlg(fname, OutputLevel = DEBUG, FileName=FileName, Output=Output, Input=Input) - allAlgs[name] = a - return a - - from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestHypoAlg -def TestHypoAlg(name, Input, Output): - fname = "H_"+name - h = HLTTest__TestHypoAlg(fname, OutputLevel = DEBUG, Input=Input, Output=Output) - allAlgs[name] = h - return h - from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestHypoTool +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestComboHypoAlg + def TestHypoTool(name, prop): threshold_value=''.join(filter(lambda x: x.isdigit(), name)) value = int(threshold_value)*1000 h = HLTTest__TestHypoTool(name, OutputLevel=DEBUG, Threshold=value, Property=prop) return h -from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestComboHypoAlg -def TestComboHypoAlg(name, Input, Output, prop1, prop2): - ## TMP - fname = "H_"+name - h = HLTTest__TestComboHypoAlg(fname, OutputLevel = DEBUG, Input1=Input[0], Input2=Input[1], \ - Output1=Output[0],Output2=Output[1],\ - Property1=prop1, Property2=prop2) - #, Threshold1=8000, Threshold2=8000,\ - #DecisionLabel="HLT_mu8_e8") - allAlgs[name] = h - return h +from TrigUpgradeTest.TrigUpgradeTestConf import HLTTest__TestInputMaker +def InputMakerAlg(name): + return HLTTest__TestInputMaker(name, OutputLevel = DEBUG, LinkName="initialRoI") + + +# here define the sequences from the signatures +# signatures do this: +# - declare all the RecoAlg and the HypoAlg -> create the Sequence +# - creates the InputMaker, without the inputs + #muon signatures -def muMSRecAlg(name, Output, FileName="noreco.dat", Input=""): - return TestRecoAlg(name, Output, FileName, Input) +def muMSRecAlg(name, FileName="noreco.dat"): + return HLTTest__TestRecoAlg(name=name, FileName=FileName, OutputLevel = DEBUG) -def Step1MuHypo(name, Input, Output): - return TestHypoAlg(name, Input, Output) +def Step1MuHypo(name): + return HLTTest__TestHypoAlg(name=name, OutputLevel = DEBUG) def mu20(name): return TestHypoTool(name, "pt") @@ -49,13 +36,28 @@ def mu8(name): return TestHypoTool(name, "pt") +muIM= InputMakerAlg(name="Step1MuInputMaker") +step1_mu_inputMaker = AlgNode(Alg=muIM, inputProp='Input', outputProp='Output') + +muAlg = muMSRecAlg(name="muMSRecAlg", FileName="msmu.dat") +step1_mu_recoAlg = AlgNode( Alg=muAlg,inputProp='Input', outputProp='Output') + +muHypo = Step1MuHypo(name="Step1MuHypo") +step1_mu_HypoAlg = HypoAlgNode( Alg=muHypo,inputProp='Input', outputProp='Output') +recoSequence_mu = MenuRecoSequence("muRecoSeq1", InputMaker=step1_mu_inputMaker, Algs=[step1_mu_recoAlg], Hypo=step1_mu_HypoAlg, Seed="MU") + +def muStep1Sequence(): + return MenuSequence("muStep1Seq", recoSeqList=[recoSequence_mu]) + + #electron signatures -def CaloClustering(name, Output, FileName="noreco.dat", Input=""): - return TestRecoAlg(name, Output, FileName, Input) +def CaloClustering(name, FileName="noreco.dat"): + return HLTTest__TestRecoAlg(name=name, FileName=FileName, OutputLevel = DEBUG) + +def Step1ElGamHypo(name): + return HLTTest__TestHypoAlg(name=name, OutputLevel = DEBUG) -def Step1ElGamHypo(name, Input, Output): - return TestHypoAlg(name, Input, Output) def e20(name): return TestHypoTool(name, "et") @@ -63,11 +65,35 @@ def e20(name): def e8(name): return TestHypoTool(name, "et") +elIM= InputMakerAlg(name="Step1ElInputMaker") +step1_el_inputMaker = AlgNode(Alg=elIM, inputProp='Input', outputProp='Output') + +elAlg = CaloClustering(name="CaloClustering", FileName="emclusters.dat") +step1_el_recoAlg = AlgNode( Alg=elAlg,inputProp='Input', outputProp='Output') + +elHypo = Step1ElGamHypo(name="Step1ElHypo") +step1_el_HypoAlg = HypoAlgNode( Alg=elHypo,inputProp='Input', outputProp='Output') + +recoSequence_el = MenuRecoSequence("elRecoSeq1", InputMaker=step1_el_inputMaker, Algs=[step1_el_recoAlg], Hypo=step1_el_HypoAlg, Seed="EM") +def elStep1Sequence(): + return MenuSequence("elStep1Seq", recoSeqList=[recoSequence_el]) + #combined Hypo -def Step1ComboMuEHypo(name, Input, Output): - return TestComboHypoAlg(name, Input, Output, "pt", "et") +def Step1ComboMuEHypo(name): + return HLTTest__TestComboHypoAlg(name=name, OutputLevel = DEBUG, Property1="pt", Property2="et") def mu8_e8(name): return TestHypoTool(name, "pt") + +comboAlg = Step1ComboMuEHypo("Step1ComboMuElHypo") +step1_comb_HypoAlgMu = HypoAlgNode( Alg=comboAlg,inputProp='Input1', outputProp='Output1') +step1_comb_HypoAlgEl = HypoAlgNode( Alg=comboAlg,inputProp='Input2', outputProp='Output2') + +recoSequence_muComb = MenuRecoSequence("CombmuRecoSeq1", InputMaker=step1_mu_inputMaker, Algs=[step1_mu_recoAlg], Hypo=step1_comb_HypoAlgMu, Seed="MU") +recoSequence_elComb = MenuRecoSequence("CombelRecoSeq1", InputMaker=step1_el_inputMaker, Algs=[step1_el_recoAlg], Hypo=step1_comb_HypoAlgEl, Seed="EM") + +def combStep1Sequence(): + return MenuSequence("combStep1Seq", recoSeqList=[recoSequence_muComb,recoSequence_elComb]) + diff --git a/Trigger/TrigValidation/TrigUpgradeTest/share/L1CF.py b/Trigger/TrigValidation/TrigUpgradeTest/share/L1CF.py index 2a916e30595..4e21e5206f7 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/share/L1CF.py +++ b/Trigger/TrigValidation/TrigUpgradeTest/share/L1CF.py @@ -25,6 +25,7 @@ muUnpacker.ThresholdToChainMapping = ["MU6 : HLT_mu6", "MU8 : HLT_mu8", "MU8 : H l1Decoder.roiUnpackers = [emUnpacker, muUnpacker] +#print l1Decoder L1UnpackingSeq += l1Decoder TopHLTSeq += L1UnpackingSeq diff --git a/Trigger/TrigValidation/TrigUpgradeTest/src/TestInputMaker.cxx b/Trigger/TrigValidation/TrigUpgradeTest/src/TestInputMaker.cxx index 7cf875dac67..f1b5f96255d 100644 --- a/Trigger/TrigValidation/TrigUpgradeTest/src/TestInputMaker.cxx +++ b/Trigger/TrigValidation/TrigUpgradeTest/src/TestInputMaker.cxx @@ -87,7 +87,7 @@ namespace HLTTest { } auto featurePtr(featurelink.cptr()); - ATH_MSG_DEBUG("Found feature" << **featurePtr ); + ATH_MSG_DEBUG("Found feature " <<m_linkName.value() <<": " << **featurePtr ); // copy all features to a single output collection // they just happen to be TrigComposites in this case but should be whatever is needed as input by the reco algorithm next in the sequence, e.g. CaloCells. diff --git a/Trigger/TrigValidation/TrigUpgradeTest/test/test_emu_step_processing.sh b/Trigger/TrigValidation/TrigUpgradeTest/test/test_emu_step_processing.sh new file mode 100755 index 00000000000..f92b2e530b1 --- /dev/null +++ b/Trigger/TrigValidation/TrigUpgradeTest/test/test_emu_step_processing.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# art-type: build +# art-ci: master + +athena.py --threads=1 TrigUpgradeTest/EmuStepProcessingTest.py -- GitLab