From 93f5c587366a819d78feb89cab0a58daef923e63 Mon Sep 17 00:00:00 2001 From: Tomasz Bold <tomasz.bold@gmail.com> Date: Tue, 15 Sep 2020 12:03:21 +0000 Subject: [PATCH] Fix configurations of combined chains in newJO test --- .../DecisionHandling/src/TestHypoAlg.cxx | 11 +- .../L1Decoder/src/CTPUnpackingTool.cxx | 17 +- .../L1Decoder/src/CTPUnpackingTool.h | 1 + .../share/runHLT_standalone_newJO.py | 5 +- .../HLTMenuConfig/Menu/HLTCFConfig_newJO.py | 186 +++++++++++------- .../python/HLTMenuConfig/Menu/LS2_v1_newJO.py | 5 +- 6 files changed, 132 insertions(+), 93 deletions(-) diff --git a/Trigger/TrigSteer/DecisionHandling/src/TestHypoAlg.cxx b/Trigger/TrigSteer/DecisionHandling/src/TestHypoAlg.cxx index e33ed7f576c..a5c5ed3e0fc 100644 --- a/Trigger/TrigSteer/DecisionHandling/src/TestHypoAlg.cxx +++ b/Trigger/TrigSteer/DecisionHandling/src/TestHypoAlg.cxx @@ -26,10 +26,14 @@ namespace HLTTest { } - StatusCode TestHypoAlg::execute( const EventContext& context ) const { + StatusCode TestHypoAlg::execute( const EventContext& context ) const { + // new output decisions + SG::WriteHandle<DecisionContainer> outputHandle = createAndStore(decisionOutput(), context ); + auto decisions = outputHandle.ptr(); + ATH_MSG_DEBUG( "Executing " << name() << "..." ); if ( m_recoInput.key().empty() ) { - ATH_MSG_DEBUG( "No input configured, not producing the output" ); + ATH_MSG_DEBUG( "No input configured, output decisions will be empty" ); return StatusCode::SUCCESS; } @@ -44,9 +48,6 @@ namespace HLTTest { auto recoInput = SG::makeHandle(m_recoInput, context); ATH_MSG_DEBUG( " and with "<< recoInput->size() <<" reco inputs"); - // new output decisions - SG::WriteHandle<DecisionContainer> outputHandle = createAndStore(decisionOutput(), context ); - auto decisions = outputHandle.ptr(); // find features: std::vector<const FeatureOBJ*> featureFromDecision; diff --git a/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.cxx b/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.cxx index c901aaafaf3..8169572ad7d 100644 --- a/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.cxx +++ b/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.cxx @@ -102,20 +102,21 @@ StatusCode CTPUnpackingTool::start() { StatusCode CTPUnpackingTool::decode( const ROIB::RoIBResult& roib, HLT::IDVec& enabledChains ) const { - auto nTAVItems = Monitored::Scalar( "TAVItems", 0 ); + auto nItems = Monitored::Scalar( "Items", 0 ); auto nChains = Monitored::Scalar( "Chains", 0 ); - auto monitorit = Monitored::Group( m_monTool, nTAVItems, nChains ); - auto tav = roib.cTPResult().TAV(); - const size_t tavSize = tav.size(); + auto monitorit = Monitored::Group( m_monTool, nItems, nChains ); - for ( size_t wordCounter = 0; wordCounter < tavSize; ++wordCounter ) { + auto ctpbits = m_useTBPBit ? roib.cTPResult().TBP() : roib.cTPResult().TAV(); + const size_t bitsSize = ctpbits.size(); + + for ( size_t wordCounter = 0; wordCounter < bitsSize; ++wordCounter ) { for ( size_t bitCounter = 0; bitCounter < 32; ++bitCounter ) { const size_t ctpIndex = 32*wordCounter + bitCounter; - const bool decision = ( tav[wordCounter].roIWord() & ((uint32_t)1 << bitCounter) ) > 0; + const bool decision = ( ctpbits[wordCounter].roIWord() & ((uint32_t)1 << bitCounter) ) > 0; if ( decision == true or m_forceEnable ) { if ( decision ) { - nTAVItems = nTAVItems + 1; + nItems = nItems + 1; ATH_MSG_DEBUG( "L1 item " << ctpIndex << " active, enabling chains " << (m_forceEnable ? " due to the forceEnable flag" : " due to the seed")); } @@ -136,7 +137,7 @@ StatusCode CTPUnpackingTool::decode( const ROIB::RoIBResult& roib, HLT::IDVec& for ( auto chain: enabledChains ) { ATH_MSG_DEBUG( "Enabling chain: " << chain ); } - if ( nTAVItems == 0 ) { + if ( nItems == 0 ) { ATH_MSG_ERROR( "All CTP bits were disabled, this event shoudl not have shown here" ); return StatusCode::FAILURE; } diff --git a/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.h b/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.h index 0eab6c38c91..90d5794ea44 100644 --- a/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.h +++ b/Trigger/TrigSteer/L1Decoder/src/CTPUnpackingTool.h @@ -38,6 +38,7 @@ private: SG::ReadHandleKey<TrigConf::L1Menu> m_L1MenuKey{ this, "L1TriggerMenu", "DetectorStore+L1TriggerMenu", "L1 Menu" }; Gaudi::Property<bool> m_useNewConfig{ this, "UseNewConfig", false, "When true, read the menu from detector store, when false use the L1ConfigSvc" }; + Gaudi::Property<bool> m_useTBPBit{ this, "UseTBPBits", false, "When true, use Trigger Before Prescale bits instead of Trigger After Veto (for testing only)" }; }; diff --git a/Trigger/TriggerCommon/TriggerJobOpts/share/runHLT_standalone_newJO.py b/Trigger/TriggerCommon/TriggerJobOpts/share/runHLT_standalone_newJO.py index e150d49dc75..09e71a0b531 100644 --- a/Trigger/TriggerCommon/TriggerJobOpts/share/runHLT_standalone_newJO.py +++ b/Trigger/TriggerCommon/TriggerJobOpts/share/runHLT_standalone_newJO.py @@ -38,7 +38,7 @@ flags.Exec.MaxEvents=50 flags.Input.isMC = False flags.Input.Files= ["/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/TrigP1Test/data17_13TeV.00327265.physics_EnhancedBias.merge.RAW._lb0100._SFO-1._0001.1"] -flags.Trigger.L1Decoder.forceEnableAllChains = True + flags.Concurrency.NumThreads=1 flags.Concurrency.NumConcurrentEvents=1 @@ -75,6 +75,7 @@ from TrigInDetConfig.TrigInDetConfig import TrigInDetCondConfig acc.merge( TrigInDetCondConfig( flags ) ) acc.getEventAlgo( "TrigSignatureMoniMT" ).OutputLevel=DEBUG +acc.getEventAlgo( "L1Decoder" ).ctpUnpacker.UseTBPBits=True # test setup @@ -89,8 +90,6 @@ acc.foreach_component("*HLTTop/*Input*").OutputLevel = DEBUG # input makers acc.foreach_component("*HLTTop/*HLTEDMCreator*").OutputLevel = DEBUG # messaging from the EDM creators acc.foreach_component("*HLTTop/*GenericMonitoringTool*").OutputLevel = WARNING # silcence mon tools (addressing by type) acc.foreach_component("*/L1Decoder").OutputLevel = DEBUG -acc.foreach_component("*FastEMCaloAlgo*").OutputLevel = DEBUG -acc.foreach_component("VDVFastEgammaCalo").OutputLevel =DEBUG fname = "runHLT_standalone_newJO.pkl" print( "Storing config in the file {}".format( fname ) ) diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/HLTCFConfig_newJO.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/HLTCFConfig_newJO.py index 3353d4bfd39..87091d47e19 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/HLTCFConfig_newJO.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/HLTCFConfig_newJO.py @@ -2,6 +2,7 @@ from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator from AthenaConfiguration.ComponentFactory import CompFactory +from TriggerMenuMT.HLTMenuConfig.Menu.ChainDictTools import splitChainInDict from TriggerMenuMT.HLTMenuConfig.Menu.MenuComponentsNaming import CFNaming from TriggerMenuMT.HLTMenuConfig.Menu.TriggerConfigHLT import TriggerConfigHLT from TriggerMenuMT.HLTMenuConfig.Menu.MenuComponents import isInputMakerBase, isHypoBase, isComboHypoAlg @@ -61,6 +62,7 @@ def generateDecisionTree(chains): """ name = "Menu{}{}".format(stepNumber, stepName) seq = seqAND( name ) + allRecoSeqName = getRecosStepSeq( stepNumber ).name acc.addSequence(seq, parentName = allRecoSeqName ) return seq @@ -95,7 +97,7 @@ def generateDecisionTree(chains): log.debug('Creted filter {}'.format(filterName)) return filterAlg - + @memoize def findInputMaker( stepCounter, stepName ): seq = getSingleMenuSeq( stepCounter, stepName ) @@ -118,6 +120,14 @@ def generateDecisionTree(chains): return result else: raise Exception("No input maker in seq "+seq.name) + @memoize + def findComboHypoAlg( stepCounter, stepName ): + seq = getSingleMenuSeq( stepCounter, stepName ) + algs = findAllAlgorithms( seq ) + for alg in algs: + if isComboHypoAlg(alg): + return alg + raise Exception("No combo hypo alg in seq "+seq.name) @memoize def findHypoAlg( stepCounter, stepName ): @@ -128,6 +138,7 @@ def generateDecisionTree(chains): return alg raise Exception("No hypo alg in seq "+seq.name) + @memoize def findAllHypoAlgs( stepCounter, stepName ): seq = getSingleMenuSeq( stepCounter, stepName ) @@ -142,22 +153,15 @@ def generateDecisionTree(chains): else: raise Exception("No hypo alg in seq "+seq.name) - @memoize - def findComboHypoAlg( stepCounter, stepName ): - seq = getSingleMenuSeq( stepCounter, stepName ) - algs = findAllAlgorithms( seq ) - for alg in algs: - if isComboHypoAlg(alg): - return alg - raise Exception("No combo hypo alg in seq "+seq.name) def addAndAssureUniqness( prop, toadd, context="" ): - if toadd not in prop: - log.info("{} value {} not there".format(context, toadd)) - return list( prop ) + [ toadd ] - else: - log.info("{} value {} already there".format(context, toadd)) - return list( prop ) + if isinstance(toadd, str): + toadd = [toadd] + missing = [] + for t in toadd: + if t not in prop: + missing.append( t ) + return list( prop ) + missing def assureUnsetOrTheSame(prop, toadd, context): """ @@ -168,7 +172,36 @@ def generateDecisionTree(chains): if prop != toadd: raise Exception("{}, when setting property found conflicting values, existing {} and new {}".format(context, prop, toadd)) + def clearUnderscores(s): + p = s + while True: + n = p.replace("__", "_") + if p == n: + return p.rstrip("_") + p = n + @memoize + def prevStepOutput( chain, stepCounter ): + """ + Returns list of decision collections that are outputs of previous step as well as the hypo alg name that outpus it + """ + prevHypoAlgName = "" + if stepCounter == 1: # L1 seed + out = chain.L1decisions + else: + prevCounter = stepCounter-1 + prevName = chain.steps[prevCounter-1].name # counting steps from 1, for indexing need one less + prevStep = chain.steps[prevCounter-1] + if prevStep.isCombo: + prevHypoAlg = findComboHypoAlg( prevCounter, prevName ) + else: + prevHypoAlg = findHypoAlg( prevCounter, prevName ) + out = prevHypoAlg.HypoOutputDecisions + prevHypoAlgName = prevHypoAlg.name + + return [out] if isinstance( out, str) else out, prevHypoAlgName # normalise to list + + # CF construction logic # create all sequences and filter algs, merge CAs from signatures (decision CF) for chain in chains: for stepCounter, step in enumerate( chain.steps, 1 ): @@ -177,7 +210,6 @@ def generateDecisionTree(chains): if step.isCombo: # add sequences that allows reconstructions to be run in parallel, followed (in sequence) by the combo hypo comboSeq, comboRecoSeq = getComboSequences( stepCounter, step.name ) - for sequence in step.sequences: acc.merge( sequence.ca, sequenceName=comboRecoSeq.name) @@ -212,84 +244,88 @@ def generateDecisionTree(chains): comboHypoAlg.HypoOutputDecisions = [] - # connect all outputs (decision DF) + # connect all outputs (decision DF) and add chains to filter on for chain in chains: + for stepCounter, step in enumerate( chain.steps, 1 ): - for seqCounter, sequence in enumerate( step.sequences ): - - # Filters linking - filterAlg = getFilterAlg( stepCounter, step.name ) - if step.isCombo: - chainDictLegs = ' '.join(map(str, [dic['chainName'] for dic in step.chainDicts])) - filterAlg.Chains = addAndAssureUniqness( filterAlg.Chains, chainDictLegs, "{} filter alg chains".format( filterAlg.name ) ) - else: - filterAlg.Chains = addAndAssureUniqness( filterAlg.Chains, chain.name, "{} filter alg chains".format( filterAlg.name ) ) - - if stepCounter == 1: - filterAlg.Input = addAndAssureUniqness( filterAlg.Input, chain.L1decisions[0], "{} L1 input".format( filterAlg.name ) ) - else: # look into the previous step - hypoOutput = findHypoAlg( stepCounter-1, chain.steps[chain.steps.index( step )-1].name ).HypoOutputDecisions - filterAlg.Input = addAndAssureUniqness( filterAlg.Input, hypoOutput, "{} input".format( filterAlg.name ) ) - - # Input Maker linking - im = findAllInputMakers( stepCounter, step.name )[seqCounter] - for i in filterAlg.Input: - filterOutputName = CFNaming.filterOutName( filterAlg.name, i ) - filterAlg.Output = addAndAssureUniqness( filterAlg.Output, filterOutputName, "{} output".format( filterAlg.name ) ) - im.InputMakerInputDecisions = addAndAssureUniqness( im.InputMakerInputDecisions, filterOutputName, "{} input".format( im.name ) ) - - imOutputName = CFNaming.inputMakerOutName( im.name ) - im.InputMakerOutputDecisions = assureUnsetOrTheSame( im.InputMakerOutputDecisions, imOutputName, "{} IM output".format( im.name ) ) - + # Filters linking + filterAlg = getFilterAlg( stepCounter, step.name ) + + def __setup(sequenceCounter, chainDict): + ''' + Local function to setup filter/input makers/hypo algs IO + ''' + # set chain to filter on + filterAlg.Chains = addAndAssureUniqness( filterAlg.Chains, chainDict["chainName"], "{} filter alg chains".format( filterAlg.name ) ) + + filterIn, prevHypoName = prevStepOutput( chain, stepCounter) + filterAlg.Input = addAndAssureUniqness( filterAlg.Input, filterIn, "{} input".format( filterAlg.name ) ) + filterOut = [ clearUnderscores(CFNaming.filterOutName( filterAlg.name, s ).replace(prevHypoName, "")) for s in filterIn ] + filterAlg.Output = addAndAssureUniqness( filterAlg.Output, filterOut, "{} output".format( filterAlg.name ) ) + + im = findAllInputMakers( stepCounter, step.name )[sequenceCounter] + im.InputMakerInputDecisions = addAndAssureUniqness( im.InputMakerInputDecisions, filterOut[sequenceCounter], "{} input".format( im.name ) ) + imOut = CFNaming.inputMakerOutName( im.name ) + im.InputMakerOutputDecisions = assureUnsetOrTheSame( im.InputMakerOutputDecisions, imOut, "{} IM output".format( im.name ) ) + # Hypo linking - hypoAlg = findAllHypoAlgs( stepCounter, step.name )[seqCounter] + hypoAlg = findAllHypoAlgs( stepCounter, step.name )[sequenceCounter] hypoAlg.HypoInputDecisions = assureUnsetOrTheSame( hypoAlg.HypoInputDecisions, im.InputMakerOutputDecisions, "{} hypo input".format( hypoAlg.name ) ) - hypoOutName = CFNaming.hypoAlgOutName( hypoAlg.name ) - hypoAlg.HypoOutputDecisions = assureUnsetOrTheSame( hypoAlg.HypoOutputDecisions, hypoOutName, + hypoOut = CFNaming.hypoAlgOutName( hypoAlg.name ) + hypoAlg.HypoOutputDecisions = assureUnsetOrTheSame( hypoAlg.HypoOutputDecisions, hypoOut, "{} hypo output".format( hypoAlg.name ) ) + hypoAlg.HypoTools.append( step.sequences[sequenceCounter]._hypoToolConf.confAndCreate( chainDict ) ) + pass - # Hypo Tools - if step.isCombo: - from TriggerMenuMT.HLTMenuConfig.Menu.ChainDictTools import splitChainInDict - chainDictLeg = splitChainInDict(chain.name)[seqCounter] - hypoAlg.HypoTools.append( sequence._hypoToolConf.confAndCreate( chainDictLeg ) ) + if step.isCombo: + for seqCounter in range( len( step.sequences ) ) : + chainLegDict = splitChainInDict( chain.name )[seqCounter] + __setup( seqCounter, chainLegDict ) - # to be deleted after ComboHypos will be properly configured and included in DF - hypoAlg.HypoTools.append( sequence._hypoToolConf.confAndCreate( TriggerConfigHLT.getChainDictFromChainName( chain.name ) ) ) - else: - hypoAlg.HypoTools.append( sequence._hypoToolConf.confAndCreate( TriggerConfigHLT.getChainDictFromChainName( chain.name ) ) ) + comboHypoAlg = findComboHypoAlg( stepCounter, step.name ) + comboHypoAlg.MultiplicitiesMap[chain.name] = step.multiplicity - # Combo Hypo linking - if step.isCombo: - comboHypoAlg = findComboHypoAlg( stepCounter, step.name ) - comboHypoAlg.MultiplicitiesMap[chain.name] = step.multiplicity + elementaryHypos = findAllHypoAlgs( stepCounter, step.name ) + for hypo in elementaryHypos: + if hypo == comboHypoAlg: + continue + comboHypoAlg.HypoInputDecisions = addAndAssureUniqness( comboHypoAlg.HypoInputDecisions, hypo.HypoOutputDecisions, + "{} comboHypo input".format( comboHypoAlg.name ) ) - comboInputList = findAllHypoAlgs( stepCounter, step.name ) - for comboInput in comboInputList: - comboHypoAlg.HypoInputDecisions = addAndAssureUniqness( comboHypoAlg.HypoInputDecisions, comboInput.name, - "{} comboHypo input".format( comboHypoAlg.name ) ) - - comboOutName = CFNaming.comboHypoOutputName( comboHypoAlg.name, comboInput.name ) - comboHypoAlg.HypoOutputDecisions = addAndAssureUniqness( comboHypoAlg.HypoOutputDecisions, comboOutName, - "{} comboHypo output".format( comboHypoAlg.name ) ) + comboOut = CFNaming.comboHypoOutputName( comboHypoAlg.name, hypo.name ) + comboHypoAlg.HypoOutputDecisions = addAndAssureUniqness( comboHypoAlg.HypoOutputDecisions, comboOut, + "{} comboHypo output".format( comboHypoAlg.name ) ) - # Combo Hypo Tools - for comboToolConf in step.comboToolConfs: - comboHypoAlg.ComboHypoTools.append( comboToolConf.confAndCreate( TriggerConfigHLT.getChainDictFromChainName( chain.name ) ) ) + # Combo Hypo Tools + for comboToolConf in step.comboToolConfs: + comboHypoAlg.ComboHypoTools.append( comboToolConf.confAndCreate( TriggerConfigHLT.getChainDictFromChainName( chain.name ) ) ) + + else: + assert len( step.sequences ) == 1, "chain {} step {} is not combo bye has number of sequences = {}".format( chain.name, stepCounter, len( step.sequences ) ) + __setup( 0, TriggerConfigHLT.getChainDictFromChainName( chain.name ) ) for chain in chains: + log.info( "CF algorithms for chain {}".format( chain.name ) ) for stepCounter, step in enumerate( chain.steps, 1 ): filterAlg = getFilterAlg( stepCounter, step.name ) - log.info("FilterAlg {} Inputs {} Outputs {}".format( filterAlg.name, filterAlg.Input, filterAlg.Output ) ) + log.info(" FilterAlg {} Inputs {} Outputs {} Chains {}".format( filterAlg.name, filterAlg.Input, filterAlg.Output, filterAlg.Chains ) ) imAlg = findInputMaker( stepCounter, step.name ) - log.info("InputMaker {} Inputs {} Outputs {}".format( imAlg.name, imAlg.InputMakerInputDecisions, imAlg.InputMakerOutputDecisions ) ) - - hypoAlg = findHypoAlg( stepCounter, step.name ) - log.info("HypoAlg {} Inputs {} Outputs {}".format( hypoAlg.name, hypoAlg.HypoInputDecisions, hypoAlg.HypoOutputDecisions ) ) + log.info(" InputMaker {} Inputs {} Outputs {}".format( imAlg.name, imAlg.InputMakerInputDecisions, imAlg.InputMakerOutputDecisions ) ) + if step.isCombo: + hypoAlgs = findAllHypoAlgs( stepCounter, step.name ) + for hypoAlg in hypoAlgs: + if isComboHypoAlg(hypoAlg): + continue + log.info(" HypoAlg {} Inputs {} Outputs {} Tools {}".format( hypoAlg.name, hypoAlg.HypoInputDecisions, hypoAlg.HypoOutputDecisions, [t.name for t in hypoAlg.HypoTools] ) ) + combo = findComboHypoAlg( stepCounter, step.name ) + log.info(" ComboHypoAlg {} Inputs {} Outputs {} Multiplicities {}".format( combo.name, combo.HypoInputDecisions, combo.HypoOutputDecisions, combo.MultiplicitiesMap ) ) + else: + hypoAlg = findHypoAlg( stepCounter, step.name ) + log.info(" HypoAlg {} Inputs {} Outputs {} Tools {}".format( hypoAlg.name, hypoAlg.HypoInputDecisions, hypoAlg.HypoOutputDecisions, [t.name for t in hypoAlg.HypoTools] ) ) return acc diff --git a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/LS2_v1_newJO.py b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/LS2_v1_newJO.py index c6034520646..4d7d8358337 100644 --- a/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/LS2_v1_newJO.py +++ b/Trigger/TriggerCommon/TriggerMenuMT/python/HLTMenuConfig/Menu/LS2_v1_newJO.py @@ -36,6 +36,7 @@ def setupMenu(flags): SingleElectronGroup = ['RATE:SingleElectron', 'BW:Electron'] SinglePhotonGroup = ['RATE:SinglePhoton', 'BW:Photon'] SingleJetGroup = ['RATE:SingleJet', 'BW:Jet'] + CombinedGroup = ['RATE:Combined', 'BW:Combined'] flags.Trigger.menu.muon = [ ChainProp(name='HLT_mu20_L1MU20', groups=SingleMuonGroup), @@ -61,8 +62,8 @@ def setupMenu(flags): ] flags.Trigger.menu.combined = [ - ChainProp(name='HLT_e7_mu10_L1EM7_MU10', groups=SingleElectronGroup), - ChainProp(name='HLT_e7_mu12_L1EM7_MU10', groups=SingleElectronGroup) + ChainProp(name='HLT_e7_etcut_mu10_L1EM7_MU10', groups=CombinedGroup), + ChainProp(name='HLT_e7_etcut_mu12_L1EM7_MU10', groups=CombinedGroup) ] if __name__ == "__main__": -- GitLab