From 5baf0e0f7181c1b3bc270cbfc46c2ee406715e47 Mon Sep 17 00:00:00 2001 From: Nicole Skidmore <nicola.skidmore@cern.ch> Date: Mon, 7 Oct 2024 13:04:53 +0200 Subject: [PATCH] Merge branch 'SpruceMC' into '2024-patches' Sprucing MC : FULL HLT2 line filter applied See merge request lhcb/Moore!3900 (cherry picked from commit 14dc87ab92f26d62fcd4774f68ccf3645875a91c) 49bc56ad FULL filter applied b3c4c88a FULL filter applied 9d38b9a6 fix e26dc061 Fixed formatting a4a73973 Removing unused code. Co-authored-by: Miroslav Saur <miroslav.saur@cern.ch> --- Hlt/Moore/python/Moore/production.py | 62 ++++++++++++++++--- Hlt/Moore/python/Moore/stream_writers.py | 2 - .../qmtest/test_lbexec_spruce_pp_2024_mc.qmt | 2 +- .../test_lbexec_spruce_pp_default_mc.qmt | 8 +-- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/Hlt/Moore/python/Moore/production.py b/Hlt/Moore/python/Moore/production.py index 874d3f00170..0c876e62e28 100644 --- a/Hlt/Moore/python/Moore/production.py +++ b/Hlt/Moore/python/Moore/production.py @@ -293,9 +293,14 @@ def _spruce(options, require_deployed_trigger_key=False, flagging=False): - from Moore import run_moore + from Moore import run_moore, moore_control_flow from PyConf.application import metainfo_repos, retrieve_encoding_dictionary from Moore.persistence.hlt2_tistos import list_of_full_stream_lines + from PyConf.Algorithms import VoidFilter + import Functors as F + from PyConf.reading import get_decreports + from PyConf.control_flow import CompositeNode, NodeLogic + from PyConf.application import (configure_input, configure) if require_deployed_trigger_key: metainfo_repos.global_bind(repos=[]) # only use repos on cvmfs @@ -309,15 +314,41 @@ def _spruce(options, return _line_maker( options, make_lines, lines_regex, flagging, process="Spruce") - with list_of_full_stream_lines.bind( - lines=full_hlt2_lines), _apply_track_binds(): - public_tools = [ - trackMasterExtrapolator_with_simplified_geom(), - stateProvider_with_simplified_geom() - ] - config = run_moore(options, _my_line_maker, public_tools) - if options.simulation: + + config = configure_input(options) + ## Implement a filter such that exclusive Sprucing only runs on FULL stream HLT2 lines + ## Add this filter before `moore_control_node` using `LAZY_AND` logic + + line_regex = "|".join( + line for line in [line + "Decision" for line in full_hlt2_lines]) + full_stream_filter = VoidFilter( + name='full_stream_filter', + Cut=F.DECREPORTS_RE_FILTER( + Regex=line_regex, DecReports=get_decreports('Hlt2'))) + + with list_of_full_stream_lines.bind( + lines=full_hlt2_lines), _apply_track_binds(): + + public_tools = [ + trackMasterExtrapolator_with_simplified_geom(), + stateProvider_with_simplified_geom() + ] + + moore_control_node = moore_control_flow( + options, _my_line_maker(), process="spruce") + + moore_control_node = CompositeNode( + 'MC_spruce_control_node', + combine_logic=NodeLogic.LAZY_AND, + children=[full_stream_filter] + [moore_control_node], + force_order=True) + + config.update( + configure( + options, moore_control_node, public_tools=public_tools)) + + ## Remove any line prescales from Gaudi.Configuration import allConfigurables from Configurables import DeterministicPrescaler for n, c in allConfigurables.items(): @@ -325,4 +356,15 @@ def _spruce(options, DeterministicPrescaler) and n.endswith("_Prescaler"): c.AcceptFraction = 1.0 - return config + return config + + else: + with list_of_full_stream_lines.bind( + lines=full_hlt2_lines), _apply_track_binds(): + public_tools = [ + trackMasterExtrapolator_with_simplified_geom(), + stateProvider_with_simplified_geom() + ] + config = run_moore(options, _my_line_maker, public_tools) + + return config diff --git a/Hlt/Moore/python/Moore/stream_writers.py b/Hlt/Moore/python/Moore/stream_writers.py index 68cb8363e0e..e7fd595046d 100644 --- a/Hlt/Moore/python/Moore/stream_writers.py +++ b/Hlt/Moore/python/Moore/stream_writers.py @@ -321,8 +321,6 @@ def write_spruce(options, stream, hlt_raw_banks, dst_data, dec_reports): # Physics lines have a DstData bank rb_to_persist += dst_data - if options.simulation: - rb_to_persist += [get_bank_by_sourceid("Hlt2", "DstData")] # report raw banks from Hlt1 and Hlt2 for banktype in HLT1_REPORT_RAW_BANK_TYPES: diff --git a/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_2024_mc.qmt b/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_2024_mc.qmt index 1cee3fe331d..7c7ead5c60c 100644 --- a/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_2024_mc.qmt +++ b/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_2024_mc.qmt @@ -44,7 +44,7 @@ assert len(out_files) == 1, out_files in_file = in_files[0] out_file = out_files[0] print(f"sizes : {int(in_file.text)}, {int(out_file.text)}") -assert int(in_file.text) == int(out_file.text), "All events should be retained from HLT2 step" +assert 0.5*int(in_file.text) <= int(out_file.text), "Would expect at least half events retained from HLT2 step (given total event share of FULL stream lines)" assert out_file.attrib["name"] == "PFN:spruce_pp_2024_output.dst" # Ensure the pool XML was updated to contain the new output file diff --git a/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_default_mc.qmt b/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_default_mc.qmt index 73e00ec9f7f..4294578beb7 100644 --- a/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_default_mc.qmt +++ b/Hlt/Moore/tests/qmtest/test_lbexec_spruce_pp_default_mc.qmt @@ -8,11 +8,11 @@ In applying this licence, CERN does not waive the privileges and immunities granted to it by virtue of its status as an Intergovernmental Organization or submit itself to any jurisdiction. + + Test that Spruce can run from LHCbDirac and that it produces the expected outputs. Dry-run only! --> -<!-- -Test that Spruce can run from LHCbDirac and that it produces the expected outputs. -Dry-run only! ---> + + <extension class="GaudiTest.GaudiExeTest" kind="test"> <argument name="prerequisites"><set> <tuple><text>test_lbexec_hlt2_pp_2024_mc</text><enumeral>PASS</enumeral></tuple> -- GitLab