From 1e9fef6f9927d96313ba0fff08f5bc440b9b5986 Mon Sep 17 00:00:00 2001
From: Miroslav Saur <miroslav.saur@cern.ch>
Date: Tue, 1 Aug 2023 16:54:44 +0200
Subject: [PATCH] adding additional test for RawBanks SP
---
Hlt/Hlt2Conf/CMakeLists.txt | 1 -
.../options/hlt2_persistreco_rawbanksp.py | 175 ------------------
.../options/streaming/hlt2_test_streaming.py | 9 +-
.../python/Hlt2Conf/Sprucing_tests.py | 20 +-
.../python/Hlt2Conf/lines/test/spruce_test.py | 23 +++
.../tests/options/streaming/stream_check.py | 53 +++++-
...est_persistreco_rawbanksp_mdf_fromfile.qmt | 37 ----
Hlt/Moore/python/Moore/config.py | 9 +-
8 files changed, 107 insertions(+), 220 deletions(-)
delete mode 100644 Hlt/Hlt2Conf/options/hlt2_persistreco_rawbanksp.py
delete mode 100644 Hlt/Hlt2Conf/tests/qmtest/test_persistreco_rawbanksp_mdf_fromfile.qmt
diff --git a/Hlt/Hlt2Conf/CMakeLists.txt b/Hlt/Hlt2Conf/CMakeLists.txt
index 5edbbcfec0d..61dad0ef3b9 100644
--- a/Hlt/Hlt2Conf/CMakeLists.txt
+++ b/Hlt/Hlt2Conf/CMakeLists.txt
@@ -70,7 +70,6 @@ if(BUILD_TESTING AND USE_DD4HEP)
Hlt2Conf.test_hlt2_passthrough_persistreco_output
Hlt2Conf.test_hlt2_trackefflines
Hlt2Conf.test_hlt2_with_hlt1_decisions
- Hlt2Conf.test_persistreco_rawbanksp_mdf_fromfile
Hlt2Conf.test_persistreco_realtime
Hlt2Conf.test_persistreco_output_realtime
Hlt2Conf.sprucing.test_spruce_example_realtime
diff --git a/Hlt/Hlt2Conf/options/hlt2_persistreco_rawbanksp.py b/Hlt/Hlt2Conf/options/hlt2_persistreco_rawbanksp.py
deleted file mode 100644
index a784c3f551a..00000000000
--- a/Hlt/Hlt2Conf/options/hlt2_persistreco_rawbanksp.py
+++ /dev/null
@@ -1,175 +0,0 @@
-###############################################################################
-# (c) Copyright 2019-2021 CERN for the benefit of the LHCb Collaboration #
-# #
-# This software is distributed under the terms of the GNU General Public #
-# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
-# #
-# In applying this licence, CERN does not waive the privileges and immunities #
-# granted to it by virtue of its status as an Intergovernmental Organization #
-# or submit itself to any jurisdiction. #
-###############################################################################
-"""Test persistreco when using reco from file. Produces hlt2_testpersistreco.dst
-
-Run like any other options file:
-
- ./Moore/run gaudirun.py tests/options/hlt_dst_output.py hlt2_persistreco.py
- or
- ./Moore/run gaudirun.py tests/options/hlt_mdf_output.py hlt2_persistreco.py
-"""
-from Moore import options, run_moore
-from Moore.lines import Hlt2Line
-from RecoConf.global_tools import stateProvider_with_simplified_geom
-from RecoConf.reconstruction_objects import reconstruction
-from RecoConf.reconstruction_objects import upfront_reconstruction, make_pvs as make_pvs
-from RecoConf.calorimeter_reconstruction import make_digits
-from RecoConf.event_filters import require_pvs
-from Hlt2Conf.standard_particles import make_has_rich_long_kaons
-from GaudiKernel.SystemOfUnits import GeV, MeV, mm, micrometer as um
-import Functors as F
-from Hlt2Conf.algorithms_thor import ParticleCombiner, ParticleFilter
-from Functors import require_all
-
-from pprint import pprint
-
-### Define input files and basic options ###
-
-input_files = [
- # evttype 27163002
- 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00070317/0000/00070317_00000033_2.ldst',
- 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00070317/0000/00070317_00000038_2.ldst',
- 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00070317/0000/00070317_00000040_2.ldst',
- 'root://eoslhcb.cern.ch//eos/lhcb/grid/prod/lhcb/MC/Upgrade/LDST/00070317/0000/00070317_00000047_2.ldst'
-]
-
-options.input_files = input_files
-options.input_type = 'ROOT'
-options.input_raw_format = 4.3
-options.persistreco_version = 0.0
-# When running from Upgrade MC, must use the post-juggling locations of the raw
-# event
-make_digits.global_bind(calo_raw_bank=False)
-
-options.evt_max = 1000
-options.simulation = True
-options.data_type = 'Upgrade'
-options.dddb_tag = 'dddb-20171126'
-options.conddb_tag = 'sim-20171127-vc-md100'
-options.geometry_version = 'trunk'
-options.conditions_version = 'master'
-
-#following options are set in tests/options/hlt_dst_output.py or hlt_mdf_out.py
-options.output_file = 'hlt2_test_persistreco_rawbanksp.{stream}.mdf'
-options.output_type = 'MDF'
-
-# This options file is used with different output types; subsequent tests
-# require TCKs for each case
-if options.output_type == "ROOT":
- options.output_manifest_file = "hlt2_persistreco_rawbanksp_root_output.tck.json"
-elif options.output_type == "MDF":
- options.output_manifest_file = "hlt2_persistreco_rawbanksp_mdf_output.tck.json"
-
-###Define 2 orthoganol lines (PIDK cut) with and without persistreco
-
-
-def make_charm_kaons(pidk_cut):
- """Return kaons maker for D0 decay selection.
- """
- pvs = make_pvs()
- code = require_all(
- F.MINIPCUT(IPCut=60 * um, Vertices=pvs), F.PT > 800 * MeV,
- F.P > 5 * GeV, pidk_cut)
- return ParticleFilter(make_has_rich_long_kaons(), F.FILTER(code))
-
-
-def make_dzeros(particle1, particle2, name, descriptor):
- """Return D0 maker with selection tailored for two-body hadronic final
- states.
-
- Args:
- particles (list of DataHandles): Input particles used in the
- combination.
- name (string): Combiner name for identification.
- descriptor (string): Decay descriptor to be reconstructed.
- """
- pvs = make_pvs()
- combination_cut = require_all(F.MASS > 1685 * MeV, F.MASS < 2045 * MeV,
- F.PT > 2 * GeV,
- F.MAX(F.PT) > 1000 * MeV,
- F.MAXDOCACUT(0.1 * mm))
- vertex_cut = require_all(F.MASS > 1715 * MeV, F.MASS < 2015 * MeV,
- F.CHI2DOF < 10.,
- F.BPVFDCHI2(pvs) > 25.,
- F.BPVDIRA(pvs) > 0.99985)
-
- return ParticleCombiner([particle1, particle2],
- name=name,
- DecayDescriptor=descriptor,
- CombinationCut=combination_cut,
- CompositeCut=vertex_cut)
-
-
-def test_persistreco_raw_bank_sp_line(name='Hlt2_test_persistreco_raw_bank_sp',
- prescale=1,
- persistreco=False):
- kaons = make_charm_kaons(pidk_cut=F.PID_K < 15)
- dzeros = make_dzeros(
- particle1=kaons,
- particle2=kaons,
- name='Charm_D0ToHH_D0ToKmKp_persistreco',
- descriptor='D0 -> K- K+')
- return Hlt2Line(
- name=name,
- algs=upfront_reconstruction() + [require_pvs(make_pvs())] + [dzeros],
- prescale=prescale,
- persistreco=persistreco,
- raw_banks=[
- 'VP', 'UT', 'FTCluster', 'Rich', 'EcalPacked', 'HcalPacked',
- 'Muon', 'VPRetinaCluster', 'Calo', 'Plume'
- ])
-
-
-def test_raw_bank_sp_line(name='Hlt2_test_raw_bank_sp',
- prescale=1,
- persistreco=False):
- kaons = make_charm_kaons(pidk_cut=F.PID_K > 15)
- dzeros = make_dzeros(
- particle1=kaons,
- particle2=kaons,
- name='Charm_D0ToHH_D0ToKmKp_nopersistreco',
- descriptor='D0 -> K- K+')
- return Hlt2Line(
- name=name,
- algs=upfront_reconstruction() + [require_pvs(make_pvs())] + [dzeros],
- prescale=prescale,
- persistreco=persistreco,
- raw_banks=['VPRetinaCluster'])
-
-
-def make_streams():
- linedict = dict(
- test_stream_A=[test_persistreco_raw_bank_sp_line()],
- test_stream_B=[test_raw_bank_sp_line()])
- pprint(linedict)
- return linedict
-
-
-# Modify stream dictionaries for testing purposes
-import Moore.streams_hlt2
-Moore.streams_hlt2.DETECTOR_RAW_BANK_TYPES_PER_STREAM = {
- "test_stream_A": [
- 'ODIN', 'VP', 'UT', 'FTCluster', 'Rich', 'EcalPacked', 'HcalPacked',
- 'Muon', 'VPRetinaCluster', 'Calo', 'Plume'
- ],
- "test_stream_B": [
- 'VPRetinaCluster',
- ]
-}
-
-Moore.streams_hlt2.stream_bits = dict(test_stream_A=85, test_stream_B=90)
-
-print("Moore.streams_hlt2.DETECTOR_RAW_BANK_TYPES_PER_STREAM")
-pprint(Moore.streams_hlt2.DETECTOR_RAW_BANK_TYPES_PER_STREAM)
-
-public_tools = [stateProvider_with_simplified_geom()]
-with reconstruction.bind(from_file=False):
- config = run_moore(options, make_streams, public_tools)
diff --git a/Hlt/Hlt2Conf/options/streaming/hlt2_test_streaming.py b/Hlt/Hlt2Conf/options/streaming/hlt2_test_streaming.py
index 161dab0c722..35d4e8ee0ba 100644
--- a/Hlt/Hlt2Conf/options/streaming/hlt2_test_streaming.py
+++ b/Hlt/Hlt2Conf/options/streaming/hlt2_test_streaming.py
@@ -108,7 +108,8 @@ def test_B_line(name='Hlt2_test_stream_B_line', prescale=1):
return Hlt2Line(
name=name,
algs=upfront_reconstruction() + [require_pvs(make_pvs())] + [dzeros],
- prescale=prescale)
+ prescale=prescale,
+ raw_banks=['FTCluster'])
def make_streams():
@@ -125,9 +126,9 @@ Moore.streams_hlt2.DETECTOR_RAW_BANK_TYPES_PER_STREAM = {
'ODIN', 'VP', 'UT', 'FTCluster', 'Rich', 'EcalPacked', 'HcalPacked',
'Muon'
],
- "test_stream_B":
- ['ODIN', 'VP', 'UT', 'FTCluster', 'EcalPacked', 'HcalPacked',
- 'Muon'] # Remove RICH raw bank
+ "test_stream_B": [
+ 'ODIN', 'VP', 'UT', 'EcalPacked', 'HcalPacked', 'Muon'
+ ] # Remove RICH raw bank and FTCluster bank which is then requested using raw banks SP functionality
}
Moore.streams_hlt2.stream_bits = dict(test_stream_A=85, test_stream_B=90)
diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py
index 8ab45dcb2d5..4b6376b0903 100644
--- a/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py
+++ b/Hlt/Hlt2Conf/python/Hlt2Conf/Sprucing_tests.py
@@ -12,7 +12,7 @@ from Moore import Options, run_moore
from RecoConf.global_tools import stateProvider_with_simplified_geom
from RecoConf.reconstruction_objects import upfront_reconstruction
from Hlt2Conf.lines import sprucing_lines
-from Hlt2Conf.lines.test.spruce_test import Test_sprucing_line, Test_extraoutputs_sprucing_line, Test_persistreco_sprucing_line
+from Hlt2Conf.lines.test.spruce_test import Test_sprucing_line, Test_extraoutputs_sprucing_line, Test_persistreco_sprucing_line, Test_sprucing_rawbank_sp_line
from Hlt2Conf.lines.b_to_open_charm import b_to_dh
from Moore.lines import PassLine, SpruceLine
from Moore.persistence import persist_line_outputs
@@ -168,6 +168,24 @@ def spruce_example_realtime_persistreco(options: Options):
return config
+def spruce_example_realtime_rawbank_sp(options: Options):
+ """Test Sprucing line with rawbank selective persistency on output of topo{2,3} persistreco hlt2 lines (original reco real time). Produces spruce_realtimereco_persistreco.dst
+ """
+
+ def make_streams():
+ linedict = dict(stream_A=[
+ Test_sprucing_rawbank_sp_line(
+ name="Spruce_rawbank_sp_stream_A_line")
+ ])
+ pprint(linedict)
+ return linedict
+
+ streams_spruce.stream_banks = {"stream_A": ['ODIN']}
+
+ config = run_moore(options, make_streams, public_tools)
+ return config
+
+
def spruce_all_lines_realtime_test_old_json(options: Options):
"""Test running all Sprucing lines on topo{2,3} persistreco hlt2 output (use real time reco). Produces spruce_all_lines_realtimereco_newPacking.dst
diff --git a/Hlt/Hlt2Conf/python/Hlt2Conf/lines/test/spruce_test.py b/Hlt/Hlt2Conf/python/Hlt2Conf/lines/test/spruce_test.py
index 16485792dad..042b1f54c64 100644
--- a/Hlt/Hlt2Conf/python/Hlt2Conf/lines/test/spruce_test.py
+++ b/Hlt/Hlt2Conf/python/Hlt2Conf/lines/test/spruce_test.py
@@ -97,3 +97,26 @@ def Test_persistreco_sprucing_line(name='SpruceTest_PersistReco', prescale=1):
prescale=prescale,
persistreco=True,
algs=prefilters.b2oc_prefilters() + [line_alg])
+
+
+### line for testing selective persistency of detector raw banks
+### currently it is asking for one addiitonal 'FTCluster' bank
+def Test_sprucing_rawbank_sp_line(name='SpruceTest_rawbank_sp', prescale=1):
+ kaon = basic_builder.make_tight_kaons(k_pidk_min=None)
+ d = d_builder.make_dsplus_to_kpkmpip(pi_pidk_max=None, k_pidk_min=None)
+
+ line_alg = b_builder.make_b2x(
+ particles=[d, kaon],
+ descriptors=['[B0 -> D_s- K+]cc'],
+ am_min=5000 * MeV,
+ am_max=7000 * MeV,
+ am_min_vtx=5000 * MeV,
+ am_max_vtx=7000 * MeV,
+ sum_pt_min=6 * GeV,
+ bpvltime_min=0.3 * picosecond)
+
+ return SpruceLine(
+ name=name,
+ prescale=prescale,
+ algs=prefilters.b2oc_prefilters() + [line_alg],
+ raw_banks=['FTCluster'])
diff --git a/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py b/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py
index 28561ea6c60..562ffc59cc8 100644
--- a/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py
+++ b/Hlt/Hlt2Conf/tests/options/streaming/stream_check.py
@@ -1,5 +1,5 @@
###############################################################################
-# (c) Copyright 2019-2021 CERN for the benefit of the LHCb Collaboration #
+# (c) Copyright 2019-2023 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
@@ -10,6 +10,8 @@
###############################################################################
"""Test if streaming functionality for Hlt2 or Sprucing works as expected.
+ For testing HLT2 streaming this test is using the output of Hlt/Hlt2Conf/tests/qmtest/streaming.qms/test_hlt2_streaming.qmt
+ For testing Sprucing streaming this test is using the output of Hlt/Hlt2Conf/tests/qmtest/streaming.qms/test_sprucing_streaming.qmt
"""
import argparse
@@ -110,14 +112,63 @@ while True:
if not TES['/Event']:
break
+ # HLT2 reports: 'HltDecReports', 'HltLumiSummary', 'HltRoutingBits'
+ # In the test stream A: ODIN', 'VP', 'UT', 'FTCluster', 'Rich', 'EcalPacked', 'HcalPacked', 'Muon'
+ # In the test stream B: ODIN', 'VP', 'UT', 'FTCluster', 'EcalPacked', 'HcalPacked', 'Muon' # Rich removed, FTCluster added via RawBank SP
+
print("Looking in RawEvent ", raweventloc)
# Check that RawBank configuration per stream is working
Rich_banks = TES[raweventloc].banks(LHCb.RawBank.Rich)
+ FT_banks = TES[raweventloc].banks(LHCb.RawBank.FTCluster)
print("Rich_banks ", Rich_banks.size())
if "A" in args.stream and Rich_banks.size() == 0:
error("Expected RICH rawbanks in this stream")
if "B" in args.stream and Rich_banks.size() != 0:
error("Expected no RICH rawbanks in this stream")
+ if "B" in args.stream and FT_banks.size() == 0:
+ error("Expected selectively persisted FT rawbanks in this stream")
+
+ # check that only expected raw banks are non-empty
+ if "A" in args.stream:
+ # RawBank IDs as per LHCb/Event/DAQEvent/include/Event/RawBank.h
+ if args.input_process == 'Hlt2':
+ saved_banks = {9, 13, 15, 16, 17, 21, 22, 53, 54, 60, 63, 64, 66}
+ elif args.input_process == 'Spruce':
+ saved_banks = {9, 13, 16, 17, 21, 22, 53, 54, 60, 63, 64, 66}
+ else:
+ error("Undefined process")
+ for i in range(0, 100):
+ if i in saved_banks:
+ if TES[raweventloc].banks(i).size() == 0:
+ error(
+ f"Bank {GP.gbl.LHCb.RawBank.typeName(i)} is expected to be non-empty but has a size of {TES[raweventloc].banks(i).size()}"
+ )
+ else:
+ if TES[raweventloc].banks(i).size() != 0:
+ error(
+ f"Bank {GP.gbl.LHCb.RawBank.typeName(i)} is expected to be empty but has a size of {TES[raweventloc].banks(i).size()}"
+ )
+
+ if "B" in args.stream:
+ # RawBank IDs as per LHCb/Event/DAQEvent/include/Event/RawBank.h
+ # Rich bank removed
+ if args.input_process == 'Hlt2':
+ saved_banks = {13, 15, 16, 17, 21, 22, 53, 54, 60, 63, 64, 66}
+ elif args.input_process == 'Spruce':
+ saved_banks = {13, 16, 17, 21, 22, 53, 54, 60, 63, 64, 66}
+ else:
+ error("Undefined process")
+ for i in range(0, 100):
+ if i in saved_banks:
+ if TES[raweventloc].banks(i).size() == 0:
+ error(
+ f"Bank {GP.gbl.LHCb.RawBank.typeName(i)} is expected to be non-empty but has a size {TES[raweventloc].banks(i).size()}"
+ )
+ else:
+ if TES[raweventloc].banks(i).size() != 0:
+ error(
+ f"Bank {GP.gbl.LHCb.RawBank.typeName(i)} is expected to be empty but has a size {TES[raweventloc].banks(i).size()}"
+ )
# Check that routing bits per stream are working
if args.input_process == "Hlt2":
diff --git a/Hlt/Hlt2Conf/tests/qmtest/test_persistreco_rawbanksp_mdf_fromfile.qmt b/Hlt/Hlt2Conf/tests/qmtest/test_persistreco_rawbanksp_mdf_fromfile.qmt
deleted file mode 100644
index ba8a222d21f..00000000000
--- a/Hlt/Hlt2Conf/tests/qmtest/test_persistreco_rawbanksp_mdf_fromfile.qmt
+++ /dev/null
@@ -1,37 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE extension PUBLIC '-//QM/2.3/Extension//EN' 'http://www.codesourcery.com/qm/dtds/2.3/-//qm/2.3/extension//en.dtd'>
-<!--
- (c) Copyright 2021 CERN for the benefit of the LHCb Collaboration
-
- This software is distributed under the terms of the GNU General Public
- Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".
-
- In applying this licence, CERN does not waive the privileges and immunities
- granted to it by virtue of its status as an Intergovernmental Organization
- or submit itself to any jurisdiction.
--->
-<!--
-Check persist reco using reco from file.
--->
-<extension class="GaudiTest.GaudiExeTest" kind="test">
-<argument name="program"><text>gaudirun.py</text></argument>
-<argument name="timeout"><integer>3600</integer></argument>
-<argument name="args"><set>
- <text>$HLT2CONFROOT/tests/options/hlt2_mdf_output.py</text>
- <text>$MOOREROOT/options/muon_geometry_v2.py</text>
- <text>$HLT2CONFROOT/options/hlt2_persistreco_rawbanksp.py</text>
- <text>--output=test_persistreco_mdf_fromfile.opts.py</text>
- <text>--all-opt</text>
-</set></argument>
-<argument name="options"><text>
-from Configurables import HiveDataBrokerSvc
-HiveDataBrokerSvc().OutputLevel = 5
-</text></argument>
-<argument name="use_temp_dir"><enumeral>true</enumeral></argument>
-<argument name="validator"><text>
-
-from Moore.qmtest.exclusions import remove_known_warnings
-countErrorLines({"FATAL": 0, "ERROR": 0},
- stdout=remove_known_warnings(stdout))
-
-</text></argument>
-</extension>
diff --git a/Hlt/Moore/python/Moore/config.py b/Hlt/Moore/python/Moore/config.py
index a6fd5fa5603..0efcba92888 100644
--- a/Hlt/Moore/python/Moore/config.py
+++ b/Hlt/Moore/python/Moore/config.py
@@ -537,6 +537,11 @@ def moore_control_flow(options, streams, process, analytics=False):
banks_from_previous_step = _unique(banks_from_previous_step)
+ # Assert condition to filter out HLT2 streaming when using DST output as writing multiple DST files is not supported for now
+ assert not (len(streams.items()) > 1 and process == "hlt2"
+ and options.output_type == "ROOT"
+ ), "This configuration is not supported at this time"
+
for stream, stream_lines in streams.items():
new_raw_banks = list(new_report_raw_banks.values())
@@ -567,7 +572,9 @@ def moore_control_flow(options, streams, process, analytics=False):
],
dec_reports=dec_reports,
extra_locations=extra_outputs)
- stream_writers_setup += pre_algs
+ for alg in pre_algs:
+ if alg not in stream_writers_setup:
+ stream_writers_setup.append(alg)
streamFilter = VoidFilter(
# no explicit name to allow sharing of instances between TURBO and TURBORAW
--
GitLab