diff --git a/CMakeLists.txt b/CMakeLists.txt index 34a68d0561ee9f079d06bb957c50d67b371313d4..0bf441d138fdbaa40a4b0996bd1a832c111cdaa8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ include(GetProjectVersion) # === Add the subdirectories with the tests add_subdirectory(FSR_lumi) add_subdirectory(FSR_decodingKeys) +add_subdirectory(FSR_decodingKeys_data) add_subdirectory(Moore+DaVinci) add_subdirectory(Tupling_default) add_subdirectory(Tupling_veloSP) diff --git a/FSR_decodingKeys/CMakeLists.txt b/FSR_decodingKeys/CMakeLists.txt index 8c5a42b9f8b18f73ae79af6588643ab5e8e8653b..735d3e575336dfc0d41bef0a6cbb8f615557beed 100644 --- a/FSR_decodingKeys/CMakeLists.txt +++ b/FSR_decodingKeys/CMakeLists.txt @@ -52,7 +52,7 @@ if(idx EQUAL -1) # Validate decoding Keys add_test(NAME FSR_decodingKeys.WriteDecKeystoFSR.ValidateDecKeys COMMAND ${util_dir}/logscript.sh FSR_decodingKeys.ValidateDecKeys.stdout.log FSR_decodingKeys.ValidateDecKeys.stderr.log - ${Moore_run} python ${src_dir}/validate_fsr.py spruce_test_lines_production.dst ${src_dir}/expected.json "HltANNSvc.DecodingKeys") + ${Moore_run} python ${src_dir}/validate_fsr.py spruce_test_lines_production.dst ${src_dir}/expected.json "ann") set_property(TEST FSR_decodingKeys.WriteDecKeystoFSR.ValidateDecKeys APPEND PROPERTY DEPENDS FSR_decodingKeys.WriteDecKeystoFSR.RunSpruce) diff --git a/FSR_decodingKeys/expected.json b/FSR_decodingKeys/expected.json index 81a7a0d68c2731300a5b9f88fe99d461799146b9..ae6bce01b65e5dde43acdf0b6b1785e5ae10c6aa 100644 --- a/FSR_decodingKeys/expected.json +++ b/FSR_decodingKeys/expected.json @@ -1,4 +1,4 @@ -{"HltANNSvc.DecodingKeys":{"ann": +{"ann": {"0x58cf95d7": {"PackedObjectLocations": {"1": "/Event/HLT2/Hlt2Topo2Body/Particles", @@ -51,7 +51,7 @@ "6": "/Event/Spruce/HLT2/Rec/ProtoP/Long", "7": "/Event/Spruce/HLT2/Rec/ProtoP/Neutrals", "8": "/Event/Spruce/HLT2/Rec/ProtoP/Upstream", - "9": "/Event/Spruce/HLT2/Rec/Summary"}, "version": "0"}}}, + "9": "/Event/Spruce/HLT2/Rec/Summary"}, "version": "0"}}, "application_options": {"append_decoding_keys_to_output_manifest": true, "auditors": [], @@ -100,4 +100,4 @@ "write_decoding_keys_to_git": true, "write_options_to_fsr": true, "xml_file_catalog": null, - "xml_summary_file": null}} \ No newline at end of file + "xml_summary_file": null}} diff --git a/FSR_decodingKeys/validate_fsr.py b/FSR_decodingKeys/validate_fsr.py index 6c4a08adbeff9c64e885cf43794e5ea9c5b1947c..80696a3fd56cc783e4bf3c91a661c612246b7592 100644 --- a/FSR_decodingKeys/validate_fsr.py +++ b/FSR_decodingKeys/validate_fsr.py @@ -19,10 +19,11 @@ EXPECTEDKEYSNAME = f"expected.json" def check_keys(dst_file, expected_file, item): # Load the JSON data if not os.path.isfile(dst_file): - raise Exception("DST File: {dst_file} does not exist!") + raise Exception(f"DST File: {dst_file} does not exist!") if not os.path.isfile(expected_file): - raise Exception("Expected json: {expected_file} does not exist!") + raise Exception(f"Expected json: {expected_file} does not exist!") + print(dst_file) with ROOT.TFile.Open(dst_file) as f_dst: fsr = json.loads(str(f_dst.FileSummaryRecord)) print("Found these options in FSR: \n", json.dumps(fsr[item])) diff --git a/FSR_decodingKeys_data/CMakeLists.txt b/FSR_decodingKeys_data/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..392978dd42857d78f75505e481e344638c965a89 --- /dev/null +++ b/FSR_decodingKeys_data/CMakeLists.txt @@ -0,0 +1,46 @@ +############################################################################### +# (c) Copyright 2024 CERN for the benefit of the LHCb Collaboration # +# # +# This software is distributed under the terms of the GNU General Public # +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". # +# # +# In applying this licence, CERN does not waive the privileges and immunities # +# granted to it by virtue of its status as an Intergovernmental Organization # +# or submit itself to any jurisdiction. # +############################################################################### +# Test to check that the decoding Keys from Sprucing and Tupling are propagated correctly to the FSR (FileSummaryRecord) +# Creating a dedicated test for each steps of the chain. +# All tests are split in two steps: +# Run: via 'logscript.sh' script for running the test and dumping the related stdout and stderr +# Validate: via 'validator.sh' script for checking that the stderr file exists and is empty. +# +# From stack run with "make fast/LHCbIntegrationTests/test ARGS='-R FSR_decodingKeys_data'" + +get_project_version(Moore) +get_project_version(DaVinci) + +set(src_dir ${CMAKE_CURRENT_SOURCE_DIR}) +set(util_dir ${src_dir}/../Utilities/) +set(validation_dir ${src_dir}/../FSR_decodingKeys/) + +string(FIND "$ENV{BINARY_TAG}" "detdesc" idx) +if(idx EQUAL -1) + + # Run and validate Sprucing + add_test(NAME FSR_decodingKeys_data.RunSpruce + COMMAND ${util_dir}/logscript.sh FSR_decodingKeys_data.RunSpruce.stdout.log FSR_decodingKeys_data.RunSpruce.stderr.log + ${Moore_run} lbexec Hlt2Conf.sprucing_settings.Sprucing_2022_1_production:excl_spruce_production ${util_dir}/jobFSR_options.yaml+${src_dir}/options_moore_spruce.yaml) + set_property(TEST FSR_decodingKeys_data.RunSpruce PROPERTY TIMEOUT 3600) + + add_test(NAME FSR_decodingKeys_data.ValidateSpruce + COMMAND ${util_dir}/validator.sh FSR_decodingKeys_data.RunSpruce.stderr.log) + set_property(TEST FSR_decodingKeys_data.ValidateSpruce + APPEND PROPERTY DEPENDS FSR_decodingKeys_data.RunSpruce) + + add_test(NAME FSR_decodingKeys_data.RunTuple + COMMAND ${util_dir}/logscript.sh FSR_decodingKeys_data.RunTuple.stdout.log FSR_decodingKeys_data.RunTuple..stderr.log + ${DaVinci_run} lbexec DaVinciTests.option_davinci_tupling_with_lumi_from_spruce:main ${util_dir}/jobFSR_options.yaml+${src_dir}/options_davinci_tupling.yaml) + set_property(TEST FSR_decodingKeys_data.RunTuple + APPEND PROPERTY DEPENDS FSR_decodingKeys_data.RunSpruce) + +endif() diff --git a/FSR_decodingKeys_data/README.md b/FSR_decodingKeys_data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..91759083a10c48ad22e0e7e947a18092fdebc6b6 --- /dev/null +++ b/FSR_decodingKeys_data/README.md @@ -0,0 +1,16 @@ +Test to check propagation and usage of the Decoding Keys in FileSummaryRecord (FSR) +----------------------------------------------------------------------------------- + +Test to validate that the decoding keys from the data spruced sample are propagated correctly to the FileSummaryRecord (FSR). +The FSR is then used in the DaVinci to configure the decoding keys instead of the usual `input_manifest_file`. +All the test commands are reported in the `FSR_decodingKeys_data/CMakeLists.txt` file. + +The test chain consists of the following steps: + +1) Start the test chain running Spruce on data: + * **Function**: `Hlt2Conf.sprucing_settings.Sprucing_2022_1_production:excl_spruce_production` + * **Option file**: `Utilities/jobFSR_options.yaml+FSR_decodingKeys_data/options_moore_spruce.yaml)` +2) Validate that Keys read from FSR produced in Spruce job are identical to `FSR_decodingKeys/expected_keys.json` which were obtained from running the exact same job locally +3) Run the DaVinci on the spruced output: + * **Function**: `DaVinciTests.option_davinci_tupling_with_lumi_from_spruce:main ` + * **Option file**: `Utilities/jobFSR_options.yaml+FSR_decodingKeys_data/options_davinci_tupling` diff --git a/FSR_decodingKeys_data/options_davinci_tupling.yaml b/FSR_decodingKeys_data/options_davinci_tupling.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d79a1c7e2e749036ba422265af3a42cd798815ed --- /dev/null +++ b/FSR_decodingKeys_data/options_davinci_tupling.yaml @@ -0,0 +1,7 @@ +input_files: +- spruce_2or3bodytopo_run3fsrs.b2oc.dst +input_stream: b2oc +ntuple_file: "tuple_run3fsrs_ntp.root" +output_file: "tuple_run3fsrs_ntp.root" +histo_file: "tuple_run3fsrs_his.root" +write_fsr: True \ No newline at end of file diff --git a/FSR_decodingKeys_data/options_moore_spruce.yaml b/FSR_decodingKeys_data/options_moore_spruce.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d1a3afc63eb605a6abc8cd1c45e7378beb0c9ac1 --- /dev/null +++ b/FSR_decodingKeys_data/options_moore_spruce.yaml @@ -0,0 +1,18 @@ +process : Spruce +input_process: Hlt2 + +input_files : [ + 'mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/data/2022_1_FULL_255620_00150001_0005.raw', + 'mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/data/2022_1_FULL_255949_00150003_0000.raw' +] +input_type: RAW +evt_max: 30000 +print_freq: 3000 +persistreco_version: 0.0 + +write_options_to_fsr: True + +histo_file : 'spurce_2or3bodytopo_run3fsrs.root' +output_file : 'spruce_2or3bodytopo_run3fsrs.{stream}.dst' +output_type: ROOT +write_decoding_keys_to_git: False \ No newline at end of file diff --git a/doc/guide/existing_chains.rst b/doc/guide/existing_chains.rst index 0cf28dcf0dc2a39f72880d389257002e69c98a70..2e4f57d9c6d1a8720a619bf85aa9426040e2d308 100644 --- a/doc/guide/existing_chains.rst +++ b/doc/guide/existing_chains.rst @@ -33,4 +33,10 @@ Or, if you are working on an LHCb stack: .. include:: ../../Tupling_neutrals/README.md -.. include:: ../../Tupling_DTF_pvs/README.md \ No newline at end of file +.. include:: ../../Tupling_DTF_pvs/README.md + + +.. include:: ../../Tupling_decodingKeys/README.md + + +.. include:: ../../Tupling_decodingKeys_data/README.md