[RTADPA BW Tests] New Sprucing bandwidth test using latest-available HLT2 input.

cc: @rjhunter.

related to: lhcb/Moore!2763 (merged) + lhcb-core/LHCbPR2HD!270 (merged) + lhcb-core/LHCbNightlyConf!1121 (merged)

Changes:

Adds a new sprucing test, Moore_spruce_latest_bandwidth, which uses the Hlt2 nominal full-stream output which was uploaded to eos on the previous day.

Added a new script, generate_hlt2_fullstream_metadata.py, which runs only for Hlt2 nominal tests and writes the metadata.yaml based on information from the full-stream output of the test currently running. This metadata.yaml is used as an input config for the new sprucing test.

• a little safety around division in zero from the calculation of overlap_matrix, never comes up in the actual tests because every stream has a non-zero retention in 100k events, but when running test jobs it can come up and it was annoying me.

• changes to facilitate the new test + metadata.

ToDo:

• Maybe revisit the N_EVTS and calculate via a gaudipython job?
  • Wait to see if BWTesters know a way to tell the N_EVTS available in a file.
  • Else, adapt list_event_numbers to extract the N_EVTS by iterating a gaudipython job.
• Rebase after !362 (merged) was merged

Closes #12 (closed)

python/MooreTests/generate_metadata.py

@@ -17,9 +17,7 @@ import json
 from PRConfig.bandwidth_helpers import FileNameHelper
 from PRConfig.TestFileDB import test_file_db
 from MooreTests.run_bandwidth_test_jobs import parse_yaml
-
-LHCBPR_WWW_FLOC = "{fprefix}root://eoslhcb.cern.ch//eos/lhcb/storage/lhcbpr/www/UpgradeRateTest/hlt2_bw_testing__production{fsuffix}"
-HLT2CONF_WEB = "https://gitlab.cern.ch/lhcb/Moore/-/blob/master/Hlt/Hlt2Conf"
+import yaml
 
 
 def main():
@@ -40,43 +38,39 @@ def main():
     args = parser.parse_args()
     fname_helper = FileNameHelper(process="hlt2")
     config = parse_yaml(args.config)
-
-    opts = [
-        f"input_files:\n  - \'{LHCBPR_WWW_FLOC.format(fprefix='mdf:', fsuffix='__full.mdf')}\'"
+    fname_helper_args = {"stream_config": "production", "stream": "full"}
+    opts = {}
+    opts["input_files"] = [fname_helper.mdf_prwww_path(**fname_helper_args)]
+    opts["input_manifest_file"] = fname_helper.manifest_prwww_path(
+        **fname_helper_args)
+    for key in ['input_raw_format', 'velo_radial_opening', 'nu']:
+        opts[key] = config[key]
+    opts['filter_info_links'] = [
+        args.config.replace('$HLT2CONFROOT',
+                            FileNameHelper.gitlab_hlt2conf_webpage)
     ]
-    opts.append(
-        f"input_manifest_file: \'{LHCBPR_WWW_FLOC.format(fprefix='', fsuffix='.tck.json')}\'"
-    )
-    opts.append(f"input_raw_format: {config['input_raw_format']}")
-    opts.append(f"velo_radial_opening: {config['velo_radial_opening']}")
-    opts.append(f"nu: {config['nu']}")
-    opts.append(
-        f"filtering_info_links:\n  - {args.config.replace('$HLT2CONFROOT',HLT2CONF_WEB)}"
-    )
-    opts.append("input_type: 'MDF'")
-    opts.append("simulation: True")
-    opts.append("data_type: 'Upgrade'")
+    opts['input_type'] = "MDF"
+    opts['simulation'] = "True"
+    opts["data_type"] = "Upgrade"
+
     conds = test_file_db[config['testfiledb_key']].qualifiers
-    opts.append(f"dddb_tag: {conds['DDDB']}")
-    opts.append(f"conddb_tag: {conds['CondDB']}")
+    opts["dddb_tag"] = conds['DDDB']
+    opts["conddb_tag"] = conds['CondDB']
 
     # Access the
-    ifile = fname_helper.event_no_fname(
-        stream_config='production', stream='full')
+    ifile = fname_helper.event_no_fname(**fname_helper_args)
     with open(ifile, 'r') as f:
         ijson = json.load(f)  # {stream: [evt_numbers]}
         full_evts = len(ijson['full'])
 
     hlt2_evts = args.events
     input_rate = config['input_rate']
-    opts.append(f"input_rate: {input_rate * full_evts / hlt2_evts} # in kHz")
-    opts.append(f"n_evts: {full_evts}")
+    opts["input_rate"] = f"{input_rate * full_evts / hlt2_evts} # in kHz"
+    opts["n_evts"] = f"{full_evts}"
 
-    ofile = fname_helper.metadata_path(
-        stream_config="production", stream="full")
+    ofile = fname_helper.metadata_path(**fname_helper_args)
     with open(ofile, 'w') as f:
-        for opt in opts:
-            f.write(opt + '\n')
+        yaml.dump(opts, f, default_flow_style=False)
 
 
 if __name__ == "__main__":