diff --git a/python/MooreTests/generate_hlt2_fullstream_metadata.py b/python/MooreTests/generate_hlt2_fullstream_metadata.py
index c1ab5ce889eb726e2894cf2d913e19baaed66986..41eb8aada4dc178117d0e7623163ce38b1c666f5 100755
--- a/python/MooreTests/generate_hlt2_fullstream_metadata.py
+++ b/python/MooreTests/generate_hlt2_fullstream_metadata.py
@@ -68,7 +68,7 @@ def main():
opts["input_manifest_file"] = fname_helper.manifest_prwww_path(
fname_helper_args['stream_config'])
metadata_config_for_use_in_sprucing_hlt2_output_from_eos_test = fname_helper.metadata_path(
- **fname_helper_args)
+ **fname_helper_args, for_local_use=False)
with open(metadata_config_for_use_in_sprucing_hlt2_output_from_eos_test,
'w') as f:
yaml.dump(opts, f, default_flow_style=False)
@@ -81,7 +81,7 @@ def main():
local_opts["input_manifest_file"] = fname_helper.tck(
stream_config=fname_helper_args["stream_config"])
metadata_config_for_use_in_sprucing_output_locally_test = fname_helper.metadata_path(
- **fname_helper_args)
+ **fname_helper_args, for_local_use=True)
with open(metadata_config_for_use_in_sprucing_output_locally_test,
'w') as f:
yaml.dump(opts, f, default_flow_style=False)
diff --git a/python/MooreTests/generate_tistos_option_file.py b/python/MooreTests/generate_tistos_option_file.py
new file mode 100644
index 0000000000000000000000000000000000000000..31d36961b8b8fe49cd7d6a7b1ce9008a57054859
--- /dev/null
+++ b/python/MooreTests/generate_tistos_option_file.py
@@ -0,0 +1,49 @@
+###############################################################################
+# (c) Copyright 2000-2024 CERN for the benefit of the LHCb Collaboration #
+# #
+# This software is distributed under the terms of the GNU General Public #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
+# #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization #
+# or submit itself to any jurisdiction. #
+###############################################################################
+import argparse
+import json
+from PRConfig.bandwidth_helpers import FileNameHelper
+
+
+def main():
+ parser = argparse.ArgumentParser(description=main.__doc__)
+ parser.add_argument(
+ '--stream-config-location',
+ type=str,
+ required=True,
+ help=
+ "Location for stream_config, if empty prompts the usage of fixed_line_configs from Moore"
+ )
+
+ args = parser.parse_args()
+
+ hlt2_fnames = FileNameHelper('hlt2')
+ tistos_content = f'with open(\'{args.stream_config_location}\', "r") as f: lines_for_TISTOS=json.load(f)["full"]' if args.stream_config_location else 'from Hlt2Conf.sprucing_settings.fixed_line_configs import lines_for_TISTOS_BW_Aug2023 as lines_for_TISTOS'
+
+ content = [
+ 'import json',
+ tistos_content,
+ 'from Moore.persistence.hlt2_tistos import tistos_incompatible_hlt2_lines, list_of_full_stream_lines',
+ 'for line in tistos_incompatible_hlt2_lines:',
+ ' if line in lines_for_TISTOS:',
+ ' print(f"Manually removed {line} for TisTos incompatibility")',
+ ' lines_for_TISTOS.remove(line)',
+ 'list_of_full_stream_lines.global_bind(lines=lines_for_TISTOS)',
+ ]
+
+ output_fname_helper = FileNameHelper("spruce")
+ ofile = output_fname_helper.tistos_option_file()
+ with open(ofile, "w") as f:
+ f.write('\n'.join(content))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/python/MooreTests/make_bandwidth_test_page.py b/python/MooreTests/make_bandwidth_test_page.py
index 79d18be57748679cc98fab65aa664c04cbb08563..cc79f147072b391d565deffe6b6f64f8562393d6 100644
--- a/python/MooreTests/make_bandwidth_test_page.py
+++ b/python/MooreTests/make_bandwidth_test_page.py
@@ -192,7 +192,7 @@ SPRUCE_REPORT_TEMPLATE = jinja2.Template("""<p>
HLT1_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
- $$hlt1__comparison$$
+ $${{PROCESS}}__comparison$$
</ul>
""")
@@ -205,8 +205,8 @@ HLT2_ALL_RESULTS = jinja2.Template("""
<li><a href="{{BASE_PATH}}/{{PROCESS}}__rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{line_descr}}"> PersistReco and ExtraOutput for selection lines</a></li>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_wg_stream}}"> Split by working group: rates, event sizes and bandwidths of all lines</a></li>
- $$hlt2__comparison$$
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_prod_stream}}"> Split by production stream: rates, event sizes and bandwidths of all lines</a></li>
+ $${{PROCESS}}__comparison$$
</b></b>
</ul>
""")
diff --git a/python/PRConfig/bandwidth_helpers.py b/python/PRConfig/bandwidth_helpers.py
index f82bb76c1fe4f73e1a69105f984693fb0bc57b80..c25f169d34d03f4b2f351053dae1d3fd11ded9a9 100644
--- a/python/PRConfig/bandwidth_helpers.py
+++ b/python/PRConfig/bandwidth_helpers.py
@@ -74,15 +74,31 @@ class FileNameHelper(object):
def stream_config_json_path(self, stream_config, full_path=True):
fname = self._join(self._file_pfx(), "streaming",
stream_config) + ".json"
- return os.path.join(self.base_dir, self.output_subdir,
+ return os.path.join(self.base_dir, self.mdf_subdir,
+ fname) if full_path else fname
+
+ def metadata_path(self,
+ stream_config,
+ stream,
+ full_path=True,
+ for_local_use=False):
+ fname = self._join(
+ self._file_pfx(), stream_config, stream, "metadata__for_local_use"
+ if for_local_use else "metadata") + ".yaml"
+ return os.path.join(self.base_dir, self.mdf_subdir,
fname) if full_path else fname
- def metadata_path(self, stream_config, stream, full_path=True):
- fname = self._join(self._file_pfx(), stream_config, stream,
- "metadata") + ".yaml"
+ def tistos_option_file(self, full_path=True):
+ fname = self._join(self.process, "tistos_option_file") + ".py"
return os.path.join(self.base_dir, self.mdf_subdir,
fname) if full_path else fname
+ def stream_config_json_prwww_path(self, stream_config):
+ return self._prwww_path(
+ fname=self.stream_config_json_path(
+ stream_config=stream_config, full_path=False),
+ starts_mdf=False)
+
def mdf_prwww_path(self, stream_config, stream):
return self._prwww_path(
fname=self.mdf_fname_for_reading(
@@ -95,7 +111,8 @@ class FileNameHelper(object):
def metadata_prwww_path(self, stream_config, stream):
return self._prwww_path(
- fname=self.metadata_path(stream_config, stream, full_path=False),
+ fname=self.metadata_path(
+ stream_config, stream, full_path=False, for_local_use=False),
starts_mdf=False)
def input_nevts_json(self):
diff --git a/scripts/benchmark-scripts/Moore_bandwidth_test.sh b/scripts/benchmark-scripts/Moore_bandwidth_test.sh
index faa7376ef2fe7497ac3c051d327446747820e98a..02d6d8921e36eff6d47744a39410236dfb911b02 100755
--- a/scripts/benchmark-scripts/Moore_bandwidth_test.sh
+++ b/scripts/benchmark-scripts/Moore_bandwidth_test.sh
@@ -160,7 +160,7 @@ case $PROCESS in
hlt2-output-locally)
# "hlt2-output-locally" corresponds to using the locally-run full-stream output from "process=hlt2, input-data=nominal" test.
CONFIG_FILE="tmp/spruce_hlt2_output_locally_input.yaml"
- EXTRA_OPTS='-um --read-evt-max-from-config $MOOREROOT/options/muon_geometry_v2.py'
+ EXTRA_OPTS='-um --read-evt-max-from-config'
# Flag to make a top-level human-readable output page directing to the Hlt2 and Spruce output pages.
TEST_PAGE_EXTRA_OPTS='--multiple-processes'
;;
@@ -168,7 +168,7 @@ case $PROCESS in
# "hlt2-output-from-eos" corresponds to using the uploaded full-stream output from a "process=hlt2, input-data=nominal" test.
# These files are overwritten during "lhcb-master" builds of "process=hlt2, input-data=nominal", i.e. ~daily.
CONFIG_FILE="tmp/spruce_hlt2_output_from_eos_input.yaml"
- EXTRA_OPTS='-um --read-evt-max-from-config $MOOREROOT/options/muon_geometry_v2.py'
+ EXTRA_OPTS='-um --read-evt-max-from-config'
;;
*)
echo "ERROR: --input-data must be \"nominal\", \"hlt2-output-locally\", \"hlt2-output-from-eos\" for process \"$PROCESS\""
@@ -183,19 +183,41 @@ case $PROCESS in
esac
### Now run the tests
+# 0. Pre-Job initialising
+if [ $PROCESS = "spruce" ]; then
+ if [ $INPUTDATA = "hlt2-output-from-eos" ]; then
+ echo "Downloading the Hlt2 output metadata to use as input config."
+ DOWNLOAD_INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_prwww_path(stream_config='production', stream='full') )"`)
+ xrdcp -f $DOWNLOAD_INPUT_CONFIG_LOCATION $CONFIG_FILE
+ STORE_ERR_CODE
-# 1. Run Moore.
-# -d downloads the input files locally for speed-up running Moore. Not helpful unless that download is fast for you (e.g. you're at CERN)
-if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "hlt2-output-from-eos" ]; then
- echo "Downloading the Hlt2 output metadata to use as input config."
- DOWNLOAD_INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_prwww_path(stream_config='production', stream='full') )"`)
- xrdcp -f $DOWNLOAD_INPUT_CONFIG_LOCATION $CONFIG_FILE
+ echo "Downloading the Hlt2 stream config json to generate lines for TisTos"
+ TISTOS_STREAM_CONFIG_LOCATION='tmp/spruce__from_eos__stream_config_location.json'
+ DOWNLOAD_TISTOS_STREAM_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.stream_config_json_prwwww_path(stream_config='production') )"`)
+ xrdcp -f $DOWNLOAD_TISTOS_STREAM_CONFIG_LOCATION $TISTOS_STREAM_CONFIG_LOCATION
+ STORE_ERR_CODE
+ elif [ $INPUTDATA = "hlt2-output-locally" ]; then
+ echo "Use local Hlt2 output metadata from previous job as input config"
+ INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_path(stream_config='production', stream='full', for_local_use=True)) "`)
+ cp -f $INPUT_CONFIG_LOCATION $CONFIG_FILE
+
+ echo "Using the local Hlt2 stream config json to generate lines for TisTos"
+ TISTOS_STREAM_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.stream_config_json_path(stream_config='production') )"`)
+ elif [ $INPUTDATA = "nominal" ]; then
+ echo "Using the fixed_line_configs to generate lines for TisTos"
+ TISTOS_STREAM_CONFIG_LOCATION=''
+ fi
+
+ echo "Generating TISTOS option file"
+ time python -m MooreTests.generate_tistos_option_file --stream-config-location=$TISTOS_STREAM_CONFIG_LOCATION
STORE_ERR_CODE
-elif [ $PROCESS = "spruce" ] && [ $INPUTDATA = "hlt2-output-locally" ]; then
- echo "Use local Hlt2 output metadata from previous job as input config"
- INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_path(stream_config='production', stream='full')) "`)
- cp -f $INPUT_CONFIG_LOCATION $CONFIG_FILE
+ TISTOS_OPTION_FILE_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('${PROCESS}'); print( hlpr.tistos_option_file() ) "`)
+ EXTRA_OPTS+=" ${TISTOS_OPTION_FILE_LOCATION}"
+
fi
+
+# 1. Run Moore.
+# -d downloads the input files locally for speed-up running Moore. Not helpful unless that download is fast for you (e.g. you're at CERN)
for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
echo "Running trigger to obtain MDF files with ${STREAM_CONFIG} streams for comparison over ${CONFIG_FILE}"
time python -m MooreTests.run_bandwidth_test_jobs -d -c=$CONFIG_FILE -n=$EVTMAX -p=$PROCESS -t=$MOORE_THREADS -a=$EVENT_SIZE_UPPER_LIMIT $EXTRA_OPTS "${TEST_PATH_PREFIX}${PROCESS}_bandwidth_${STREAM_CONFIG}_streams.py"
@@ -255,4 +277,4 @@ fi
# 8. Produce plots and HTML pages; add the --building-locally flag to make the links work if you are building the html pages locally
echo 'Making plots and HTML pages'
-time python -m MooreTests.make_bandwidth_test_page -p $PROCESS -c $CONFIG_FILE -s $SCRIPT_PATH -e $ERR_CODE $TOP_LEVEL_FLAG $TEST_PAGE_EXTRA_OPTS
\ No newline at end of file
+time python -m MooreTests.make_bandwidth_test_page -p $PROCESS -c $CONFIG_FILE -s $SCRIPT_PATH -e $ERR_CODE $TOP_LEVEL_FLAG $TEST_PAGE_EXTRA_OPTS
diff --git a/scripts/benchmark-scripts/Moore_spruce_latest_bandwidth.sh b/scripts/benchmark-scripts/Moore_spruce_latest_bandwidth.sh
index c922232c11bf415de91df3bbfccfcf326c1bd066..115d06b1cdc3317f0245237521e7bcda41779e91 100755
--- a/scripts/benchmark-scripts/Moore_spruce_latest_bandwidth.sh
+++ b/scripts/benchmark-scripts/Moore_spruce_latest_bandwidth.sh
@@ -14,7 +14,7 @@
# this path ends up printed on the BW test page; export so it can be picked up in the child process
export SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/$(basename "$0")"
-$PRCONFIGROOT/scripts/benchmark-scripts/Moore_bandwidth_test.sh --process spruce --input-data latest
+$PRCONFIGROOT/scripts/benchmark-scripts/Moore_bandwidth_test.sh --process spruce --input-data hlt2-output-from-eos
# force 0 return code so the handler runs even for failed jobs
exit 0
\ No newline at end of file