Skip to content
Snippets Groups Projects
Commit 776fcb67 authored by Luke Grazette's avatar Luke Grazette Committed by Nicole Skidmore
Browse files

[RTA/DPA BW Tests] Introduce new hlt2_and_spruce_bandwidth test 


Co-authored-by: default avatarRoss John Hunter <ross.john.hunter@cern.ch>
Co-authored-by: default avatarlugrazet <Luke.Grazette@CERN.CH>
parent cfba74bb
No related branches found
No related tags found
1 merge request!374[RTA/DPA BW Tests] Introduce new hlt2_and_spruce_bandwidth test
Hide whitespace changes
Inline Side-by-side
Showing
with 262 additions and 99 deletions
python/MooreTests/generate_hlt2_fullstream_metadata.py
+ 21
− 6
View file @ 776fcb67
......@@ -11,7 +11,7 @@
'''
Takes the config file describing the Hlt2 job's input file.
Generates a metadata yaml about the FULL stream output of the Hlt2 job.
This metadata can then be used as a config file for a 'latest' Spruce job's input file.
This metadata can then be used as a config file for a Spruce job's input file.
'''
import argparse
import json
......@@ -34,9 +34,6 @@ def main():
fname_helper_args = {"stream_config": "production", "stream": "full"}
opts = {}
opts["input_files"] = [fname_helper.mdf_prwww_path(**fname_helper_args)]
opts["input_manifest_file"] = fname_helper.manifest_prwww_path(
fname_helper_args['stream_config'])
for key in ['input_raw_format', 'velo_radial_opening', 'nu']:
opts[key] = config[key]
opts['filtering_info_links'] = [
......@@ -61,9 +58,27 @@ def main():
4)
opts["n_evts"] = n_triggered_full
metadata_config_for_use_in_sprucing_latesthlt2_test = fname_helper.metadata_path(
# Metadata with 'eos' file paths.
opts["input_files"] = [fname_helper.mdf_prwww_path(**fname_helper_args)]
opts["input_manifest_file"] = fname_helper.manifest_prwww_path(
fname_helper_args['stream_config'])
metadata_config_for_use_in_sprucing_hlt2_output_from_eos_test = fname_helper.metadata_path(
**fname_helper_args)
with open(metadata_config_for_use_in_sprucing_hlt2_output_from_eos_test,
'w') as f:
yaml.dump(opts, f, default_flow_style=False)
# Metadata with 'local' file paths.
local_opts = opts
local_opts["input_files"] = [
fname_helper.mdf_fname_for_reading(**fname_helper_args)
]
local_opts["input_manifest_file"] = fname_helper.tck(
stream_config=fname_helper_args["stream_config"])
metadata_config_for_use_in_sprucing_output_locally_test = fname_helper.metadata_path(
**fname_helper_args)
with open(metadata_config_for_use_in_sprucing_latesthlt2_test, 'w') as f:
with open(metadata_config_for_use_in_sprucing_output_locally_test,
'w') as f:
yaml.dump(opts, f, default_flow_style=False)
......
python/MooreTests/make_bandwidth_test_page.py
+ 187
− 80
View file @ 776fcb67
......@@ -22,10 +22,7 @@ from PRConfig.bandwidth_helpers import FileNameHelper
plt.ioff()
REPORT_TEMPLATE = jinja2.Template("""
<html>
<head></head>
<body>
TOP_LEVEL_HEADER = jinja2.Template("""
<p>
slot.build_id: $$version$$<br>
start time: $$start_time$$<br>
......@@ -38,6 +35,27 @@ REPORT_TEMPLATE = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/run.log">Logs</a></li>
</ul>
""")
MEMORY_CONSUMPTION = jinja2.Template("""
<object type="image/png" data="memory_consumption.png"></object>
<p>
Memory consumption as functions of Wall-time. <br>
The virtual memory size is the total amount of memory the process may hypothetically access. <br>
The resident set size (RSS) is the portion of memory occupied by the run that is held in main memory (RAM). <br>
The proportional set size (PSS) is the private memory occupied by the run itself plus the proportion of shared memory with one or more other processes. <br>
As we only launch one test at the same time, PSS should be close to RSS in this case, and PSS gives the real memory that is used by this test. <br>
Swap memory is used when RAM is full. <br>
The maximum resident set size usage is $$max_rss$$ GB. <br>
The maximum proportional set size usage is $$max_pss$$ GB. <br>
</p>
""")
SINGLE_PROCESS_REPORT_TEMPLATE = jinja2.Template("""
<html>
<head></head>
<body>
{{TOP_LEVEL_HEADER}}
<p style="color:{{EXIT_CODE_COLOUR}}">
<b>{{EXIT_CODE_SENTENCE}}</b>
</p>
......@@ -73,7 +91,7 @@ REPORT_TEMPLATE = jinja2.Template("""
Other results are shown by plots or tables (in the links) below. <br>
</p>
{{LINES_PER_WG}}
<object type="image/png" data="hist__rate.png"></object>
<object type="image/png" data="{{PROCESS}}__hist__rate.png"></object>
<p>
Distribution of rate of selection lines. <br>
The total distribution is shown as a stacked histogram, split into several histograms of WGs. <br>
......@@ -85,7 +103,7 @@ REPORT_TEMPLATE = jinja2.Template("""
{{DST_DATA_HIST}}
<object type="image/png" data="hist__total_size.png"></object>
<object type="image/png" data="{{PROCESS}}__hist__total_size.png"></object>
<p>
Distribution of total event size of selection lines. <br>
The total distribution is shown as a stacked histogram, split into several histograms of WGs. <br>
......@@ -97,7 +115,7 @@ REPORT_TEMPLATE = jinja2.Template("""
{{DST_BW_HIST}}
<object type="image/png" data="hist__tot_bandwidth.png"></object>
<object type="image/png" data="{{PROCESS}}__hist__tot_bandwidth.png"></object>
<p>
Distribution of bandwidth computed from total event size. <br>
The total distribution is shown as a stacked histogram, split into several histograms of WGs. <br>
......@@ -106,22 +124,29 @@ REPORT_TEMPLATE = jinja2.Template("""
is larger than 200 MB/s, which requires some attention. This is a temporary limit. <br>
The event sizes of all lines are listed in a html page attached below. <br>
</p>
<object type="image/png" data="memory_consumption.png"></object>
<p>
Memory consumption as functions of Wall-time. <br>
The virtual memory size is the total amount of memory the process may hypothetically access. <br>
The resident set size (RSS) is the portion of memory occupied by the run that is held in main memory (RAM). <br>
The proportional set size (PSS) is the private memory occupied by the run itself plus the proportion of shared memory with one or more other processes. <br>
As we only launch one test at the same time, PSS should be close to RSS in this case, and PSS gives the real memory that is used by this test. <br>
Swap memory is used when RAM is full. <br>
The maximum resident set size usage is $$max_rss$$ GB. <br>
The maximum proportional set size usage is $$max_pss$$ GB. <br>
</p>
{{MEMORY_CONSUMPTION}}
{{ALL_RESULTS}}
</body>
</html>
""")
HLT2_AND_SPRUCE_REPORT_TEMPLATE = jinja2.Template("""
<html>
<head></head>
<body>
{{TOP_LEVEL_HEADER}}
<p>
The bandwidth test ran an Hlt2 test, and then a Sprucing test on the Full-stream output. <br>
The appropriate process-specific webpages can be found below.
</p>
<ul>
<li><a href="{{BASE_PATH}}/hlt2__index.html">Hlt2 index</a></li>
<li><a href="{{BASE_PATH}}/spruce__index.html">Spruce index</a></li>
</ul>
{{MEMORY_CONSUMPTION}}
</body>
</html>""")
HLT1_REPORT_TEMPLATE = jinja2.Template("""<p>
The bandwidth test was run under a single streamless configuration. <br>
The definition of the configuration can be found below.
......@@ -138,7 +163,7 @@ HLT1_REPORT_TEMPLATE = jinja2.Template("""<p>
{{table_streamless_rates}}""")
HLT2_REPORT_TEMPLATE = jinja2.Template("""<p>
The bandwidth test was run under 3 streaming configurations: streamless (all lines written to the same output file), production-stream and wg-stream. <br>
The bandwidth test was run under 2 streaming configurations: production streams (Full, Turbo etc.) and one stream per WG. <br>
The definition of the production streaming and working-group streaming can be found below.
</p>
<ul>
......@@ -152,7 +177,7 @@ HLT2_REPORT_TEMPLATE = jinja2.Template("""<p>
{{table_5stream_rates}}""")
SPRUCE_REPORT_TEMPLATE = jinja2.Template("""<p>
The bandwidth test was run under 2 streaming configurations: streamless and one stream per WG. <br>
The bandwidth test was run under 1 streaming configuration: one stream per WG. <br>
The definition of per-WG-stream configuration can be found below.
</p>
<ul>
......@@ -166,20 +191,21 @@ SPRUCE_REPORT_TEMPLATE = jinja2.Template("""<p>
HLT1_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
$$hlt1__comparison$$
</ul>
""")
HLT2_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{line_descr}}"> PersistReco and ExtraOutput for selection lines</a></li>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_wg_stream}}"> Split by working group: rates, event sizes and bandwidths of all lines</a></li>
$$comparison$$
$$hlt2__comparison$$
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_prod_stream}}"> Split by production stream: rates, event sizes and bandwidths of all lines</a></li>
</b></b>
</ul>
......@@ -187,14 +213,14 @@ HLT2_ALL_RESULTS = jinja2.Template("""
SPRUCING_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/{{PROCESS}}__rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{line_descr}}"> PersistReco and ExtraOutput for selection lines</a></li>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_wg_stream}}"> Split by working group: rates, event sizes and bandwidths of all lines</a></li>
$$comparison$$
$${{PROCESS}}__comparison$$
</b></b>
</ul>
""")
......@@ -204,7 +230,7 @@ HLT1_DST_DATA_HIST = jinja2.Template("""""")
HLT1_DST_BW_HIST = jinja2.Template("""""")
HLT2_OR_SPRUCING_LINES_PER_WG = jinja2.Template("""
<object type="image/png" data="lines_per_wg.png"></object>
<object type="image/png" data="{{PROCESS}}__lines_per_wg.png"></object>
<p>
The number of selection lines per working group. <br>
"Other" category contains those lines with a parsed name that doesn't belong to any known WG. <br>
......@@ -214,7 +240,7 @@ HLT2_OR_SPRUCING_LINES_PER_WG = jinja2.Template("""
""")
HLT2_OR_SPRUCING_DST_DATA_HIST = jinja2.Template("""
<object type="image/png" data="hist__dst_data_size.png"></object>
<object type="image/png" data="{{PROCESS}}__hist__dst_data_size.png"></object>
<p>
Distribution of DstData RawBank size of selection lines. <br>
The total distribution is shown as a stacked histogram, split into several histograms of WGs. <br>
......@@ -223,7 +249,7 @@ HLT2_OR_SPRUCING_DST_DATA_HIST = jinja2.Template("""
""")
HLT2_OR_SPRUCING_DST_BW_HIST = jinja2.Template("""
<object type="image/png" data="hist__dst_bandwidth.png"></object>
<object type="image/png" data="{{PROCESS}}__hist__dst_bandwidth.png"></object>
<p>
Distribution of bandwidth computed from DstData RawBank size. <br>
The total distribution is shown as a stacked histogram, split into several histograms of WGs. <br>
......@@ -351,7 +377,7 @@ def make_plots_per_wg(fname_helper, wg_name, wg_bw_info, process):
]):
histo_maker(
getattr(wg_bw_info, attrib), xtitle, title,
fname_helper.html_page_outputs_path(
fname_helper.process_dependent_html_page_outputs_path(
f"hist__{plot_bit}__{wg_name}.png"))
......@@ -408,7 +434,7 @@ def make_plots(all_lines_bw_info,
labels=labels,
wedgeprops=dict(width=0.4, edgecolor="w"))
plt.title(f"Number of {process.capitalize()} lines per WG")
plt.savefig("tmp/Output/lines_per_wg.png", format="png")
plt.savefig(f"tmp/Output/{args.process}__lines_per_wg.png", format="png")
plt.close(fig)
### Make hist plots
......@@ -432,7 +458,8 @@ def make_plots(all_lines_bw_info,
[getattr(info, attrib) for info in rate_info_per_wg.values()],
xtitle,
title,
fname_helper.html_page_outputs_path(f"hist__{plot_bit}.png"),
fname_helper.process_dependent_html_page_outputs_path(
f"hist__{plot_bit}.png"),
range=range,
take_log=take_log,
log_th=log_th,
......@@ -460,18 +487,18 @@ def make_other_line_table(name_list):
return table_html_str
def make_plots_per_wg_list(wg_list):
def make_plots_per_wg_list(wg_list, process):
list_html_str = ''
for wg_name in wg_list:
list_html_str += f'''
<p>
Plots of {wg_name} group:
</p>
<object type="image/png" data="hist__rate__{wg_name}.png"></object>
<object type="image/png" data="hist__dst_data_size__{wg_name}.png"></object>
<object type="image/png" data="hist__total_size__{wg_name}.png"></object>
<object type="image/png" data="hist__dst_bandwidth__{wg_name}.png"></object>
<object type="image/png" data="hist__tot_bandwidth__{wg_name}.png"></object>
<object type="image/png" data="{process}__hist__rate__{wg_name}.png"></object>
<object type="image/png" data="{process}__hist__dst_data_size__{wg_name}.png"></object>
<object type="image/png" data="{process}__hist__total_size__{wg_name}.png"></object>
<object type="image/png" data="{process}__hist__dst_bandwidth__{wg_name}.png"></object>
<object type="image/png" data="{process}__hist__tot_bandwidth__{wg_name}.png"></object>
'''
return list_html_str
......@@ -481,6 +508,25 @@ def parse_yaml(file_path):
return yaml.safe_load(f)
def _write_message(message,
args,
tot_rate,
tot_bandwidth,
n_low_rate,
n_high_rate,
process_dependent_message=False):
lines = [
f"all_jobs_successful_bool = {1 if args.exit_code == 0 else 0}\n",
f"total_rate = {tot_rate:.2f} kHz\n",
f"total_bandwidth = {tot_bandwidth:.2f} GB/s\n",
f"n_low_rate = {n_low_rate:d}\n", f"n_high_rate = {n_high_rate:d}\n"
]
if process_dependent_message:
lines = [f'{args.process}__{line}' for line in lines]
message.writelines(lines)
return 0
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='make_bandwidth_test_page')
parser.add_argument(
......@@ -489,7 +535,7 @@ if __name__ == '__main__':
type=str,
choices=['hlt1', 'hlt2', 'spruce'],
required=True,
help='Which stage was the test run on')
help='Which stage was the test run on.')
parser.add_argument(
'-c',
'--input-config',
......@@ -502,7 +548,7 @@ if __name__ == '__main__':
type=str,
required=True,
help=
'Path to the top-level testing script that is running/calling this script'
'Path to the top-level testing script that is running/calling this script.'
)
parser.add_argument(
'-e',
......@@ -510,6 +556,16 @@ if __name__ == '__main__':
type=int,
required=True,
help="Cumulative exit code of all previous jobs.")
parser.add_argument(
'--multiple-processes',
action='store_true',
help="Flag to use index page appropriate for multiple processes.")
parser.add_argument(
'--skip-top-level-information-for-process-dependent-testpage',
action='store_true',
help=
'Flag to avoid memory-consumption and build information sections of {proces}__index page.'
)
parser.add_argument(
'--building-locally',
action='store_true',
......@@ -572,7 +628,7 @@ if __name__ == '__main__':
process=args.process)
other_line_table = make_other_line_table(other_line_list)
plots_per_wg = make_plots_per_wg_list(wg_list)
plots_per_wg = make_plots_per_wg_list(wg_list, args.process)
with open(
fname_helper.final_rate_table_all_streams_path(main_stream_config),
......@@ -594,10 +650,13 @@ if __name__ == '__main__':
"production", full_path=False),
rate_table_split_by_wg_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"wg", full_path=False))
lines_per_wg = HLT2_OR_SPRUCING_LINES_PER_WG.render()
dst_data_hist = HLT2_OR_SPRUCING_DST_DATA_HIST.render()
dst_bw_hist = HLT2_OR_SPRUCING_DST_BW_HIST.render()
"wg", full_path=False),
PROCESS=args.process)
lines_per_wg = HLT2_OR_SPRUCING_LINES_PER_WG.render(
PROCESS=args.process)
dst_data_hist = HLT2_OR_SPRUCING_DST_DATA_HIST.render(
PROCESS=args.process)
dst_bw_hist = HLT2_OR_SPRUCING_DST_BW_HIST.render(PROCESS=args.process)
elif args.process == 'spruce':
template = SPRUCE_REPORT_TEMPLATE.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
......@@ -609,10 +668,13 @@ if __name__ == '__main__':
line_descr=fname_helper.line_descr_path(full_path=False),
rate_table_split_by_wg_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"wg", full_path=False))
lines_per_wg = HLT2_OR_SPRUCING_LINES_PER_WG.render()
dst_data_hist = HLT2_OR_SPRUCING_DST_DATA_HIST.render()
dst_bw_hist = HLT2_OR_SPRUCING_DST_BW_HIST.render()
"wg", full_path=False),
PROCESS=args.process)
lines_per_wg = HLT2_OR_SPRUCING_LINES_PER_WG.render(
PROCESS=args.process)
dst_data_hist = HLT2_OR_SPRUCING_DST_DATA_HIST.render(
PROCESS=args.process)
dst_bw_hist = HLT2_OR_SPRUCING_DST_BW_HIST.render(PROCESS=args.process)
elif args.process == 'hlt1':
template = HLT1_REPORT_TEMPLATE.render(
......@@ -621,16 +683,24 @@ if __name__ == '__main__':
"streamless", full_path=False),
table_streamless_rates=table_main_stream_rates)
all_results = HLT1_ALL_RESULTS.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally))
BASE_PATH=fname_helper.base_html_path(args.building_locally),
PROCESS=args.process)
lines_per_wg = HLT1_LINES_PER_WG.render()
dst_data_hist = HLT1_DST_DATA_HIST.render()
dst_bw_hist = HLT1_DST_BW_HIST.render()
with open(fname_helper.html_page_outputs_path("index.html"),
"w") as html_file:
html = REPORT_TEMPLATE.render(
SCRIPTPATH=args.script_path,
BASE_PATH=fname_helper.base_html_path(args.building_locally),
with open(
fname_helper.html_page_outputs_path(f"{args.process}__index.html"),
"w") as html_file:
html = SINGLE_PROCESS_REPORT_TEMPLATE.render(
TOP_LEVEL_HEADER=""
if args.skip_top_level_information_for_process_dependent_testpage
else TOP_LEVEL_HEADER.render(
SCRIPTPATH=args.script_path,
BASE_PATH=fname_helper.base_html_path(args.building_locally)),
MEMORY_CONSUMPTION=""
if args.skip_top_level_information_for_process_dependent_testpage
else MEMORY_CONSUMPTION.render(),
TEMPLATE=template,
ALL_RESULTS=all_results,
LINES_PER_WG=lines_per_wg,
......@@ -641,22 +711,26 @@ if __name__ == '__main__':
INPUT_NU=input_info['nu'],
INPUT_VELO_RADIUS=input_info['velo_radial_opening'],
EXIT_CODE_SENTENCE=exit_code_sentence,
EXIT_CODE_COLOUR=exit_code_col)
EXIT_CODE_COLOUR=exit_code_col,
PROCESS=args.process)
html_file.write(html)
with open(fname_helper.html_page_outputs_path("other_lines.html"),
"w") as html_file:
with open(
fname_helper.process_dependent_html_page_outputs_path(
"other_lines.html"), "w") as html_file:
html = TABLE_OTHER_LINE_TEMPLATE.render(
table_other_lines=other_line_table)
html_file.write(html)
with open(fname_helper.html_page_outputs_path("plots_per_wg.html"),
"w") as html_file:
with open(
fname_helper.process_dependent_html_page_outputs_path(
"plots_per_wg.html"), "w") as html_file:
html = PLOTS_PER_WG_TEMPLATE.render(plots_per_wg=plots_per_wg)
html_file.write(html)
with open(fname_helper.html_page_outputs_path("all_rates.html"),
"w") as html_file:
with open(
fname_helper.process_dependent_html_page_outputs_path(
"all_rates.html"), "w") as html_file:
html = ALL_RATE_TEMPLATE.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
CSV_PATH=fname_helper.final_rate_table_all_lines_path(
......@@ -674,7 +748,7 @@ if __name__ == '__main__':
if args.process != "hlt1":
with open(
fname_helper.html_page_outputs_path(
fname_helper.process_dependent_html_page_outputs_path(
"similarity_matrices.html"), "w") as html_file:
html_file.write("""
<p>
......@@ -712,8 +786,8 @@ if __name__ == '__main__':
html_file.write(jaccard.read())
with open(
fname_helper.html_page_outputs_path("rates_streaming.html"),
"w") as html_file:
fname_helper.process_dependent_html_page_outputs_path(
"rates_streaming.html"), "w") as html_file:
for stream_config in stream_configs:
html_file.write(f"""
<p>
......@@ -725,12 +799,45 @@ if __name__ == '__main__':
stream_config), "r") as rate_html:
html_file.write(rate_html.read())
with open(fname_helper.html_page_outputs_path("message.txt"),
"w") as message:
message.write(
f'all_jobs_successful_bool = {1 if args.exit_code == 0 else 0}\n')
message.write(f'total_rate = {tot_rate:.2f} kHz\n')
message.write(f'total_bandwidth = {tot_bandwidth:.2f} GB/s\n')
message.write(f'n_low_rate = {n_low_rate:d}\n')
message.write(f'n_high_rate = {n_high_rate:d}\n')
with open(fname_helper.html_page_outputs_path("index.html"),
"w") as html_file:
if args.multiple_processes:
html = HLT2_AND_SPRUCE_REPORT_TEMPLATE.render(
TOP_LEVEL_HEADER=TOP_LEVEL_HEADER.render(
SCRIPTPATH=args.script_path,
BASE_PATH=fname_helper.base_html_path(
args.building_locally)),
BASE_PATH=fname_helper.base_html_path(args.building_locally),
MEMORY_CONSUMPTION=MEMORY_CONSUMPTION.render())
html_file.write(html)
else:
# In single-process tests, need 'index.html' to be picked up.
with open(
fname_helper.html_page_outputs_path(
f"{args.process}__index.html"),
"r") as process_dependent_html_file:
html_file.write(process_dependent_html_file.read())
with open(
fname_helper.html_page_outputs_path(
f"{args.process}__message.txt"), "w") as message:
_write_message(
message=message,
args=args,
tot_rate=tot_rate,
tot_bandwidth=tot_bandwidth,
n_low_rate=n_low_rate,
n_high_rate=n_high_rate,
process_dependent_message=False)
with open(
fname_helper.html_page_outputs_path("message.txt"),
"a" if args.multiple_processes else "w") as message:
_write_message(
message=message,
args=args,
tot_rate=tot_rate,
tot_bandwidth=tot_bandwidth,
n_low_rate=n_low_rate,
n_high_rate=n_high_rate,
process_dependent_message=True)
pass
python/MooreTests/run_bandwidth_test_jobs.py
+ 1
− 1
View file @ 776fcb67
......@@ -189,7 +189,7 @@ if __name__ == '__main__':
'read_evt_max_from_config only makes sense for Sprucing jobs with config = metadata generated about Hlt2 BW job outputs.'
)
config = parse_yaml(args.config)
args.evt_max = int(config['n_evts'])
args.evt_max = min(args.evt_max, int(config['n_evts']))
if args.evt_max == -1 or args.evt_max > 1e5:
raise RuntimeError(
......
python/PRConfig/bandwidth_helpers.py
+ 6
− 2
View file @ 776fcb67
......@@ -128,8 +128,8 @@ class FileNameHelper(object):
def _tmp_rate_table_path(self, stream_config, stream, line_or_stream):
return os.path.join(
self.base_dir, self.output_subdir, self.inter_subsubdir,
self._join(f"rates_per_{line_or_stream}", stream_config, stream) +
".csv")
self._join(self.process, f"rates_per_{line_or_stream}",
stream_config, stream) + ".csv")
def tmp_rate_table_per_line_path(self, stream_config, stream):
return self._tmp_rate_table_path(stream_config, stream, "line")
......@@ -163,3 +163,7 @@ class FileNameHelper(object):
def html_page_outputs_path(self, fname):
return os.path.join(self.base_dir, self.output_subdir, fname)
def process_dependent_html_page_outputs_path(self, fname):
return os.path.join(self.base_dir, self.output_subdir,
self._join(self.process, fname))
scripts/benchmark-scripts/Moore_bandwidth_test.sh
+ 26
− 10
View file @ 776fcb67
......@@ -25,8 +25,8 @@ Usage: Moore/run /path/to/Moore_bandwidth_test.sh [options] 2>&1 | tee <path-to-
This is useful to e.g. run from a released version, where one cannot easily edit the hard-coded number of events below.
For example: Moore/run env OVERRIDE_EVTMAX=1e4 /path/to/Moore_bandwidth_test.sh ...
--process: "hlt1", "hlt2" or "spruce".
--input-data: "nominal" or "latest".
"latest" not currently available for process == hlt1 or hlt2.
--input-data: "nominal", "hlt2-output-locally" or "hlt2-output-from-eos".
"hlt2-output-from-eos" and "hlt2-output-locally" are not currently available for process == hlt1 or hlt2.
-h|--help: print this message and exit.
EOF
......@@ -77,6 +77,10 @@ while [[ $# -gt 0 ]]; do
shift # parse argument
shift # parse value
;;
--skip-top-level-information-for-process-dependent-testpage)
export TOP_LEVEL_FLAG="--skip-top-level-information-for-process-dependent-testpage"
shift # parse argument
;;
*)
echo "ERROR: Unknown argument \"$1\""
exit 1
......@@ -105,6 +109,7 @@ if [ -z "${OVERRIDE_EVTMAX}" ]; then
else
EVTMAX=${OVERRIDE_EVTMAX}
fi
TEST_PAGE_EXTRA_OPTS=""
case $PROCESS in
hlt1)
......@@ -152,14 +157,21 @@ case $PROCESS in
CONFIG_FILE="${TEST_PATH_PREFIX}spruce_bandwidth_input.yaml"
EXTRA_OPTS='$MOOREROOT/options/muon_geometry_v2.py'
;;
latest)
# "latest" corresponds to using the uploaded full-stream output from a "process=hlt2, input-data=nominal" test.
hlt2-output-locally)
# "hlt2-output-locally" corresponds to using the locally-run full-stream output from "process=hlt2, input-data=nominal" test.
CONFIG_FILE="tmp/spruce_hlt2_output_locally_input.yaml"
EXTRA_OPTS='-um --read-evt-max-from-config $MOOREROOT/options/muon_geometry_v2.py'
# Flag to make a top-level human-readable output page directing to the Hlt2 and Spruce output pages.
TEST_PAGE_EXTRA_OPTS='--multiple-processes'
;;
hlt2-output-from-eos)
# "hlt2-output-from-eos" corresponds to using the uploaded full-stream output from a "process=hlt2, input-data=nominal" test.
# These files are overwritten during "lhcb-master" builds of "process=hlt2, input-data=nominal", i.e. ~daily.
CONFIG_FILE="tmp/hlt2_bw_testing__production__full__metadata.yaml"
CONFIG_FILE="tmp/spruce_hlt2_output_from_eos_input.yaml"
EXTRA_OPTS='-um --read-evt-max-from-config $MOOREROOT/options/muon_geometry_v2.py'
;;
*)
echo "ERROR: --input-data must be \"nominal\" or \"latest\" for process \"$PROCESS\""
echo "ERROR: --input-data must be \"nominal\", \"hlt2-output-locally\", \"hlt2-output-from-eos\" for process \"$PROCESS\""
exit 1
;;
esac
......@@ -174,11 +186,15 @@ esac
# 1. Run Moore.
# -d downloads the input files locally for speed-up running Moore. Not helpful unless that download is fast for you (e.g. you're at CERN)
if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "latest" ]; then
if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "hlt2-output-from-eos" ]; then
echo "Downloading the Hlt2 output metadata to use as input config."
DOWNLOAD_INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_prwww_path(stream_config='production', stream='full') )"`)
xrdcp -f $DOWNLOAD_INPUT_CONFIG_LOCATION $CONFIG_FILE
STORE_ERR_CODE
elif [ $PROCESS = "spruce" ] && [ $INPUTDATA = "hlt2-output-locally" ]; then
echo "Use local Hlt2 output metadata from previous job as input config"
INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_path(stream_config='production', stream='full')) "`)
cp -f $INPUT_CONFIG_LOCATION $CONFIG_FILE
fi
for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
echo "Running trigger to obtain MDF files with ${STREAM_CONFIG} streams for comparison over ${CONFIG_FILE}"
......@@ -230,13 +246,13 @@ echo 'Combining all rate and bandwidth tables'
time python $PRCONFIGROOT/python/MooreTests/combine_rate_output.py --process $PROCESS
STORE_ERR_CODE
# 7. Required information for 'latest' sprucing jobs.
# 7. Required information for 'hlt2-output-locally' or 'hlt2-output-from-eos' sprucing jobs.
if [ $PROCESS = "hlt2" ] && [ $INPUTDATA = "nominal" ]; then
echo 'Generating yaml metadata to upload to eos for Moore_spruce_latest_bandwidth test'
echo 'Generating yaml metadata to potentially use in a sprucing ["hlt2-output-locally", "hlt2-output-from-eos"] test'
time python -m MooreTests.generate_hlt2_fullstream_metadata -c $CONFIG_FILE
STORE_ERR_CODE
fi
# 8. Produce plots and HTML pages; add the --building-locally flag to make the links work if you are building the html pages locally
echo 'Making plots and HTML pages'
time python -m MooreTests.make_bandwidth_test_page -p $PROCESS -c $CONFIG_FILE -s $SCRIPT_PATH -e $ERR_CODE
time python -m MooreTests.make_bandwidth_test_page -p $PROCESS -c $CONFIG_FILE -s $SCRIPT_PATH -e $ERR_CODE $TOP_LEVEL_FLAG $TEST_PAGE_EXTRA_OPTS
\ No newline at end of file
scripts/benchmark-scripts/Moore_hlt2_and_spruce_bandwidth.sh 0 → 100755
+ 21
− 0
View file @ 776fcb67
#!/bin/bash
###############################################################################
# (c) Copyright 2022-2024 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
# this path ends up printed on the BW test page; export so it can be picked up in the child process
export SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/$(basename "$0")"
$PRCONFIGROOT/scripts/benchmark-scripts/Moore_bandwidth_test.sh --process hlt2 --input-data nominal --skip-top-level-information-for-process-dependent-testpage
$PRCONFIGROOT/scripts/benchmark-scripts/Moore_bandwidth_test.sh --process spruce --input-data hlt2-output-locally --skip-top-level-information-for-process-dependent-testpage
# force 0 return code so the handler runs even for failed jobs
exit 0
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment