Skip to content
Snippets Groups Projects

[RTADPA BW Tests] Introducing an Hlt1-bandwidth test via Moore_in_Allen

Merged [RTADPA BW Tests] Introducing an Hlt1-bandwidth test via Moore_in_Allen
lugrazet-BW-initialhlt1test into master
Merged Luke Grazette requested to merge lugrazet-BW-initialhlt1test into master
Compare
and
Show latest version
1 file
+ 2
− 3
Compare changes
  • Side-by-side
  • Inline
python/MooreTests/make_bandwidth_test_page.py
+ 145
− 82
@@ -68,7 +68,7 @@ REPORT_TEMPLATE = jinja2.Template("""
<li>Radius of VELO opening: {{INPUT_VELO_RADIUS}} mm</li>
</ul>
</p>
{{HLT2_OR_SPRUCE_TEMPLATE}}
{{TEMPLATE}}
<p>
Other results are shown by plots or tables (in the links) below. <br>
</p>
@@ -77,7 +77,7 @@ REPORT_TEMPLATE = jinja2.Template("""
The number of selection lines per working group. <br>
"Other" category contains those lines with a parsed name that doesn't belong to any known WG. <br>
To make lines properly categorized, one should follow the naming convention,
name of lines should start with `Hlt2/Spruce[WG]_`.
name of lines should start with `Hlt1/Hlt2/Spruce[WG]_`.
</p>
<object type="image/png" data="hist__rate.png"></object>
<p>
@@ -129,25 +129,26 @@ REPORT_TEMPLATE = jinja2.Template("""
The maximum resident set size usage is $$max_rss$$ GB. <br>
The maximum proportional set size usage is $$max_pss$$ GB. <br>
</p>
<ul>
<li><a href="{{BASE_PATH}}/other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{line_descr}}"> PersistReco and ExtraOutput for selection lines</a></li>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_wg_stream}}"> Split by working group: rates, event sizes and bandwidths of all lines</a></li>
$$comparison$$
</b></b>
</ul>
<p> Additional results for HLT2 Bandwidth test (not available for Sprucing test) </p>
<ul>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_prod_stream}}"> Split by production stream: rates, event sizes and bandwidths of all lines</a></li>
</ul>
{{ALL_RESULTS}}
</body>
</html>
""")
HLT1_REPORT_TEMPLATE = jinja2.Template("""<p>
The bandwidth test was run under a single streamless configuration. <br>
The definition of the configuration can be found below.
</p>
<ul>
<li><a href="{{BASE_PATH}}/{{stream_config_json_wg}}">Streamless configuration</a></li>
</ul>
<p>
The streamless configuration is representative of data taking. <br>
The rates, event sizes and bandwidth results from the streamless configuration is: <br>
</p>
<p>
</p>
{{table_streamless_rates}}""")
HLT2_REPORT_TEMPLATE = jinja2.Template("""<p>
The bandwidth test was run under 3 streaming configurations: streamless (all lines written to the same output file), production-stream and wg-stream. <br>
The definition of the production streaming and working-group streaming can be found below.
@@ -175,6 +176,41 @@ SPRUCE_REPORT_TEMPLATE = jinja2.Template("""<p>
</p>
{{table_wgstream_rates}}""")
HLT1_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
</ul>
""")
HLT2_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{line_descr}}"> PersistReco and ExtraOutput for selection lines</a></li>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_wg_stream}}"> Split by working group: rates, event sizes and bandwidths of all lines</a></li>
$$comparison$$
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_prod_stream}}"> Split by production stream: rates, event sizes and bandwidths of all lines</a></li>
</b></b>
</ul>
""")
SPRUCING_ALL_RESULTS = jinja2.Template("""
<ul>
<li><a href="{{BASE_PATH}}/other_lines.html">Show list of lines in "Other" category</a></li>
<li><a href="{{BASE_PATH}}/plots_per_wg.html">Show plots split by WGs</a></li>
<li><a href="{{BASE_PATH}}/all_rates.html">Show rates, event sizes and bandwidths of all lines</a></li>
<li><a href="{{BASE_PATH}}/similarity_matrices.html"> Show similarity Jaccards and overlap matrices between streams for different stream configurations</a></li>
<li><a href="{{BASE_PATH}}/rates_streaming.html"> Show rates of streams under different configurations</a></li>
<li><a href="{{BASE_PATH}}/{{line_descr}}"> PersistReco and ExtraOutput for selection lines</a></li>
<li><a href="{{BASE_PATH}}/{{rate_table_split_by_wg_stream}}"> Split by working group: rates, event sizes and bandwidths of all lines</a></li>
$$comparison$$
</b></b>
</ul>
""")
TABLE_OTHER_LINE_TEMPLATE = jinja2.Template("""
<p>
List of line names that categorized to "Others".
@@ -322,7 +358,7 @@ def make_plots(all_lines_bw_info,
list_other_lines = []
for line, bw_info in all_lines_bw_info.items():
found_wg = False
# Expect e.g {Hlt2,Spruce}<WG>_<rest-of-line-name>
# Expect e.g {Hlt1,Hlt2,Spruce}<WG>_<rest-of-line-name>
wg_guess = line.split("_")[0].removeprefix(process.capitalize())
for wg in rate_info_per_wg.keys():
if wg_guess.startswith(wg):
@@ -431,7 +467,7 @@ if __name__ == '__main__':
'-p',
'--process',
type=str,
choices=['hlt2', 'spruce'],
choices=['hlt1', 'hlt2', 'spruce'],
required=True,
help='Which stage was the test run on')
parser.add_argument(
@@ -489,16 +525,22 @@ if __name__ == '__main__':
}
# Prepare messages to GitLab
# limits on rate: 1 kHz for Hlt2 rate and 0.5% for Sprucing retention
tol = 1000 if args.process == 'hlt2' else 500
# limits on rate: 1 MHz for Hlt1, 1 kHz for Hlt2 rate and 0.5% for Sprucing retention
tol = {'hlt1': 1e6, 'hlt2': 1000, 'spruce': 500}[args.process]
n_low_rate = len(
[info for info in rate_bw_info_by_line.values() if info.rate == 0])
n_high_rate = len(
[info for info in rate_bw_info_by_line.values() if info.rate > tol])
main_stream_config = {
"hlt1": "streamless",
"hlt2": "production",
"spruce": "wg"
}[args.process]
prod_df = pd.read_csv(
fname_helper.final_rate_table_all_streams_path(
"production" if args.process == "hlt2" else "wg", ext="csv"))
main_stream_config, ext="csv"))
tot_rate = sum(prod_df['Rate (kHz)'])
tot_bandwidth = sum(prod_df['Total Bandwidth (GB/s)'])
@@ -512,47 +554,61 @@ if __name__ == '__main__':
other_line_table = make_other_line_table(other_line_list)
plots_per_wg = make_plots_per_wg_list(wg_list)
with open(
fname_helper.final_rate_table_all_streams_path(main_stream_config),
"r") as rate_html:
table_main_stream_rates = rate_html.read()
if args.process == 'hlt2':
with open(
fname_helper.final_rate_table_all_streams_path("production"),
"r") as rate_html:
table_5stream_rates = rate_html.read()
hlt2_or_spruce_template = HLT2_REPORT_TEMPLATE.render(
template = HLT2_REPORT_TEMPLATE.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
stream_config_json_prod=fname_helper.stream_config_json_path(
"production", full_path=False),
stream_config_json_wg=fname_helper.stream_config_json_path(
"wg", full_path=False),
table_5stream_rates=table_5stream_rates)
table_5stream_rates=table_main_stream_rates)
all_results = HLT2_ALL_RESULTS.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
line_descr=fname_helper.line_descr_path(full_path=False),
rate_table_split_by_prod_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"production", full_path=False),
rate_table_split_by_wg_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"wg", full_path=False))
elif args.process == 'spruce':
with open(fname_helper.final_rate_table_all_streams_path("wg"),
"r") as rate_html:
table_wgstream_rates = rate_html.read()
hlt2_or_spruce_template = SPRUCE_REPORT_TEMPLATE.render(
template = SPRUCE_REPORT_TEMPLATE.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
stream_config_json_wg=fname_helper.stream_config_json_path(
"wg", full_path=False),
table_wgstream_rates=table_wgstream_rates)
table_wgstream_rates=table_main_stream_rates)
all_results = HLT2_ALL_RESULTS.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
line_descr=fname_helper.line_descr_path(full_path=False),
rate_table_split_by_wg_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"wg", full_path=False))
elif args.process == 'hlt1':
template = HLT1_REPORT_TEMPLATE.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally),
stream_config_json_wg=fname_helper.stream_config_json_path(
"streamless", full_path=False),
table_streamless_rates=table_main_stream_rates)
all_results = HLT1_ALL_RESULTS.render(
BASE_PATH=fname_helper.base_html_path(args.building_locally))
with open(fname_helper.html_page_outputs_path("index.html"),
"w") as html_file:
html = REPORT_TEMPLATE.render(
SCRIPTPATH=args.script_path,
BASE_PATH=fname_helper.base_html_path(args.building_locally),
HLT2_OR_SPRUCE_TEMPLATE=hlt2_or_spruce_template,
TEMPLATE=template,
ALL_RESULTS=all_results,
INPUT_CONFIG_PATH=os.path.expandvars(args.input_config),
INPUT_RATE=input_info['input_rate'],
INPUT_NU=input_info['nu'],
INPUT_VELO_RADIUS=input_info['velo_radial_opening'],
EXIT_CODE_SENTENCE=exit_code_sentence,
EXIT_CODE_COLOUR=exit_code_col,
line_descr=fname_helper.line_descr_path(full_path=False),
rate_table_split_by_prod_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"production", full_path=False),
rate_table_split_by_wg_stream=fname_helper.
final_rate_table_all_lines_split_by_stream_path(
"wg", full_path=False))
EXIT_CODE_COLOUR=exit_code_col)
html_file.write(html)
with open(fname_helper.html_page_outputs_path("other_lines.html"),
@@ -577,57 +633,64 @@ if __name__ == '__main__':
"r") as rate_table:
html_file.write(rate_table.read())
stream_configs = ["production", "wg"] if args.process == "hlt2" else ["wg"]
with open(
fname_helper.html_page_outputs_path("similarity_matrices.html"),
"w") as html_file:
html_file.write("""
<p>
The overlap between two streams, A and B, w.r.t to one of the stream, A, is computed as |A n B| / |A|.
It shows how much events in the stream A are covered by another stream B. <br>
The columns in the overlap matrices are target streams (A) and the rows are comparison streams (B),
i.e. the numbers correspond to overlaps w.r.t to the column streams. <br>
</p>
""")
for stream_config in stream_configs:
html_file.write(f"""
stream_configs = {
"hlt1": ["streamless"],
"hlt2": ["production", "wg"],
"spruce": ["wg"]
}[args.process]
if args.process != "hlt1":
with open(
fname_helper.html_page_outputs_path(
"similarity_matrices.html"), "w") as html_file:
html_file.write("""
<p>
The overlap matrix of the {stream_config} streams is:
The overlap between two streams, A and B, w.r.t to one of the stream, A, is computed as |A n B| / |A|.
It shows how much events in the stream A are covered by another stream B. <br>
The columns in the overlap matrices are target streams (A) and the rows are comparison streams (B),
i.e. the numbers correspond to overlaps w.r.t to the column streams. <br>
</p>
""")
with open(fname_helper.overlap_matrix_path(stream_config),
"r") as overlap:
html_file.write(overlap.read())
html_file.write("""
<p>
The Jaccard index between two streams, A and B, is computed as |A n B| / |A u B|.
It shows how similar the two streams are and is useful in bandwidth division. <br>
</p>
""")
for stream_config in stream_configs:
html_file.write(f"""
for stream_config in stream_configs:
html_file.write(f"""
<p>
The overlap matrix of the {stream_config} streams is:
</p>
""")
with open(
fname_helper.overlap_matrix_path(stream_config),
"r") as overlap:
html_file.write(overlap.read())
html_file.write("""
<p>
The Jaccard similarity matrix of the {stream_config} streams is:
The Jaccard index between two streams, A and B, is computed as |A n B| / |A u B|.
It shows how similar the two streams are and is useful in bandwidth division. <br>
</p>
""")
for stream_config in stream_configs:
html_file.write(f"""
<p>
The Jaccard similarity matrix of the {stream_config} streams is:
</p>
""")
with open(
fname_helper.jaccard_similarities_path(stream_config),
"r") as jaccard:
html_file.write(jaccard.read())
with open(
fname_helper.html_page_outputs_path("rates_streaming.html"),
"w") as html_file:
for stream_config in stream_configs:
html_file.write(f"""
<p>
The rates, event sizes and bandwidths of the {stream_config} streams are:
</p>
""")
with open(
fname_helper.final_rate_table_all_streams_path(
stream_config), "r") as rate_html:
html_file.write(rate_html.read())
with open(
fname_helper.html_page_outputs_path("rates_streaming.html"),
"w") as html_file:
for stream_config in stream_configs:
html_file.write(f"""
<p>
The rates, event sizes and bandwidths of the {stream_config} streams are:
</p>
""")
with open(
fname_helper.final_rate_table_all_streams_path(
stream_config), "r") as rate_html:
html_file.write(rate_html.read())
with open(fname_helper.html_page_outputs_path("message.txt"),
"w") as message: