Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • mstahl/PRConfig
  • chaen/PRConfig
  • lhcb-datapkg/PRConfig
3 results
Show changes
Commits on Source (11)
Hide whitespace changes
Inline Side-by-side
Showing
with 251 additions and 99 deletions
options/Moore/DataChallenges/DC_Sim10aU1_B2Kstee.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"11124001", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_B2Kstee_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"11124001", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Bd2Kstgamma.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"11102202", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Bd2Kstgamma_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"11102202", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Bs2JPsiPhi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"13144011", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Bs2JPsiPhi_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"13144011", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Bs2PhiPhi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"13104012", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Bs2PhiPhi_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"13104012", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Dst2D0pi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"27265100", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Dst2D0pi_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_url_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"27265100", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_MinBias.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"30000000", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_MinBias_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"30000000", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Z2mumu.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu7.6-25ns-Pythia8/Sim10aU1",
"42112000", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10aU1_Z2mumu_LowLumi.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/Upgrade/Beam7000GeV-Upgrade-MagDown-Nu3.8-25ns-Pythia8/Sim10aU1",
"42112000", ["XDIGI"])
options.input_type = "ROOT"
......
options/Moore/DataChallenges/DC_Sim10b_Digi16_MinBias_HLT2_pp_commissioning.py
View file @ 6f88e4aa
......@@ -10,9 +10,9 @@
###############################################################################
from Moore import options
from PRConfig.FilesFromDirac import get_access_urls
from PRConfig.FilesFromDirac import get_access_urls_mc
options.input_files = get_access_urls(
options.input_files = get_access_urls_mc(
"/MC/2022/Beam6800GeV-2022-MagDown-NoUT-Nu2.1-25ns-Pythia8/Sim10b/Digi16/hlt1_pp_no_gec_no_ut/HLT2-pp-commissioning",
"30000000", ["DST"])
......
python/MooreTests/combine-hlt2-rate-output.py
View file @ 6f88e4aa
......@@ -68,9 +68,21 @@ def rates_all_lines():
def rates_all_lines_split_wg():
with open('tmp/Output/line-rates-split-wg.html', 'w') as f:
for file in glob.glob(f'tmp/Output/Inter/rates-all-lines-wg-*.csv'):
stream = str(re.search("-(?!.*-)(.*).csv", file).group(1))
files = glob.glob('tmp/Output/Inter/rates-all-lines-wg-*.csv')
files_by_stream = {
str(re.search("-(?!.*-)(.*).csv", file).group(1)): file
for file in files
}
f.write('<head></head>\n<p>')
f.write('Jump to:\n<ul>')
for stream in files_by_stream.keys():
f.write(
f'<li><a href="#{stream}_label"> {stream.upper()}</a></li>')
f.write('</ul>\n</p>')
for stream, file in files_by_stream.items():
f.write(f'<head>{stream.upper()}</head>')
f.write(f'<a id="{stream}_label">')
df = pd.read_csv(file, header=None)
df.columns = [
'Line', 'Total Retention (%)', 'Rate (kHz)',
......@@ -78,7 +90,11 @@ def rates_all_lines_split_wg():
'Avg Total Event Size (kB)', 'Total Bandwidth (GB/s)',
'Avg DstData Size (kB)', 'DstData Bandwidth (GB/s)'
]
df = df.sort_values(
by=['Total Retention (%)'],
ascending=False).reset_index(drop=True)
f.write(df.to_html())
f.write('</a>')
f.write('<br/><br/>')
return
......@@ -87,10 +103,21 @@ def rates_all_lines_split_wg():
def rates_all_lines_split_stream():
with open('tmp/Output/line-rates-split-production.html', 'w') as f:
for file in glob.glob(
f'tmp/Output/Inter/rates-all-lines-production-*.csv'):
stream = str(re.search("-(?!.*-)(.*).csv", file).group(1))
files = glob.glob('tmp/Output/Inter/rates-all-lines-production-*.csv')
files_by_stream = {
str(re.search("-(?!.*-)(.*).csv", file).group(1)): file
for file in files
}
f.write('<head></head>\n<body>\n<p>')
f.write('Jump to:\n<ul>')
for stream in files_by_stream.keys():
f.write(
f'<li><a href="#{stream}_label"> {stream.upper()}</a></li>')
f.write('</ul>\n</p>')
for stream, file in files_by_stream.items():
f.write(f'<head>{stream.upper()}</head>')
f.write(f'<a id="{stream}_label">')
df = pd.read_csv(file, header=None)
df.columns = [
'Line', 'Total Retention (%)', 'Rate (kHz)',
......@@ -98,7 +125,11 @@ def rates_all_lines_split_stream():
'Avg Total Event Size (kB)', 'Total Bandwidth (GB/s)',
'Avg DstData Size (kB)', 'DstData Bandwidth (GB/s)'
]
df = df.sort_values(
by=['Total Retention (%)'],
ascending=False).reset_index(drop=True)
f.write(df.to_html())
f.write('</a>')
f.write('<br/><br/>')
return
......
python/MooreTests/line-and-stream-rates.py
View file @ 6f88e4aa
......@@ -22,6 +22,7 @@ import re
import argparse
import csv
import os
import yaml
'''
Run snippet with 'python line-rates.py and [1] <MDF file name> [2] <TCK config file name> [3] <JSON file name specifying configuration>'
......@@ -56,6 +57,11 @@ RAW_BANK_TYPES = [(i, LHCb.RawBank.typeName(i))
for i in range(LHCb.RawBank.LastType)]
def parse_yaml(file_path):
with open(os.path.expandvars(file_path), 'r') as f:
return yaml.safe_load(f)
def rawbank_sizes(rawevent, lst):
"""Return (name, size) for each raw bank type."""
if rawevent:
......@@ -239,6 +245,12 @@ if __name__ == '__main__':
type=lambda x: int(round(float(x))),
help='nb of events to process',
required=True)
parser.add_argument(
'-c',
'--config',
type=str,
required=True,
help='Path to yaml config file defining the input.')
parser.add_argument(
'-r',
'--rate',
......@@ -258,8 +270,8 @@ if __name__ == '__main__':
help='Stream configuration specified as JSON',
required=True)
parser.add_argument(
'-c',
'--config',
'-s',
'--stream-config',
type=str,
help='Choose production or per-WG stream configuration',
choices=['production', 'wg'],
......@@ -275,6 +287,8 @@ if __name__ == '__main__':
n_events = args.events
input_config = parse_yaml(args.config)
LHCbApp(
DataType="Upgrade",
Simulation=True,
......@@ -319,13 +333,13 @@ if __name__ == '__main__':
# Two conditions for HLT2 run:
# Use production-stream config to compute rate/size/bandwidth per line and stream
# Use wg-stream config to compute rate/size/bandwidth per line and stream
if args.config == 'production': configname = 'production'
elif args.config == 'wg': configname = 'wg'
if args.stream_config == 'production': configname = 'production'
elif args.stream_config == 'wg': configname = 'wg'
elif args.process == 'Spruce':
# Three conditions for Spruce run:
# Use wg-stream config to compute rate/size/bandwidth per line
# Use wg-stream config to compute rate/size/bandwidth per stream
if not args.config == 'wg': exit()
if not args.stream_config == 'wg': exit()
configname = 'wg-stream'
stream = str(re.search("-(?!.*-)(.*).mdf", file).group(
......@@ -346,11 +360,11 @@ if __name__ == '__main__':
dst,
configname,
stream,
input_rate=args.rate)
input_rate=input_config['input_rate'])
rates_per_stream(
evts_all,
rawbanks_all,
dst_all,
configname,
stream,
input_rate=args.rate)
input_rate=input_config['input_rate'])
python/MooreTests/make_bandwidth_test_page.py
View file @ 6f88e4aa
......@@ -12,6 +12,8 @@ import argparse
import jinja2
import matplotlib.pyplot as plt
import pandas as pd
import yaml
import os
plt.ioff()
......@@ -25,11 +27,15 @@ REPORT_TEMPLATE = jinja2.Template("""
slot.build_id: $$version$$<br>
platform: $$platform$$<br>
hostname: $$hostname$$<br>
cpu_info: $$cpu_info$$
cpu_info: $$cpu_info$$<br>
testing script path: {{SCRIPTPATH}}
</p>
<ul>
<li><a href="{{WWW_BASE_URL}}/$$dirname$$/run.log">Logs</a></li>
</ul>
<p style="color:{{EXIT_CODE_COLOUR}}">
<b>{{EXIT_CODE_SENTENCE}}</b>
</p>
<p>
Results per working group and stream:
<ul>
......@@ -46,7 +52,16 @@ REPORT_TEMPLATE = jinja2.Template("""
<li>Descriptives (whether persistreco and/or extra outputs is enabled)</li>
</ul>
</p>
<p> See: <a href="https://lbfence.cern.ch/alcm/public/figure/details/32">RTA Workflow</a> for reference figures regarding bandwidth.</p>
<p> See: <a href="https://lbfence.cern.ch/alcm/public/figure/details/32">RTA & DPA Workflow</a> for reference figures regarding bandwidth.</p>
<p>
Input sample information:
<ul>
<li>Config file: {{INPUT_CONFIG_PATH}}</li>
<li>Input rate: {{INPUT_RATE}} kHz</li>
<li>Number of interactions per bunch crossing (&#957): {{INPUT_NU}}</li>
<li>Radius of VELO opening: {{INPUT_VELO_RADIUS}} mm</li>
</ul>
</p>
{{HLT2_OR_SPRUCE_TEMPLATE}}
<p>
Other results are shown by plots or tables (in the links) below. <br>
......@@ -66,7 +81,6 @@ REPORT_TEMPLATE = jinja2.Template("""
A line is considered to be "problematic" if it has a rate of 0 Hz
or larger than 1 kHz, which requires some attention. <br>
The rates of all lines are listed in a html page attached below. <br>
{{INPUT_RATE_SENTENCE}}
</p>
<object type="image/png" data="hist_dst_size.png"></object>
<p>
......@@ -518,6 +532,11 @@ def make_plots_per_wg_list(wg_list):
return list_html_str
def parse_yaml(file_path):
with open(os.path.expandvars(file_path), 'r') as f:
return yaml.safe_load(f)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='make_bandwidth_test_page')
parser.add_argument(
......@@ -534,13 +553,38 @@ if __name__ == '__main__':
required=True,
help='Which stage was the test run on')
parser.add_argument(
'-r',
'--rate',
default=500, # kHz
type=float,
help='Input rate corresponding to the input file in kHz')
'-c',
'--input-config',
type=str,
required=True,
help='Path to yaml config file defining the input.')
parser.add_argument(
'-s',
'--script-path',
type=str,
required=True,
help=
'Path to the top-level testing script that is running/calling this script'
)
parser.add_argument(
'-e',
'--exit-code',
type=int,
required=True,
help="Cumulative exit code of all previous jobs.")
args = parser.parse_args()
input_info = parse_yaml(args.input_config)
if args.exit_code == 0:
exit_code_sentence = "All sub-jobs in this test exited successfully."
exit_code_bool = 1
exit_code_col = "green"
else:
exit_code_sentence = "There were errors in some of the sub-jobs of this test; please see the logs."
exit_code_bool = 0
exit_code_col = "red"
# Read info of all lines
df = pd.read_csv(f'{args.input}/rates-for-all-lines.csv', sep=',')
number_of_lines = len(df)
......@@ -596,7 +640,6 @@ if __name__ == '__main__':
table_5stream_rates = rate_html.read()
hlt2_or_spruce_template = HLT2_REPORT_TEMPLATE.render(
WWW_BASE_URL=WWW_BASE_URL, table_5stream_rates=table_5stream_rates)
input_rate_sentence = f"The input rate to this job was {args.rate} kHz (output rate of Hlt1)."
elif args.process == 'Sprucing':
with open(f"{args.input}/rates-wg-stream-configuration.html",
"r") as rate_html:
......@@ -604,13 +647,18 @@ if __name__ == '__main__':
hlt2_or_spruce_template = SPRUCE_REPORT_TEMPLATE.render(
WWW_BASE_URL=WWW_BASE_URL,
table_wgstream_rates=table_wgstream_rates)
input_rate_sentence = f"The input rate to this job was {args.rate} kHz (output rate of Hlt2)."
with open(f"{args.input}/index.html", "w") as html_file:
html = REPORT_TEMPLATE.render(
SCRIPTPATH=args.script_path,
WWW_BASE_URL=WWW_BASE_URL,
HLT2_OR_SPRUCE_TEMPLATE=hlt2_or_spruce_template,
INPUT_RATE_SENTENCE=input_rate_sentence)
INPUT_CONFIG_PATH=os.path.expandvars(args.input_config),
INPUT_RATE=input_info['input_rate'],
INPUT_NU=input_info['nu'],
INPUT_VELO_RADIUS=input_info['velo_radial_opening'],
EXIT_CODE_SENTENCE=exit_code_sentence,
EXIT_CODE_COLOUR=exit_code_col)
html_file.write(html)
with open(f"{args.input}/other_lines.html", "w") as html_file:
......@@ -691,6 +739,9 @@ if __name__ == '__main__':
html_file.write(rate_html.read())
with open(f"{args.input}/message.txt", "w") as message:
message.write(
f'all_jobs_successful_bool = {1 if args.exit_code == 0 else 0}\n'
)
message.write(f'total_rate = {tot_rate:.2f} kHz\n')
message.write(f'total_bandwidth = {tot_bandwidth:.2f} GB/s\n')
message.write(f'n_low_rate = {n_low_rate:d}\n')
......
python/MooreTests/make_profile_plots.py
View file @ 6f88e4aa
#!/usr/bin/env python
###############################################################################
# (c) Copyright 2019 CERN for the benefit of the LHCb Collaboration #
# (c) Copyright 2019-2023 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
......@@ -17,8 +17,6 @@ Example:
you can run:
make_profile_plots.py -l 'HLT1' --logs Profile.log
"""
from __future__ import print_function
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
......@@ -37,8 +35,8 @@ HEAPTRACK_ARGS = [
]
def main(listOfLogs, hltlabel, throughput, ht_file, heaptrack, perf_exe,
no_inline):
def main(listOfLogs, hltlabel, throughput, produceYAML, ht_file, heaptrack,
perf_exe, no_inline):
llvm_cxxfilt_path = "/cvmfs/sft.cern.ch/lcg/contrib/clang/12/x86_64-centos7/bin/llvm-cxxfilt"
if os.path.isfile(llvm_cxxfilt_path):
......@@ -82,7 +80,8 @@ def main(listOfLogs, hltlabel, throughput, ht_file, heaptrack, perf_exe,
return
## reads the text logs and extracts timing shares of different steps of the algorithm
from MooreTests import readTimingTable
timingTable = readTimingTable.readTimings(hltlabel, listOfLogs)
timingTable = readTimingTable.readTimings(hltlabel, listOfLogs,
produceYAML)
# last entry in the sorted list will always be the total time so let's remove that one
sortedList = [
......@@ -162,7 +161,7 @@ if __name__ == '__main__':
dest='hltlabel',
type=str,
required=True,
choices=['HLT1', 'HLT2'],
choices=['HLT1', 'HLT2', 'Sprucing'],
help=
'Pick HLT configuration for plot labels and checking of algorithms in log file)'
)
......@@ -174,6 +173,14 @@ if __name__ == '__main__':
required=False,
help='Events/s throughput number to be added as label to FlameBars')
parser.add_argument(
'--produce-yaml',
dest='produce_yaml',
action='store_true',
help=
'Produce a YAML file with categorised list of algos/tools that make the throughput rate plot'
)
parser.add_argument(
'--ht_file',
dest='ht_file',
......@@ -197,5 +204,5 @@ if __name__ == '__main__':
parser.add_argument('--perf-path', default='perf', help='Path to perf')
args = parser.parse_args()
main(args.logs, args.hltlabel, args.throughput, args.ht_file,
args.heaptrack, args.perf_path, args.no_inline)
main(args.logs, args.hltlabel, args.throughput, args.produce_yaml,
args.ht_file, args.heaptrack, args.perf_path, args.no_inline)
python/MooreTests/readTimingTable.py
View file @ 6f88e4aa
###############################################################################
# (c) Copyright 2000-2022 CERN for the benefit of the LHCb Collaboration #
# (c) Copyright 2000-2023 CERN for the benefit of the LHCb Collaboration #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
......@@ -8,7 +8,6 @@
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
from __future__ import print_function, division, absolute_import
import pandas
import re
from io import StringIO, open
......@@ -123,53 +122,82 @@ algGroups = {
OrderedDict([
("Framework", [
"LHCb__MDF__IOAlg", "reserveIOV", "createODIN", "DummyEventTime",
"LHCb__UnpackRawEvent"
"LHCb__UnpackRawEvent", "CombineRawBanks_for_AllStreams",
"Decode_ODIN", "LumiCounter", "SelectHlt1HltDecReports",
"SelectHlt1HltLumiSummary", "SelectHlt1HltSelReports",
"SelectHlt2HltDecReports", "SelectHlt2HltLumiSummary",
"SelectHlt2HltRoutingBits"
]),
("Converters", [
".*?[c,C]onverter.*", "TracksToSelection.*",
"TrackSelectionToContainer", "fromPrSeedingTracksV1Tracks",
".*Merger.*"
]), #matches either Converter or converter
"TrackSelectionToContainer", "fromPrSeedingTracksV1Tracks"
]), ("Selection (mergers)", [".*Merger.*", "Topo_.*_merger"]),
("Particles", [
"FunctionalParticleMaker",
"FunctionalParticleMaker", "FunctionalDiElectronMaker_.*",
"LHCb__Phys__ParticleMakers__PhotonMaker", "ParticleWithBremMaker",
"LHCb__Phys__ParticleMakers__MergedPi0Maker",
"UniqueIDGeneratorAlg", "ParticleMakerForParticleFlow"
"UniqueIDGeneratorAlg", "ParticleMakerForParticleFlow",
"PFTracksOnly_.*", "Functional.*Tagger_.*",
"WeightedRelTableAlg_.*", "Topo_has_rich_long_pion_.*",
"topo_2body", "topo_2body_with_svtag", "SimpleJets_.*",
"TagsSimpleJets_.*"
]),
("Unpackers", [
("Persistence (unpacking)", [
"MuonPIDUnpacker", "P2VRelationUnpacker", "ParticleUnpacker",
"ProtoParticleUnpacker", "RecVertexUnpacker", "TrackUnpacker",
"UnpackMCParticle", "UnpackMCVertex", "VertexUnpacker",
"CaloHypoUnpacker", "RichPIDUnpacker"
"CaloHypoUnpacker", "RichPIDUnpacker", "RecSummaryUnpacker",
"UnpackRawEvent_.*", ".*Unpack_Hlt2__Event_HLT2_Hlt2Toop.*",
"Unpack_Hlt2__Event_HLT2_Hlt2Topo2Body_Particles ",
"Unpack_Hlt2__Event_HLT2_Hlt2Topo3Body_Particles "
]), ("Decoders", ["HltPackedBufferDecoder", "Hlt2DecReportsDecoder"]),
("Selection (filters)", [
"DefaultGECFilter", "FilterDstDataSize_Hlt2", "require_pvs",
"ParticleRangeFilter", ".*_Hlt2Filter$", "Hlt2_prefilter",
"^filter_RD_.*", "HighPtIsolatedMuonMaker_.*", ".*Filter.*",
"LumiFilter", "PhysFilter"
]),
("Decoders", ["HltPackedBufferDecoder", "HltDecReportsDecoder"]),
("Selection", [
"require_pvs", "DeterministicPrescaler", "ParticleRangeFilter",
".*[c,C]ombiner.*", ".*[b,B]uilder.*", "FilterDesktop",
"FunctionalDiElectronMaker", "GetFlavourTaggingParticles",
"Hlt2Ks0ToPiPiLLMonitoringLineKsLLMonitor",
"Hlt2L0ToPPiLLMonitoringLineLambdaLLMonitor",
"JpsiMaker_ChicJpsiDiMuonLine", "make_chic2jpsimumu",
"onia_prompt_pions", "^Charm.*", "^B2CC_.*", "^bandq_.*",
".*[M,m]ajo.*", "^SLB.*", "^Monitoring_Hlt1KsLine_.*",
"^beauty_to.*", "^charmonium_.*", "^detached_.*", "^dilepton_.*",
"^make_bhadron_.*", "^make_rd_.*", "^prompt_dielectron_.*",
"^rd_.*", "dihadron_from_pk", "^filter_RD_.*", "^jpsi.*",
"^EvtSize_*", "Hlt2_prefilter"
]),
("Streaming", ["Streaming_filter"]),
("PersistenceCloning", [
("Selection (scalers)",
["DeterministicPrescaler", ".*_Prescaler$", ".*_Postscaler$"]),
("Selection (combiners)", [
".*[c,C]ombiner.*", ".*[b,B]uilder.*", "FunctionalDiElectronMaker",
"GetFlavourTaggingParticles", "JpsiMaker_ChicJpsiDiMuonLine",
"make_chic2jpsimumu", "^SpruceB2CC_.*", "^SpruceB2OC_.*",
"^SpruceBandQ_.*", "^SpruceBnoC_.*", "^SpruceCharm.*",
"^SpruceQEE_.*", "^SpruceRD_.*", "^SpruceSLB_.*", "^EW_.*",
"^SLB_.*", "^bandq_.*", "^bnoc_.*", "^SL_.*",
"^Kstp_Maker_resolved_.*", ".*[M,m]ajo.*",
"^Monitoring_Hlt1KsLine_.*", "^beauty_to.*", "^charmonium_.*",
"^detached_.*", "^dilepton_.*", "^make_B0.*", "^make_Bc.*",
"^make_Beauty.*", "^make_Bm.*", "^make_Bp.*", "^make_Bs.*",
"^make_Jpsi.*", "^make_Lb.*", "^make_Omb.*", "^make_Tbc.*",
"^make_V0.*", "^make_Xib.*", "^make_psi.*",
"^make_bbaryon_4body_.*", "^make_phi2kk_.*", "^make_bhadron_.*",
"^make_rd_.*", "^prompt_dielectron_.*", "make_detached_.*",
".*Maker$", "^rd_.*", "dihadron_from_pk", "^jpsi.*",
"ThOrParticleSelection_.*", "^onia_prompt.*", "qee_gamma_.*",
"qee_photons", "^psi2s_.*", "^K.*_topo_2body$",
"^K.*_topo_2body_SVtag$", "SelectionFromWeightedRelationTable_.*",
"^EvtSize_*", "AdvancedCloneKiller"
]), ("Streaming", ["Streaming_filter"]),
("Persistence (cloning)", [
"CopyLinePersistenceLocations",
"CopyParticle2PVRelationsFromLinePersistenceLocations",
"CopyParticlesWithPVRelations", "CopyProtoParticle2MCRelations",
"CopySignalMCParticles", "Gaudi__Hive__FetchDataFromFile"
"CopySignalMCParticles", "Gaudi__Hive__FetchDataFromFile",
"CopyFlavourTagsWithParticlePointers_.*"
]),
("PersistenceWriting", [
("Persistence (writing)", [
"ExecutionReportsWriter", "HltDecReportsWriter",
"HltPackedBufferWriter", "Hlt__RoutingBitsWriter"
"HltPackedBufferWriter", "Hlt__RoutingBitsWriter", "VoidFilter_.*"
]),
("PersistencePacking", [".*Packer.*", "PackMC.*"]),
("Monitoring", ["^Monitor_.*"])
("Persistence (packing)",
[".*Packer.*", "PackMC.*", "PackedBufferDecoder_Hlt2"]),
("Monitoring", [
"^Monitor_.*", "SPRUCEDecReportsMonitor",
"SPRUCEPostscaleDecReportsMonitor",
"SPRUCEPrescaleDecReportsMonitor"
])
])
}
......@@ -260,9 +288,14 @@ def checkTimingVals(timeVals, grouping):
print("\t", a)
def groupValues(groups, timeVals):
def groupValues(groups, timeVals, produceYAML):
"""
Gathers timing values in easy-to-understand groups. Also checks for algorithms that are not included in any grouping or some expected algorithms that are not present
Gathers timing values in easy-to-understand groups.
Also checks for algorithms that are not included in any grouping or some expected algorithms that are not present,
and collects those under a 'Other' category (this category should ideally be empty).
Optionally prints out a YAML file with the detailed per-category list of algorithms and tools
that make each group.
"""
# Checks which values are not in any grouping
checkTimingVals(timeVals, groups)
......@@ -280,14 +313,30 @@ def groupValues(groups, timeVals):
for name in groups:
groupedTimings[name] = 0
groupedTimings["Other"] = 0
# Dictionary to collect details in the YAML
content = {}
for k, v in timeVals.items():
groupedTimings[find_group(k)] += v
g = find_group(k)
groupedTimings[g] += v
if g in content.keys():
content[g].append(k)
else:
content[g] = [k]
for k, v in groupedTimings.items():
if k != "Total":
groupedTimings[k] /= groupedTimings["Total"] / 100.
if produceYAML:
import yaml
with open('contents_timing_plot.yaml', 'w') as myfile:
yaml.dump(content, myfile)
return groupedTimings
def readTimings(groupName, listOfLogs):
return groupValues(algGroups[groupName], parseAllTimingTables(listOfLogs))
def readTimings(groupName, listOfLogs, produceYAML):
return groupValues(algGroups[groupName], parseAllTimingTables(listOfLogs),
produceYAML)