Commit 3d20080d authored by Vladimir Gligorov's avatar Vladimir Gligorov
Browse files

resolve conflicts

parents 232d6c04 eb496908
......@@ -3,11 +3,30 @@ stages:
- run
- publish
check-formatting:
stage: build
image: debian:testing
script:
- apt update && apt install -y clang-format-7 python-pip git curl
- pip install yapf==0.24.0 whichcraft
- if [ ! -e .clang-format ] ; then
- curl -o .clang-format "https://gitlab.cern.ch/lhcb-parallelization/Allen/raw/master/.clang-format?inline=false"
- echo '.clang-format' >> .gitignore
- git add .gitignore
- fi
- curl -o lb-format "https://gitlab.cern.ch/lhcb-core/LbDevTools/raw/master/LbDevTools/SourceTools.py?inline=false"
- python lb-format --format-patch apply-formatting.patch origin/master
artifacts:
paths:
- apply-formatting.patch
when: on_failure
expire_in: 1 week
allow_failure: true
.build_job: &build_job_def
only:
refs:
- master
- review_june
- schedules
- web
stage: build
......@@ -183,10 +202,10 @@ DefaultSequence:build:
tags:
- t4
DataPreparation:build:
<<: *build_job_def
tags:
- rtx2080ti
# DataPreparation:build:
# <<: *build_job_def
# tags:
# - rtx2080ti
# Run on all platforms
# DefaultSequence
......@@ -260,77 +279,77 @@ gtx1080ti:DefaultSequence:run_throughput:
dependencies:
- DefaultSequence:build
# DataPreparation
# Run on all platforms
rtx2080ti:DataPreparation:run_throughput:
<<: *run_throughput_job_def
tags:
- rtx2080ti
dependencies:
- DataPreparation:build
v100:DataPreparation:run_throughput:
<<: *run_throughput_job_def
tags:
- v100
dependencies:
- DataPreparation:build
t4:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- t4
dependencies:
- DataPreparation:build
gtx780ti:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtx780ti
dependencies:
- DataPreparation:build
gtx10606g:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtx10606g
dependencies:
- DataPreparation:build
gtx980:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtx980
dependencies:
- DataPreparation:build
gtx680:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtx680
dependencies:
- DataPreparation:build
gtxtitanx:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtxtitanx
dependencies:
- DataPreparation:build
gtx670:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtx670
dependencies:
- DataPreparation:build
gtx1080ti:DataPreparation:run_throughput:
<<: *run_throughput_job_no_profiling_def
tags:
- gtx1080ti
dependencies:
- DataPreparation:build
# # DataPreparation
# # Run on all platforms
# rtx2080ti:DataPreparation:run_throughput:
# <<: *run_throughput_job_def
# tags:
# - rtx2080ti
# dependencies:
# - DataPreparation:build
# v100:DataPreparation:run_throughput:
# <<: *run_throughput_job_def
# tags:
# - v100
# dependencies:
# - DataPreparation:build
# t4:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - t4
# dependencies:
# - DataPreparation:build
# gtx780ti:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtx780ti
# dependencies:
# - DataPreparation:build
# gtx10606g:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtx10606g
# dependencies:
# - DataPreparation:build
# gtx980:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtx980
# dependencies:
# - DataPreparation:build
# gtx680:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtx680
# dependencies:
# - DataPreparation:build
# gtxtitanx:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtxtitanx
# dependencies:
# - DataPreparation:build
# gtx670:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtx670
# dependencies:
# - DataPreparation:build
# gtx1080ti:DataPreparation:run_throughput:
# <<: *run_throughput_job_no_profiling_def
# tags:
# - gtx1080ti
# dependencies:
# - DataPreparation:build
# Publish
......@@ -377,43 +396,43 @@ speedup:DefaultSequence:publish_speedup:
- gtx1080ti:DefaultSequence:run_throughput
- gtx980:DefaultSequence:run_throughput
# Data Preparation
rtx2080ti:DataPreparation:publish_algo_breakdown_plot:
<<: *publish_algo_breakdown_plot_def
tags:
- gpu
dependencies:
- rtx2080ti:DataPreparation:run_throughput
throughput:DataPreparation:publish_throughput:
<<: *publish_throughput_job_def
tags:
- gpu
dependencies:
- rtx2080ti:DataPreparation:run_throughput
- v100:DataPreparation:run_throughput
- t4:DataPreparation:run_throughput
- gtx10606g:DataPreparation:run_throughput
- gtx680:DataPreparation:run_throughput
- gtxtitanx:DataPreparation:run_throughput
- gtx670:DataPreparation:run_throughput
- gtx780ti:DataPreparation:run_throughput
- gtx1080ti:DataPreparation:run_throughput
- gtx980:DataPreparation:run_throughput
speedup:DataPreparation:publish_speedup:
<<: *publish_speedup_job_def
tags:
- gpu
dependencies:
- rtx2080ti:DataPreparation:run_throughput
- v100:DataPreparation:run_throughput
- t4:DataPreparation:run_throughput
- gtx10606g:DataPreparation:run_throughput
- gtx680:DataPreparation:run_throughput
- gtx980:DataPreparation:run_throughput
- gtxtitanx:DataPreparation:run_throughput
- gtx670:DataPreparation:run_throughput
- gtx780ti:DataPreparation:run_throughput
- gtx1080ti:DataPreparation:run_throughput
- gtx980:DataPreparation:run_throughput
# # Data Preparation
# rtx2080ti:DataPreparation:publish_algo_breakdown_plot:
# <<: *publish_algo_breakdown_plot_def
# tags:
# - gpu
# dependencies:
# - rtx2080ti:DataPreparation:run_throughput
# throughput:DataPreparation:publish_throughput:
# <<: *publish_throughput_job_def
# tags:
# - gpu
# dependencies:
# - rtx2080ti:DataPreparation:run_throughput
# - v100:DataPreparation:run_throughput
# - t4:DataPreparation:run_throughput
# - gtx10606g:DataPreparation:run_throughput
# - gtx680:DataPreparation:run_throughput
# - gtxtitanx:DataPreparation:run_throughput
# - gtx670:DataPreparation:run_throughput
# - gtx780ti:DataPreparation:run_throughput
# - gtx1080ti:DataPreparation:run_throughput
# - gtx980:DataPreparation:run_throughput
# speedup:DataPreparation:publish_speedup:
# <<: *publish_speedup_job_def
# tags:
# - gpu
# dependencies:
# - rtx2080ti:DataPreparation:run_throughput
# - v100:DataPreparation:run_throughput
# - t4:DataPreparation:run_throughput
# - gtx10606g:DataPreparation:run_throughput
# - gtx680:DataPreparation:run_throughput
# - gtx980:DataPreparation:run_throughput
# - gtxtitanx:DataPreparation:run_throughput
# - gtx670:DataPreparation:run_throughput
# - gtx780ti:DataPreparation:run_throughput
# - gtx1080ti:DataPreparation:run_throughput
# - gtx980:DataPreparation:run_throughput
This diff is collapsed.
import os,sys,fnmatch
import os, sys, fnmatch
import ROOT
from ROOT import *
import re
bins = [0,2000,3000,4000,5000,6000,7000,8000,9000,10000,25000]
tpts = [247678,229501,204182,182705,169174,156493,157174,166272,191051,145908]
geceffs = [0.049,0.110,0.136,0.148,0.142,0.136,0.107,0.056,0.047,0.067]
bins = [0, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 25000]
tpts = [
247678, 229501, 204182, 182705, 169174, 156493, 157174, 166272, 191051,
145908
]
geceffs = [
0.049, 0.110, 0.136, 0.148, 0.142, 0.136, 0.107, 0.056, 0.047, 0.067
]
for i,j in enumerate(tpts) :
print tpts[i]*geceffs[i]
for i, j in enumerate(tpts):
print tpts[i] * geceffs[i]
......@@ -45,7 +45,6 @@ def getCuts():
# "Long_eta25", "LongFromB_eta25"
# ]
basedict["Velo"] = [
"VeloTracks", "VeloTracks_eta25", "LongFromB_eta25", "LongFromD_eta25",
"LongStrange_eta25"
......
......@@ -2,16 +2,17 @@ from ROOT import gStyle
from ROOT import gROOT
from ROOT import TStyle
def setLHCbStyle() :
def setLHCbStyle():
global lhcbStyle
lhcbFont = 132
lhcbTSize = 0.06
lhcbWidth = 2
lhcbFont = 132
lhcbTSize = 0.06
lhcbWidth = 2
lhcbStyle= TStyle("lhcbStyle","LHCb plots style");
lhcbStyle = TStyle("lhcbStyle", "LHCb plots style")
lhcbStyle.SetFillColor(1)
lhcbStyle.SetFillStyle(1001) # solid
lhcbStyle.SetFillStyle(1001) # solid
lhcbStyle.SetFrameFillColor(0)
lhcbStyle.SetFrameBorderMode(0)
lhcbStyle.SetPadBorderMode(0)
......@@ -26,57 +27,58 @@ def setLHCbStyle() :
lhcbStyle.SetTextFont(lhcbFont)
lhcbStyle.SetTitleFont(lhcbFont)
lhcbStyle.SetTextSize(lhcbTSize)
lhcbStyle.SetLabelFont(lhcbFont,"x")
lhcbStyle.SetLabelFont(lhcbFont,"y")
lhcbStyle.SetLabelFont(lhcbFont,"z")
lhcbStyle.SetLabelSize(lhcbTSize,"x")
lhcbStyle.SetLabelSize(lhcbTSize,"y")
lhcbStyle.SetLabelSize(lhcbTSize,"z")
lhcbStyle.SetLabelFont(lhcbFont, "x")
lhcbStyle.SetLabelFont(lhcbFont, "y")
lhcbStyle.SetLabelFont(lhcbFont, "z")
lhcbStyle.SetLabelSize(lhcbTSize, "x")
lhcbStyle.SetLabelSize(lhcbTSize, "y")
lhcbStyle.SetLabelSize(lhcbTSize, "z")
lhcbStyle.SetTitleFont(lhcbFont)
lhcbStyle.SetTitleFont(lhcbFont,"x")
lhcbStyle.SetTitleFont(lhcbFont,"y")
lhcbStyle.SetTitleFont(lhcbFont,"z")
lhcbStyle.SetTitleSize(1.2*lhcbTSize,"x")
lhcbStyle.SetTitleSize(1.2*lhcbTSize,"y")
lhcbStyle.SetTitleSize(1.2*lhcbTSize,"z")
lhcbStyle.SetTitleFont(lhcbFont, "x")
lhcbStyle.SetTitleFont(lhcbFont, "y")
lhcbStyle.SetTitleFont(lhcbFont, "z")
lhcbStyle.SetTitleSize(1.2 * lhcbTSize, "x")
lhcbStyle.SetTitleSize(1.2 * lhcbTSize, "y")
lhcbStyle.SetTitleSize(1.2 * lhcbTSize, "z")
# set the paper & margin sizes
lhcbStyle.SetPaperSize(20,26)
lhcbStyle.SetPaperSize(20, 26)
lhcbStyle.SetPadTopMargin(0.05)
lhcbStyle.SetPadRightMargin(0.08) # increase for colz plots
lhcbStyle.SetPadRightMargin(0.08) # increase for colz plots
lhcbStyle.SetPadBottomMargin(0.16)
lhcbStyle.SetPadLeftMargin(0.14)
# use medium bold lines and thick markers
lhcbStyle.SetLineWidth(lhcbWidth);
lhcbStyle.SetFrameLineWidth(lhcbWidth);
lhcbStyle.SetHistLineWidth(lhcbWidth);
lhcbStyle.SetFuncWidth(lhcbWidth);
lhcbStyle.SetGridWidth(lhcbWidth);
lhcbStyle.SetLineStyleString(2,"[12 12]"); # postscript dashes
lhcbStyle.SetMarkerStyle(20);
lhcbStyle.SetMarkerSize(1.0);
lhcbStyle.SetLineWidth(lhcbWidth)
lhcbStyle.SetFrameLineWidth(lhcbWidth)
lhcbStyle.SetHistLineWidth(lhcbWidth)
lhcbStyle.SetFuncWidth(lhcbWidth)
lhcbStyle.SetGridWidth(lhcbWidth)
lhcbStyle.SetLineStyleString(2, "[12 12]")
# postscript dashes
lhcbStyle.SetMarkerStyle(20)
lhcbStyle.SetMarkerSize(1.0)
# label offsets
lhcbStyle.SetLabelOffset(0.010,"X");
lhcbStyle.SetLabelOffset(0.010,"Y");
lhcbStyle.SetLabelOffset(0.010, "X")
lhcbStyle.SetLabelOffset(0.010, "Y")
# by default, do not display histogram decorations:
lhcbStyle.SetOptStat(0)
#lhcbStyle.SetOptStat("emr") # show only nent -e , mean - m , rms -r
# full opts at http:#root.cern.ch/root/html/TStyle.html#TStyle:SetOptStat
lhcbStyle.SetStatFormat("6.3g") # specified as c printf options
lhcbStyle.SetStatFormat("6.3g") # specified as c printf options
lhcbStyle.SetOptTitle(0)
lhcbStyle.SetOptFit(0)
#lhcbStyle.SetOptFit(1011) # order is probability, Chi2, errors, parameters
#titles
lhcbStyle.SetTitleOffset(0.85,"X")
lhcbStyle.SetTitleOffset(0.85,"Y")
lhcbStyle.SetTitleOffset(1.2,"Z")
lhcbStyle.SetTitleOffset(0.85, "X")
lhcbStyle.SetTitleOffset(0.85, "Y")
lhcbStyle.SetTitleOffset(1.2, "Z")
lhcbStyle.SetTitleFillColor(0)
lhcbStyle.SetTitleStyle(0)
lhcbStyle.SetTitleBorderSize(0)
lhcbStyle.SetTitleFont(lhcbFont,"title")
lhcbStyle.SetTitleFont(lhcbFont, "title")
lhcbStyle.SetTitleX(0.0)
lhcbStyle.SetTitleY(1.0)
lhcbStyle.SetTitleW(1.0)
......@@ -96,8 +98,8 @@ def setLHCbStyle() :
lhcbStyle.SetPadTickY(1)
# histogram divisions: only 5 in x to avoid label overlaps
lhcbStyle.SetNdivisions(505,"x")
lhcbStyle.SetNdivisions(510,"y")
lhcbStyle.SetNdivisions(505, "x")
lhcbStyle.SetNdivisions(510, "y")
gROOT.SetStyle("lhcbStyle")
return
......@@ -3,20 +3,20 @@ import itertools
# Some convenience function to easily iterate over the parts of the collections
# Needed if importing this script from another script in case TMultiGraphs are used
#ROOT.SetMemoryPolicy(ROOT.kMemoryStrict)
# Start a bit right of the Yaxis and above the Xaxis to not overlap with the ticks
start, stop = 0.18, 0.89
x_width, y_width = 0.3, 0.2
PLACES = [(start, stop - y_width, start + x_width, stop), # top left opt
(start, start, start + x_width, start + y_width), # bottom left opt
(stop - x_width, stop - y_width, stop, stop), # top right opt
(stop - x_width, start, stop, start + y_width), # bottom right opt
(stop - x_width, 0.5 - y_width / 2, stop, 0.5 + y_width / 2), # right
(start, 0.5 - y_width / 2, start + x_width, 0.5 + y_width / 2)] # left
PLACES = [
(start, stop - y_width, start + x_width, stop), # top left opt
(start, start, start + x_width, start + y_width), # bottom left opt
(stop - x_width, stop - y_width, stop, stop), # top right opt
(stop - x_width, start, stop, start + y_width), # bottom right opt
(stop - x_width, 0.5 - y_width / 2, stop, 0.5 + y_width / 2), # right
(start, 0.5 - y_width / 2, start + x_width, 0.5 + y_width / 2)
] # left
def transform_to_user(canvas, x1, y1, x2, y2):
......@@ -53,7 +53,8 @@ def overlap_h(hist, x1, y1, x2, y2):
if y1 <= val <= y2:
return True
# Errors
if val + hist.GetBinErrorUp(i) > y1 and val - hist.GetBinErrorLow(i) < y2:
if val + hist.GetBinErrorUp(i) > y1 and val - hist.GetBinErrorLow(
i) < y2:
# print "Overlap with histo", hist.GetName(), "at bin", i
return True
return False
......@@ -67,6 +68,7 @@ def overlap_rect(rect1, rect2):
return False
return True
def overlap_g(graph, x1, y1, x2, y2):
x_values = list(graph.GetX())
y_values = list(graph.GetY())
......@@ -80,7 +82,14 @@ def overlap_g(graph, x1, y1, x2, y2):
return True
return False
def place_legend(canvas, x1=None, y1=None, x2=None, y2=None, header="", option="LP"):
def place_legend(canvas,
x1=None,
y1=None,
x2=None,
y2=None,
header="",
option="LP"):
# If position is specified, use that
if all(x is not None for x in (x1, x2, y1, y2)):
return canvas.BuildLegend(x1, y1, x2, y2, header, option)
......@@ -101,10 +110,12 @@ def place_legend(canvas, x1=None, y1=None, x2=None, y2=None, header="", option="
# Make sure there are no overlaps
if any(obj.Overlap(*place_user) for obj in objects):
continue
return canvas.BuildLegend(place[0], place[1], place[2], place[3], header, option)
return canvas.BuildLegend(place[0], place[1], place[2], place[3],
header, option)
# As a fallback, use the default values, taken from TCanvas::BuildLegend
return canvas.BuildLegend(0.5, 0.67, 0.88, 0.88, header, option)
def find_place(canvas, x1=None, y1=None, x2=None, y2=None):
# Make sure all objects are correctly registered
canvas.Update()
......@@ -124,7 +135,8 @@ def find_place(canvas, x1=None, y1=None, x2=None, y2=None):
continue
return place
return PLACES[3] # fallback solution
return PLACES[3] # fallback solution
# Monkey patch ROOT objects to make it all work
ROOT.THStack.__iter__ = lambda self: iter(self.GetHists())
......
......@@ -7,16 +7,27 @@ import operator
import csv
from group_algos import group_algos
from optparse import OptionParser
"""
Produces a plot of the performance breakdown of the sequence under execution
"""
def main(argv):
global final_msg
parser = OptionParser()
parser.add_option('-d', '--dir', dest='output_directory', help='The directory to scan for build_* directories')
parser.add_option('-f', '--file_pattern', dest='file_pattern', default='profiler_output.txt',
help='The file name to look for profiler data in each build_ directoy. default: profiler_output.txt')
parser.add_option(
'-d',
'--dir',
dest='output_directory',
help='The directory to scan for build_* directories')
parser.add_option(
'-f',
'--file_pattern',
dest='file_pattern',
default='profiler_output.txt',
help=
'The file name to look for profiler data in each build_ directoy. default: profiler_output.txt'
)
(options, args) = parser.parse_args()
......@@ -40,7 +51,7 @@ def main(argv):
dirs.append(file)
for dir in dirs:
filepath = options.output_directory +"/" + dir + "/" + options.file_pattern
filepath = options.output_directory + "/" + dir + "/" + options.file_pattern
try:
f = open(filepath)
s = f.read()
......@@ -68,7 +79,6 @@ def main(argv):
print('Failed to read runtime from output')
print(traceback.format_exc())
# Regexp for one line
# Note: An algorithm line looks like:
# 11.08% 6.47377s 700 9.2482ms 3.7639ms 15.887ms lf_search_uv_windows(unsigned int const *, unsigned int const *, int const *, SciFi::TrackHits const *, int const *, char const *, LookingForward::Constants const *, float const *, MiniState const *, short*)
......@@ -88,11 +98,12 @@ def main(argv):
for k in algorithm_times.keys():
algorithm_times[k] = 100 * algorithm_times[k] / full_addition
output_list = sorted(algorithm_times.items(), key=operator.itemgetter(1), reverse=True)
output_list = sorted(
algorithm_times.items(), key=operator.itemgetter(1), reverse=True)
print(output_list)
output_path = options.output_directory +"/" + dir + "/algo_breakdown.csv"
output_path = options.output_directory + "/" + dir + "/algo_breakdown.csv"
with open(output_path, 'w') as out:
csv_out = csv.writer(out)
for row in output_list:
......@@ -101,12 +112,12 @@ def main(argv):
timings = group_algos(algorithm_times)
print(timings)
output_path = options.output_directory +"/" + dir + "/algo_summary.csv"
output_path = options.output_directory + "/" + dir + "/algo_summary.csv"
with open(output_path, 'w') as out:
csv_out = csv.writer(out)
for row in timings:
csv_out.writerow(row)
if __name__ == "__main__":
main(sys.argv[1:])
......@@ -2,27 +2,65 @@
import operator
def group_algos(algorithm_times):
# Algorithms of each sequence
velo_algorithms = ["consolidate_velo_tracks", "copy_velo_track_hit_number", "estimate_input_size", "masked_velo_clustering", "calculate_phi_and_sort", "search_by_triplet", "fill_candidates", "weak_tracks_adder", "copy_and_prefix_sum_single_block"]
pv_algorithms = ["pv_beamline_peak", "pv_beamline_multi_fitter", "pv_beamline_histo", "pv_beamline_extrapolate"]
ut_algorithms = ["consolidate_ut_tracks", "copy_ut_track_hit_number", "ut_decode_raw_banks_in_order", "ut_pre_decode", "ut_find_permutation", "ut_calculate_number_of_hits", "compass_ut", "ut_search_windows"]
velo_algorithms = [
"consolidate_velo_tracks", "copy_velo_track_hit_number",
"estimate_input_size", "masked_velo_clustering",