make_profile_plots.py

#!/usr/bin/env python
###############################################################################
# (c) Copyright 2019-2023 CERN for the benefit of the LHCb Collaboration      #
#                                                                             #
# This software is distributed under the terms of the GNU General Public      #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
#                                                                             #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization  #
# or submit itself to any jurisdiction.                                       #
###############################################################################
"""
Make flamegraph from profiling output and "FlameBars" from timing table in log file.

Example:
    Assuming a perf.data directory in current directory and a job log named Profile.log from a HLT1 job,
    you can run:
    make_profile_plots.py  -l 'HLT1' --logs Profile.log
"""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import numpy as np
import subprocess
import os
import argparse
import logging

DIR = os.path.dirname(os.path.abspath(__file__))
STACK_COLLAPSE_SCRIPT = os.path.join(DIR, "stackcollapse-perf.pl")
FLAME_GRAPH_SCRIPT = os.path.join(DIR, "flamegraph.pl")
HEAPTRACK_ARGS = [
    '-p', '0', '-a', '0', '-T', '0', '-F', 'ht_flamy', '--filter-bt-function',
    'BasicNode'
]


def main(listOfLogs, hltlabel, throughput, produceYAML, ht_file, heaptrack,
         perf_exe, no_inline):

    binary_tag = os.environ["BINARY_TAG"]
    if "centos7" in binary_tag.split("-"):
        llvm_cxxfilt_path = "/cvmfs/sft.cern.ch/lcg/contrib/clang/12/x86_64-centos7/bin/llvm-cxxfilt"
    elif "el9" in binary_tag.split("-"):
        llvm_cxxfilt_path = "/cvmfs/sft.cern.ch/lcg/releases/clang/14.0.6-14bdb/x86_64-centos9/bin/llvm-cxxfilt"
    else:
        raise RuntimeError(f"{binary_tag=} not supported")
    if os.path.isfile(llvm_cxxfilt_path):
        demangle = llvm_cxxfilt_path
    else:
        demangle = "cat"
        logging.warning(
            "Could not find llvm-cxxfilt command on cvmfs, some stack might not be properly demangled"
        )

    # call the perl scripts from https://github.com/brendangregg/FlameGraph
    # to create a flamegraph
    if perf_exe:
        if no_inline:
            run_perf_script = perf_exe + " script --no-inline -i ./perf.data"
        else:
            run_perf_script = perf_exe + " script -i ./perf.data"
        make_flamegraph = "{FLS} --hash --title '{hltlabel} Flame Graph' --minwidth 2 --width 1600 > flamy.svg".format(
            FLS=FLAME_GRAPH_SCRIPT, hltlabel=hltlabel)

        flamy_cmd = " | ".join([
            run_perf_script, STACK_COLLAPSE_SCRIPT, demangle, make_flamegraph
        ])
        logging.info(f"Creating flamegraph: {flamy_cmd!r}")
        subprocess.Popen(flamy_cmd, shell=True).wait()

    if ht_file:
        heaptrack_print = heaptrack + '_print'
        cmd = [heaptrack_print] + HEAPTRACK_ARGS + [ht_file]
        logging.info("Running heaptrack_print: " + " ".join(map(repr, cmd)))
        subprocess.check_call(cmd)
        with open("ht_flamy", 'r') as inp, open('ht_flamy.svg', 'w') as out:
            cmd = [
                FLAME_GRAPH_SCRIPT, '--hash', '--colors', 'mem', '--title',
                'Allocations of Algorithms (100 Events)', '--minwidth', '4',
                '--width', '1600', '--countname', 'Allocations'
            ]
            logging.info("Creating mem flamegraph: " +
                         " ".join(map(repr, cmd)))
            subprocess.check_call(cmd, stdin=inp, stdout=out)

    if not listOfLogs:
        return
    ## reads the text logs and extracts timing shares of different steps of the algorithm
    from MooreTests import readTimingTable
    timingTable = readTimingTable.readTimings(hltlabel, listOfLogs,
                                              produceYAML)

    # last entry in the sorted list will always be the total time so let's remove that one
    sortedList = [
        elem for elem in sorted(timingTable.items(), key=lambda item: item[1])
        if not (elem[0] == 'Total' or elem[1] < 0.1)
    ]

    # spread colors out over the hot_r colormap +1 makes sure last color isn't white
    colors = matplotlib.cm.hot_r(np.linspace(0., 1., len(sortedList) + 1))
    fig, ax = plt.subplots(figsize=(15, 10))

    pos = np.arange(len(sortedList)) + .5
    plt.barh(
        pos, [elem[1] for elem in sortedList], align='center', color=colors)
    plt.yticks(pos, [elem[0] for elem in sortedList], fontsize=18)
    plt.xticks(fontsize=18)

    #Add values to bars
    for i, elem in enumerate(sortedList):
        plt.text(
            elem[1],
            i + .5,
            '{0:.2f} '.format(elem[1]),
            va='center',
            color='black',
            fontweight='bold',
            fontsize=14)

    # make sure there is enough room to the right of the largest bar for the text
    plt.xlim(0, sortedList[-1][1] + 5)
    plt.xlabel(
        "Timing fraction within the " + hltlabel + " sequence [%]",
        fontsize=18)

    throughput_text = ''
    if throughput:
        if throughput >= 1e3:
            throughput = 1.e-3 * throughput
            tp_SI = 'k'
        else:
            tp_SI = ''

        throughput_text = r"{0} Throughput Rate {1:03.1f} {2}Hz".format(
            hltlabel, throughput, tp_SI)

    #Add LHCbSimulation & throughput text
    textStr = '\n'.join(("LHCb Simulation", '', throughput_text))
    ax.text(
        0.99,
        0.20,
        textStr,
        transform=ax.transAxes,
        fontsize=24,
        verticalalignment='top',
        horizontalalignment="right",
        bbox={
            'facecolor': 'white',
            'edgecolor': 'none'
        })

    try:
        # this only works in main
        if args.output_dir and args.output_dir[-1] != "/":
            args.output_dir = args.output_dir + "/"
        plt.savefig(f"{args.output_dir}FlameBars.png", bbox_inches='tight')
        plt.savefig(f"{args.output_dir}FlameBars.pdf", bbox_inches='tight')
    except NameError:
        plt.savefig("FlameBars.png", bbox_inches='tight')
        plt.savefig("FlameBars.pdf", bbox_inches='tight')


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument(
        '--logs',
        dest='logs',
        type=str,
        default=[],
        nargs='+',
        help='List of log files')
    parser.add_argument(
        '-l',
        '--hltlabel',
        dest='hltlabel',
        type=str,
        required=True,
        choices=['HLT1', 'HLT2', 'Sprucing'],
        help=
        'Pick HLT configuration for plot labels and checking of algorithms in log file)'
    )
    parser.add_argument(
        '-t',
        '--throughput',
        dest='throughput',
        type=float,
        required=False,
        help='Events/s throughput number to be added as label to FlameBars')

    parser.add_argument(
        '--produce-yaml',
        dest='produce_yaml',
        action='store_true',
        help=
        'Produce a YAML file with categorised list of algos/tools that make the throughput rate plot'
    )

    parser.add_argument(
        '--ht_file',
        dest='ht_file',
        type=str,
        default=None,
        help=
        'Optional file name of a Heaptrack profile to produce memory allocations flamegrap.'
    )

    parser.add_argument(
        '--heaptrack',
        type=str,
        required=False,
        help='Enable heaptrack profiling by providing path to executable')

    parser.add_argument(
        '--no-inline',
        action='store_true',
        help='Keep perf script from analysing the stack of inlined functions.')

    parser.add_argument('--perf-path', default='perf', help='Path to perf')

    parser.add_argument(
        '--output-dir',
        type=str,
        default="",
        help='Output directory for plots.')

    args = parser.parse_args()
    main(args.logs, args.hltlabel, args.throughput, args.produce_yaml,
         args.ht_file, args.heaptrack, args.perf_path, args.no_inline)