From 225148faeae993af7d5a4f69737f6c7f05f08393 Mon Sep 17 00:00:00 2001 From: David Southwick Date: Wed, 15 Jun 2022 15:50:41 +0200 Subject: [PATCH 1/8] initial build step using cvmfs common script --- cms/mlpf/Dockerfile.append | 32 +++------ cms/mlpf/cms-mlpf.spec | 2 +- cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh | 99 ++++++++++++++++++++-------- cms/mlpf/cms-mlpf/prepare-dataset.sh | 8 +-- 4 files changed, 84 insertions(+), 57 deletions(-) diff --git a/cms/mlpf/Dockerfile.append b/cms/mlpf/Dockerfile.append index 8837f2a0..dd54f87c 100644 --- a/cms/mlpf/Dockerfile.append +++ b/cms/mlpf/Dockerfile.append @@ -1,28 +1,14 @@ -# Set the environment -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 - -# Install some basic utilities -RUN apt update && apt install -y jq && apt clean && rm -rf /var/lib/apt/lists/* - -# Prepare the workspace folder -RUN mkdir -p /workspace -ENV PATH=/workspace:$PATH - # Get the source code for MLPF -COPY particleflow /workspace/particleflow +COPY particleflow /bmk/cms-mlpf/particleflow # Copy in the dependencies -COPY cms-mlpf/requirements.txt /workspace/ -COPY cms-mlpf/cms-mlpf-bmk.sh /workspace/ -COPY cms-mlpf/prepare-dataset.sh /workspace/ +COPY cms-mlpf/requirements.txt cms-mlpf/cms-mlpf-bmk.sh cms-mlpf/prepare-dataset.sh /bmk/cms-mlpf/ # Install dependencies -RUN python3 -m pip install --upgrade pip==21.3.1 setuptools==59.5.0 -RUN python3 -m pip install --no-cache-dir -r /workspace/requirements.txt -RUN python3 -m pip install --no-cache-dir /workspace/particleflow/hep_tfds - -# Download and pre-process the dataset -RUN bash /workspace/prepare-dataset.sh - -ENTRYPOINT ["/workspace/cms-mlpf-bmk.sh"] +RUN \ + # cd /bmk/cms-mlpf && git submodule update --init --recursive + python3 -m pip install --upgrade pip==21.3.1 setuptools==59.5.0 && \ + python3 -m pip install --no-cache-dir -r /bmk/cms-mlpf/requirements.txt && \ + python3 -m pip install --no-cache-dir /bmk/cms-mlpf/particleflow/hep_tfds && \ + # Download and pre-process the dataset + bash /bmk/cms-mlpf/prepare-dataset.sh diff --git a/cms/mlpf/cms-mlpf.spec b/cms/mlpf/cms-mlpf.spec index c59321d4..16695849 100644 --- a/cms/mlpf/cms-mlpf.spec +++ b/cms/mlpf/cms-mlpf.spec @@ -5,4 +5,4 @@ HEPWL_BMKDESCRIPTION="" HEPWL_DOCKERIMAGENAME=cms-mlpf-bmk HEPWL_DOCKERIMAGETAG=v0.1 HEPWL_CVMFSREPOS=sft.cern.ch -HEPWL_BMKOS="tensorflow/tensorflow:2.6.0-gpu" +HEPWL_BMKOS="gitlab-registry.cern.ch/dsouthwi/tensorflow/tensorflow-gpu:latest" diff --git a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh index 793a1f48..698611a7 100755 --- a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh +++ b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh @@ -14,44 +14,85 @@ NTRAIN=0 # 0 is None NTEST=0 # 0 is None BSIZE=0 # 0 is Default NDEVICES=0 # 0 is Default -RESULTSDIR=/results +DEBUG=0 +resultsDir="/results" +skipSubDir=0 +MOP="none" DESCRIPTION="Machine Learning Particle Flow (MLPF) benchmark" log() { case $1 in - error) shift 1; echo -e "\e[31m>>> ERROR:\e[0m $*\n" | tee -a $RESULTSDIR/out.log ; exit 2 ;; - info) shift 1; echo -e "\e[34m$*\e[0m\n" | tee -a $RESULTSDIR/out.log ;; - silent) shift 1; echo "$*" >> $RESULTSDIR/out.log ;; - *) echo "$*" | tee -a $RESULTSDIR/out.log ; + error) shift 1; echo -e "\e[31m>>> ERROR:\e[0m $*\n" | tee -a $resultsDir/out.log ; exit 2 ;; + info) shift 1; echo -e "\e[34m$*\e[0m\n" | tee -a $resultsDir/out.log ;; + silent) shift 1; echo "$*" >> $resultsDir/out.log ;; + *) echo "$*" | tee -a $resultsDir/out.log ; esac } -usage() { - echo "Usage: $0 [options]" - echo "Options:" - echo "-h, --help Prints this message and exit." - echo "-w, --resultsdir Results directory. Default: $RESULTSDIR" - echo "-e, --nepochs Number of epochs. Default: $NEPOCHS" - echo "-B, --batch_size Batch size per device. Default: $BSIZE" - echo "-D, --num_devices Number of devices to use. Default: $NDEVICES" - echo " --ntrain Train steps limit. Default: $NTRAIN" - echo " --ntest Test steps limit. Default: $NTEST" - exit 0 +function usage(){ + echo "" + echo "Usage: $0 [-w | --resultsdir ] [-W] [-c | --copies ] [-n | --nepochs ] " \ + "[-B | --batch_size ] [-D | --num_devices ] [--ntrain ] [--ntest ] " \ + "[-m | --mop ] [-d | --debug] [-h | --help]" + echo " -w --resultsdir : (path) results directory (default: /results , current: $resultsDir)" + echo " -W : store results in directly" + echo " -n --nepochs : (int) Number of epochs (default: 1, current: $NEPOCHS)" + echo " -B --batch_size : (int) Batch size per device (default: 0, current: $BSIZE)" + echo " -D --num_devices : (int) Number of devices to use (default: 0, current: $NDEVICES)" + echo " --ntrain : (int) Train steps limit (default: 0, current: $NTRAIN)" + echo " --ntest : (int) Test steps limit (default: 0, current: $NTEST)" + echo " -m --mop : (none|all|custom) clean working directory mode: none/all/custom (current: $MOP)" + echo " -d --debug : debug mode" + echo " -h --help : display this help and exit" + echo "" + echo "Mop mode: + none == do not remove working files, + all == remove all produced files (but summary json), + custom == custom implementation" + echo "Without -W (default): results are stored in a new subdirectory of :" + echo " //*.json" + echo " //proc_1/*.log" + echo " //proc_.../*.log" + echo " //proc_/*.log" + echo "With -W (e.g. in the CI): results are stored in directly:" + echo " /*.json" + echo " /proc_1/*.log" + echo " /proc_.../*.log" + echo " /proc_/*.log" + echo "" + echo "Without -w (default) and without -W: is /results" + echo "Without -w (default) and with -W: is a tmp directory /tmp/xxxx" + echo "" + if [ "$(type -t usage_detailed)" == "function" ]; then + echo -e "\nDetailed Usage:\n----------------\n" + ( usage_detailed ) # as a subprocess, just in case this has a 0 exit code... + fi + echo -e "DESCRIPTION\n" + if [ -e $BMKDIR/DESCRIPTION ]; then + cat $BMKDIR/DESCRIPTION + else + echo "Sorry there is no description included." + fi + echo "" + exit 2 # early termination (help or invalid arguments to benchmark script) } parse_args() { - options=$(getopt --long resultsdir:,nepochs:,ntrain:,ntest:,batch_size:,num_devices:,help -o wWeDB:h -- "$@") + options=$(getopt --long resultsdir:,nepochs:,ntrain:,ntest:,batch_size:,num_devices:,help,mop -o wWmnDB:h -- "$@") if [ $? != 0 ]; then echo "Invalid options provided." >&2; usage; fi eval set -- "$options" while true; do case $1 in --help | -h ) usage; exit 0;; - --resultsdir | -w ) RESULTSDIR=$2; shift ;; - --ntrain ) NTRAIN=$2; shift ;; - --ntest ) NTEST=$2; shift ;; - --nepochs | -e ) NEPOCHS=$2; shift ;; - --num_devices | -D ) NDEVICES=$2; shift ;; - --batch_size | -B ) BSIZE=$2; shift ;; + --debug | -d ) DEBUG=1; shift ;; + -W ) skipSubDir=1; shift;; + --mop | -m ) MOP=$2; shift 2;; + --resultsdir | -w ) resultsDir=$2; shift 2;; + --ntrain ) NTRAIN=$2; shift 2;; + --ntest ) NTEST=$2; shift 2;; + --nepochs | -n ) NEPOCHS=$2; shift 2;; + --num_devices | -D ) NDEVICES=$2; shift 2;; + --batch_size | -B ) BSIZE=$2; shift 2;; -- ) shift; break;; esac shift @@ -60,23 +101,23 @@ parse_args() { parse_args $* -if [ -f "$RESULTSDIR"/out.log ]; then rm "$RESULTSDIR"/out.log; fi -log info "Base working directory: $RESULTSDIR" +if [ -f "$resultsDir"/out.log ]; then rm "$resultsDir"/out.log; fi +log info "Base working directory: $resultsDir" log info "Running benchmark MLPF" -cd /workspace/particleflow/ +cd /bmk/cms-mlpf/particleflow/ python3 mlpf/pipeline.py train \ --config parameters/delphes-benchmark.yaml \ --prefix /tmp/train_ \ --plot-freq 1000000 \ - --benchmark_dir $RESULTSDIR \ + --benchmark_dir $resultsDir \ --num_devices $NDEVICES \ --batch_size $BSIZE \ --nepochs $NEPOCHS \ --ntrain $NTRAIN \ --ntest $NTEST -REPORT=$(cat $RESULTSDIR/result.json) +REPORT=$(cat $resultsDir/result.json) generate_json() { jq -n \ @@ -96,5 +137,5 @@ generate_json() { }' } -generate_json > $RESULTSDIR/mlpf-report.json +generate_json > $resultsDir/mlpf-report.json log info "Finished running MLPF" diff --git a/cms/mlpf/cms-mlpf/prepare-dataset.sh b/cms/mlpf/cms-mlpf/prepare-dataset.sh index 9543bedd..0f876447 100755 --- a/cms/mlpf/cms-mlpf/prepare-dataset.sh +++ b/cms/mlpf/cms-mlpf/prepare-dataset.sh @@ -5,12 +5,12 @@ set -e # each split is about 50MB pickled and gzipped MIRROR=https://zenodo.org/record/4559324/files -DOWNLOAD_DIR=/workspace/zenodo +DOWNLOAD_DIR=/bmk/data/zenodo DELPHES_DIR=$DOWNLOAD_DIR/delphes_pf echo "Download directory: $DOWNLOAD_DIR" -PF_DIR=/workspace/particleflow -DATA_DIR=/workspace/tensorflow_datasets +PF_DIR=/bmk/cms-mlpf/particleflow +DATA_DIR=/bmk/cms-mlpf/tensorflow_datasets echo "Data directory: $DATA_DIR" # create the download dir @@ -32,6 +32,6 @@ done # build TDFS datasets cd $PF_DIR -tfds build hep_tfds/heptfds/delphes_pf --download_dir $DOWNLOAD_DIR --data_dir /workspace/tensorflow_datasets +tfds build hep_tfds/heptfds/delphes_pf --download_dir $DOWNLOAD_DIR --data_dir $DATA_DIR rm -rf $DOWNLOAD_DIR -- GitLab From dc9e6cefc2f817fba5561c926470caca9c6d9628 Mon Sep 17 00:00:00 2001 From: David Southwick Date: Wed, 15 Jun 2022 17:15:38 +0200 Subject: [PATCH 2/8] add debug, missing opts --- cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh index 698611a7..06a6cd8f 100755 --- a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh +++ b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh @@ -78,28 +78,30 @@ function usage(){ } parse_args() { - options=$(getopt --long resultsdir:,nepochs:,ntrain:,ntest:,batch_size:,num_devices:,help,mop -o wWmnDB:h -- "$@") + options=$(getopt --long resultsdir:,nepochs:,ntrain:,ntest:,batch_size:,num_devices:,debug,help,mop -o w:Wm:n:dD:B:h -- "$@") if [ $? != 0 ]; then echo "Invalid options provided." >&2; usage; fi eval set -- "$options" while true; do - case $1 in + case "$1" in --help | -h ) usage; exit 0;; - --debug | -d ) DEBUG=1; shift ;; - -W ) skipSubDir=1; shift;; - --mop | -m ) MOP=$2; shift 2;; - --resultsdir | -w ) resultsDir=$2; shift 2;; - --ntrain ) NTRAIN=$2; shift 2;; - --ntest ) NTEST=$2; shift 2;; - --nepochs | -n ) NEPOCHS=$2; shift 2;; - --num_devices | -D ) NDEVICES=$2; shift 2;; - --batch_size | -B ) BSIZE=$2; shift 2;; + --debug | -d ) DEBUG=1 ;; + -W ) skipSubDir=1;; + --mop | -m ) MOP=$2; shift;; + --resultsdir | -w ) resultsDir=$2; shift;; + --ntrain ) NTRAIN=$2; shift;; + --ntest ) NTEST=$2; shift;; + --nepochs | -n ) NEPOCHS=$2; shift;; + --num_devices | -D ) NDEVICES=$2; shift;; + --batch_size | -B ) BSIZE=$2; shift;; -- ) shift; break;; esac shift done } -parse_args $* +# TODO: implement MOP, DEBUG, skipSubDir + +parse_args "$*" if [ -f "$resultsDir"/out.log ]; then rm "$resultsDir"/out.log; fi log info "Base working directory: $resultsDir" -- GitLab From 51230f0ae0b448e70f9e7f6c7bcb1ab7cb60cacf Mon Sep 17 00:00:00 2001 From: David Southwick Date: Tue, 21 Jun 2022 16:05:30 +0200 Subject: [PATCH 3/8] set env, work still TODO for yaml in upstream --- .gitlab-ci.yml | 2 ++ cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 328b9452..61f8708c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -270,6 +270,8 @@ build-cms-mlpf: - hep-workload-gpu-docker-builder variables: CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/cms/mlpf/cms-mlpf.spec + before_script: + - submodule update --init --recursive only: variables: - $CI_COMMIT_BRANCH == "qa-build-cms-mlpf" diff --git a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh index 06a6cd8f..bf524c6b 100755 --- a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh +++ b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh @@ -105,6 +105,12 @@ parse_args "$*" if [ -f "$resultsDir"/out.log ]; then rm "$resultsDir"/out.log; fi log info "Base working directory: $resultsDir" + +# set CUDA_VISIBLE_DEVICES for tensorflow +DEVICES=$(nvidia-smi -L | wc -l) +log info "Detected $DEVICES nvidia GPUs" +export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $(($DEVICES-1))) + log info "Running benchmark MLPF" cd /bmk/cms-mlpf/particleflow/ -- GitLab From 46acfea5777820ac0afc5a977a121fea5a8c409a Mon Sep 17 00:00:00 2001 From: David Southwick Date: Wed, 22 Jun 2022 13:22:45 +0200 Subject: [PATCH 4/8] move particleflow origin --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 38f1b8ab..540c3766 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "cms/mlpf/particleflow"] path = cms/mlpf/particleflow - url = https://github.com/edcuba/particleflow + url = https://github.com/dcsouthwick/particleflow + branch = benchmark_suite -- GitLab From aedeaa98a8cfdfcabb2781516a0743344ff6001f Mon Sep 17 00:00:00 2001 From: David Southwick Date: Wed, 22 Jun 2022 13:39:11 +0200 Subject: [PATCH 5/8] update particleflow --- cms/mlpf/particleflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/mlpf/particleflow b/cms/mlpf/particleflow index 0f1d1ead..4a29fedd 160000 --- a/cms/mlpf/particleflow +++ b/cms/mlpf/particleflow @@ -1 +1 @@ -Subproject commit 0f1d1ead7a3286e06c00a9663125e17f450b8903 +Subproject commit 4a29feddbd90632094b55e151624eb96efc78223 -- GitLab From c51dcb7a5ef2214a5deb5c181a76922edae997c8 Mon Sep 17 00:00:00 2001 From: David Southwick Date: Wed, 22 Jun 2022 15:37:38 +0200 Subject: [PATCH 6/8] update particleflow submodule --- cms/mlpf/particleflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/mlpf/particleflow b/cms/mlpf/particleflow index 4a29fedd..9324c962 160000 --- a/cms/mlpf/particleflow +++ b/cms/mlpf/particleflow @@ -1 +1 @@ -Subproject commit 4a29feddbd90632094b55e151624eb96efc78223 +Subproject commit 9324c9625555addd9367133193a3c491fbc74376 -- GitLab From 6474b125229b3793a096c0c3a00682c3f35ff0b4 Mon Sep 17 00:00:00 2001 From: David Southwick Date: Mon, 27 Jun 2022 14:20:18 +0200 Subject: [PATCH 7/8] add CUDA check, move results json to subfolder --- cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh index bf524c6b..f81a0b0b 100755 --- a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh +++ b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh @@ -106,10 +106,12 @@ parse_args "$*" if [ -f "$resultsDir"/out.log ]; then rm "$resultsDir"/out.log; fi log info "Base working directory: $resultsDir" -# set CUDA_VISIBLE_DEVICES for tensorflow -DEVICES=$(nvidia-smi -L | wc -l) -log info "Detected $DEVICES nvidia GPUs" -export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $(($DEVICES-1))) +# set CUDA_VISIBLE_DEVICES for tensorflow based on nvidia-smi (dirty nvidia-only check) +if type -P "nvidia-smi" &>/dev/null; then + DEVICES=$(nvidia-smi -L | wc -l) + log info "Detected $DEVICES nvidia GPUs" + export CUDA_VISIBLE_DEVICES=$(seq -s, 0 $(($DEVICES-1))) +fi log info "Running benchmark MLPF" @@ -144,6 +146,6 @@ generate_json() { } }' } - -generate_json > $resultsDir/mlpf-report.json +mkdir -p $resultsDir/report +generate_json > $resultsDir/report/cms-mlpf_summary.json log info "Finished running MLPF" -- GitLab From ca41dcfd1597096f32eb79a29b538edd5a7f747e Mon Sep 17 00:00:00 2001 From: David Southwick Date: Mon, 27 Jun 2022 16:24:11 +0200 Subject: [PATCH 8/8] add -W arg, fix default to epoch 2> --- cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh | 44 ++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh index f81a0b0b..a8a56ba7 100755 --- a/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh +++ b/cms/mlpf/cms-mlpf/cms-mlpf-bmk.sh @@ -9,10 +9,10 @@ else FLAVOR="unknown"; fi # Default config -NEPOCHS=1 +NEPOCHS=2 NTRAIN=0 # 0 is None NTEST=0 # 0 is None -BSIZE=0 # 0 is Default +BSIZE=4 # 0 is Default NDEVICES=0 # 0 is Default DEBUG=0 resultsDir="/results" @@ -78,7 +78,7 @@ function usage(){ } parse_args() { - options=$(getopt --long resultsdir:,nepochs:,ntrain:,ntest:,batch_size:,num_devices:,debug,help,mop -o w:Wm:n:dD:B:h -- "$@") + options=$(getopt -a -n cms-mlpf-bmk -o w:Wm:n:dD:B:h --long resultsdir:,nepochs:,ntrain:,ntest:,batch_size:,num_devices:,debug,help,mop -- "$@") if [ $? != 0 ]; then echo "Invalid options provided." >&2; usage; fi eval set -- "$options" while true; do @@ -86,22 +86,22 @@ parse_args() { --help | -h ) usage; exit 0;; --debug | -d ) DEBUG=1 ;; -W ) skipSubDir=1;; - --mop | -m ) MOP=$2; shift;; - --resultsdir | -w ) resultsDir=$2; shift;; - --ntrain ) NTRAIN=$2; shift;; - --ntest ) NTEST=$2; shift;; - --nepochs | -n ) NEPOCHS=$2; shift;; - --num_devices | -D ) NDEVICES=$2; shift;; - --batch_size | -B ) BSIZE=$2; shift;; + --mop | -m ) MOP="$2"; shift;; + --resultsdir | -w ) resultsDir="$2"; shift;; + --ntrain ) NTRAIN="$2"; shift;; + --ntest ) NTEST="$2"; shift;; + --nepochs | -n ) NEPOCHS="$2"; shift;; + --num_devices | -D ) NDEVICES="$2"; shift;; + --batch_size | -B ) BSIZE="$2"; shift;; -- ) shift; break;; esac shift done } -# TODO: implement MOP, DEBUG, skipSubDir +# TODO: implement MOP, DEBUG -parse_args "$*" +parse_args $* if [ -f "$resultsDir"/out.log ]; then rm "$resultsDir"/out.log; fi log info "Base working directory: $resultsDir" @@ -114,7 +114,16 @@ if type -P "nvidia-smi" &>/dev/null; then fi log info "Running benchmark MLPF" - +log silent "Executing 'python3 mlpf/pipeline.py train \ + --config parameters/delphes-benchmark.yaml \ + --prefix /tmp/train_ \ + --plot-freq 1000000 \ + --benchmark_dir $resultsDir \ + --num_devices $NDEVICES \ + --batch_size $BSIZE \ + --nepochs $NEPOCHS \ + --ntrain $NTRAIN \ + --ntest $NTEST'" cd /bmk/cms-mlpf/particleflow/ python3 mlpf/pipeline.py train \ --config parameters/delphes-benchmark.yaml \ @@ -147,5 +156,10 @@ generate_json() { }' } mkdir -p $resultsDir/report -generate_json > $resultsDir/report/cms-mlpf_summary.json -log info "Finished running MLPF" +if [ $skipSubDir -eq 0 ]; then + REPORT_PATH=$resultsDir/report/cms-mlpf_summary.json +else + REPORT_PATH=$resultsDir/cms-mlpf_summary.json +fi +generate_json > $REPORT_PATH +log info "Finished running MLPF. Final report written to $REPORT_PATH" -- GitLab