[RTADPA BW Tests] New Sprucing bandwidth test using latest-available HLT2 input.

cc: @rjhunter.

related to: lhcb/Moore!2763 (merged) + lhcb-core/LHCbPR2HD!270 (merged) + lhcb-core/LHCbNightlyConf!1121 (merged)

Changes:

Adds a new sprucing test, Moore_spruce_latest_bandwidth, which uses the Hlt2 nominal full-stream output which was uploaded to eos on the previous day.

Added a new script, generate_hlt2_fullstream_metadata.py, which runs only for Hlt2 nominal tests and writes the metadata.yaml based on information from the full-stream output of the test currently running. This metadata.yaml is used as an input config for the new sprucing test.

• a little safety around division in zero from the calculation of overlap_matrix, never comes up in the actual tests because every stream has a non-zero retention in 100k events, but when running test jobs it can come up and it was annoying me.

• changes to facilitate the new test + metadata.

ToDo:

• Maybe revisit the N_EVTS and calculate via a gaudipython job?
  • Wait to see if BWTesters know a way to tell the N_EVTS available in a file.
  • Else, adapt list_event_numbers to extract the N_EVTS by iterating a gaudipython job.
• Rebase after !362 (merged) was merged

Closes #12 (closed)

scripts/benchmark-scripts/Moore_bandwidth_test.sh

@@ -22,8 +22,9 @@ Usage: Moore/run /path/to/Moore_bandwidth_test.sh [options] 2>&1 | tee <path-to-
        Expected to be called by e.g. Moore_hlt2_bandwidth.sh for the periodic LHCbPR tests.
 
 --process: "hlt1", "hlt2" or "spruce".
---input-data: "nominal" or "2023".
+--input-data: "nominal", "2023" or "latest".
     "2023" not currently available for process == spruce or hlt1.
+    "latest" not currently available for process == hlt1 or hlt2.
 -h|--help: print this message and exit.
 
 EOF
@@ -107,7 +108,7 @@ case $PROCESS in
     STREAM_CONFIGS=( "streamless" )
     case $INPUTDATA in
         nominal)
-        N_EVTS=1e4
+        EVTMAX=1e4
         CONFIG_FILE="${TEST_PATH_PREFIX}hlt1_bandwidth_input.yaml"
         EXTRA_OPTS="-e 1 $MOOREROOT/options/muon_geometry_v2.py" #Requires #EvtSlots==1 due to singlethreading on DIGIs.
         ;;
@@ -118,7 +119,7 @@ case $PROCESS in
     esac
     ;;
     hlt2)
-    N_EVTS=1e5
+    EVTMAX=1e5
     MOORE_THREADS=${LBN_BUILD_JOBS:-1} # Default to single-threaded
     TEST_PATH_PREFIX='$HLT2CONFROOT/tests/options/bandwidth/'
     EVENT_SIZE_UPPER_LIMIT=200
@@ -140,7 +141,7 @@ case $PROCESS in
     esac
     ;;
     spruce)
-    N_EVTS=1e5
+    EVTMAX=1e5
     MOORE_THREADS=${LBN_BUILD_JOBS:-1} # Default to single-threaded
     TEST_PATH_PREFIX='$HLT2CONFROOT/tests/options/bandwidth/'
     EVENT_SIZE_UPPER_LIMIT=300
@@ -151,8 +152,14 @@ case $PROCESS in
         CONFIG_FILE="${TEST_PATH_PREFIX}spruce_bandwidth_input.yaml"
         EXTRA_OPTS='$MOOREROOT/options/muon_geometry_v2.py'
         ;;
+        latest)
+        # "latest" corresponds to using the uploaded full-stream output from a "process=hlt2, input-data=nominal" test.
+        # These files are overwritten during "lhcb-master" builds of "process=hlt2, input-data=nominal", i.e. Daily.
+        CONFIG_FILE="tmp/hlt2_bw_testing__production__full__metadata.yaml"
+        EXTRA_OPTS='-um --read-evt-max-from-config $MOOREROOT/options/muon_geometry_v2.py'
+        ;;
         *)
-        echo "ERROR: --input-data must be \"nominal\" for process \"$PROCESS\""
+        echo "ERROR: --input-data must be \"nominal\" or \"latest\" for process \"$PROCESS\""
         exit 1
         ;;
     esac
@@ -167,15 +174,21 @@ esac
 
 # 1. Run Moore.
 # -d downloads the input files locally for speed-up running Moore. Not helpful unless that download is fast for you (e.g. you're at CERN)
+if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "latest" ]; then
+    echo "Downloading the Hlt2 output metadata to use as input config."
+    DOWNLOAD_INPUT_CONFIG_LOCATION=(`python -c "from PRConfig.bandwidth_helpers import FileNameHelper; hlpr = FileNameHelper('hlt2'); print( hlpr.metadata_prwww_path(stream_config='production', stream='full') )"`)
+    # xrdcp -f $DOWNLOAD_INPUT_CONFIG_LOCATION $CONFIG_FILE
+    cp -f "/storage/epp2/phrhbm/modern_stack/tmp_spruce_prevhlt2/hlt2_bw_testing__production__full__metadata.yaml" $CONFIG_FILE
+    STORE_ERR_CODE
+fi
 for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
     echo "Running trigger to obtain MDF files with ${STREAM_CONFIG} streams for comparison over ${CONFIG_FILE}"
-    time python -m MooreTests.run_bandwidth_test_jobs -d -c=$CONFIG_FILE -n=$N_EVTS -t=$MOORE_THREADS -a=$EVENT_SIZE_UPPER_LIMIT $EXTRA_OPTS "${TEST_PATH_PREFIX}${PROCESS}_bandwidth_${STREAM_CONFIG}_streams.py"
+    time python -m MooreTests.run_bandwidth_test_jobs -d -c=$CONFIG_FILE -n=$EVTMAX -p=$PROCESS -t=$MOORE_THREADS -a=$EVENT_SIZE_UPPER_LIMIT $EXTRA_OPTS "${TEST_PATH_PREFIX}${PROCESS}_bandwidth_${STREAM_CONFIG}_streams.py"
     STORE_ERR_CODE
 done
 
 # 2. Compute line descriptives: persist reco, extra output
-if [ $PROCESS = "hlt1" ]
-then
+if [ $PROCESS = "hlt1" ]; then
     echo 'Skipping line descriptives as $PROCESS = "hlt1"'
 else
     echo 'Obtaining line descriptives'
@@ -191,14 +204,13 @@ for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
     echo "Found ${STREAM_CONFIG} streams: ${STREAMS[@]}"
 
     # 4. Compute similarity matrices between streams by comparing event numbers
-    if [ $PROCESS = "hlt1" ]
-    then
+    if [ $PROCESS = "hlt1" ]; then
         echo 'Skipping similarity matrix per stream as $PROCESS = "hlt1"'
     else
         echo "Obtaining similarity matrix for ${STREAM_CONFIG}-stream configuration"
         for stream in "${STREAMS[@]}"; do
             echo "Stream name: ${stream}"
-            time python $PRCONFIGROOT/python/MooreTests/list_event_numbers.py -p $PROCESS -n $N_EVTS --stream-config $STREAM_CONFIG --stream $stream
+            time python $PRCONFIGROOT/python/MooreTests/list_event_numbers.py -p $PROCESS --stream-config $STREAM_CONFIG --stream $stream
             STORE_ERR_CODE
         done
         time python $PRCONFIGROOT/python/MooreTests/calculate_stream_overlap.py -p $PROCESS --stream-config $STREAM_CONFIG --streams ${STREAMS[@]}
@@ -209,7 +221,7 @@ for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
     echo "Obtaining rates and bandwidth for ${STREAM_CONFIG}-stream configuration"
     for stream in "${STREAMS[@]}"; do
         echo "Stream name: ${stream}"
-        time python $PRCONFIGROOT/python/MooreTests/line-and-stream-rates.py -c $CONFIG_FILE -n $N_EVTS -p $PROCESS -s $stream --stream-config $STREAM_CONFIG
+        time python $PRCONFIGROOT/python/MooreTests/line-and-stream-rates.py -c $CONFIG_FILE -p $PROCESS -s $stream --stream-config $STREAM_CONFIG
         STORE_ERR_CODE
     done
 done
@@ -220,12 +232,16 @@ time python $PRCONFIGROOT/python/MooreTests/combine_rate_output.py --process $PR
 STORE_ERR_CODE
 
 # 7. Test on the feasibility of a chained HLT2->Sprucing test
-if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "nominal" ]
-then
+if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "nominal" ]; then
     echo 'Testing downloads of Hlt2 output for the future'
     time python -m MooreTests.download_hlt2_output
+    STORE_ERR_CODE
+elif [ $PROCESS = "hlt2" ] && [ $INPUTDATA = "nominal" ]; then
+    # 7.1 Required information for 'latest' sprucing Jobs.
+    echo 'Generating metadata.yaml for potential upload to eos'
+    time python -m MooreTests.generate_hlt2_fullstream_metadata -c $CONFIG_FILE
+    STORE_ERR_CODE
 fi
-STORE_ERR_CODE
 
 # 8. Produce plots and HTML pages; add the --building-locally flag to make the links work if you are building the html pages locally
 echo 'Making plots and HTML pages'