Access N_EVTS via config online, export to overwrite it

python/MooreTests/generate_metadata.py

@@ -24,22 +24,15 @@
 '''
 import argparse
 import json
-import yaml
-import os
 from PRConfig.bandwidth_helpers import FileNameHelper
 from PRConfig.TestFileDB import test_file_db
+from MooreTests.run_bandwidth_test_jobs import parse_yaml
 
 LHCBPR_WWW_FLOC = "mdf:root://eoslhcb.cern.ch//eos/lhcb/storage/lhcbpr/www/UpgradeRateTest/hlt2_bw_testing__production{fsuffix}"
 HLT2CONF_WEB = "https://gitlab.cern.ch/lhcb/Moore/-/blob/master/Hlt/Hlt2Conf"
 
 
-def parse_yaml(file_path):
-    with open(os.path.expandvars(file_path), 'r') as f:
-        return yaml.safe_load(f)
-
-
 def main():
-
     parser = argparse.ArgumentParser()
     parser.add_argument(
         '-n',
@@ -66,8 +59,8 @@ def main():
     opts.append(
         f"filtering_info_links:\n  - {args.config.replace('$HLT2CONFROOT',HLT2CONF_WEB)}"
     )
-    opts.append(f"input_type: 'MDF'")
-    opts.append(f"Simulation: True")
+    opts.append("input_type: 'MDF'")
+    opts.append("Simulation: True")
 
     conds = test_file_db[config['testfiledb_key']].qualifiers
     opts.append(f"dddb_tag: {conds['DDDB']}")

python/MooreTests/run_bandwidth_test_jobs.py

@@ -39,6 +39,8 @@ FALLBACK_CACHE_DIR = os.getenv('XDG_RUNTIME_DIR', tempfile.gettempdir())
 # Limit size of output log if many options files
 MAX_NFILES_TO_PRINT_TO_LOG = 10
 
+CONFIG_TO_DOWNLAOD = "root://eoslhcb.cern.ch//eos/lhcb/storage/lhcbpr/www/UpgradeRateTest/hlt2_bw_testing__production__full__metadata.yaml"
+
 
 def default_cache_dirs():
     hostname = socket.getfqdn()
@@ -154,12 +156,22 @@ if __name__ == '__main__':
         help=
         "Flag to access and include config[input_manifest_file] as an extra option in the job."
     )
+    parser.add_argument(
+        '-dc',
+        '--download-config',
+        action='store_true',
+        help="Flag to download the config.yaml file from LHCbPR")
     args = parser.parse_args()
 
     logging.basicConfig(
         format='%(levelname)-7s %(message)s',
         level=(logging.DEBUG if args.debug else logging.INFO))
 
+    if args.process == "spruce" and args.download_config:
+        cmd = ["xrdcp", "-f", "-s", CONFIG_TO_DOWNLAOD, args.config]
+        config = parse_yaml(args.config)
+        os.system(f'export N_EVTS={config["n_evts"]}')
+
     if args.events == -1 or args.events > 1e5:
         raise RuntimeError(
             "The BW tests are limited to 1e5 events to keep them to a reasonable runtime. Please re-configure"

scripts/benchmark-scripts/Moore_bandwidth_test.sh

@@ -153,14 +153,12 @@ case $PROCESS in
         EXTRA_OPTS=''
         ;;
         previous-hlt2)
-        # 9.38% * 1e5 for FULL as of 21-Nov. -> Want to actually get this from yesterday's log somehow otherwise the rates will all be slightly wrong.
-        # "Found 9383 event numbers for full stream. Saved list to tmp/Output/hlt2__event_numbers__production__full.json." is in the log, should be able to use this :)
-        N_EVTS=1e4
-        CONFIG_FILE="${TEST_PATH_PREFIX}spruce_bandwidth_input_previous_hlt2.yaml"
-        EXTRA_OPTS='-um'
+        CONFIG_FILE="tmp/hlt2_bw_testing__production__full__metadata.yaml"
+        N_EVTS=1 # To be overwritten by run_bandwidth_test_jobs
+        EXTRA_OPTS='-um -dc'
         ;;
         *)
-        echo "ERROR: --input-data \"$INPUTDATA\" must be \"nominal\" or \"previous-hlt2\" for process \"$PROCESS\""
+        echo "ERROR: --input-data must be \"nominal\" or \"previous-hlt2\" for process \"$PROCESS\""
         exit 1
         ;;
     esac
@@ -177,10 +175,11 @@ esac
 # -d downloads the input files locally for speed-up running Moore. Not helpful unless that download is fast for you (e.g. you're at CERN)
 for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
     echo "Running trigger to obtain MDF files with ${STREAM_CONFIG} streams for comparison over ${CONFIG_FILE}"
-    time python -m MooreTests.run_bandwidth_test_jobs -d -c=$CONFIG_FILE -n=$N_EVTS -t=$MOORE_THREADS -a=$EVENT_SIZE_UPPER_LIMIT $EXTRA_OPTS "${TEST_PATH_PREFIX}${PROCESS}_bandwidth_${STREAM_CONFIG}_streams.py"
+    time python -m MooreTests.run_bandwidth_test_jobs -c=$CONFIG_FILE -n=$N_EVTS -t=$MOORE_THREADS -a=$EVENT_SIZE_UPPER_LIMIT $EXTRA_OPTS "${TEST_PATH_PREFIX}${PROCESS}_bandwidth_${STREAM_CONFIG}_streams.py"
     STORE_ERR_CODE
 done
 
+
 # 2. Compute line descriptives: persist reco, extra output
 if [ $PROCESS = "hlt1" ]
 then
@@ -217,7 +216,7 @@ for STREAM_CONFIG in "${STREAM_CONFIGS[@]}"; do
     echo "Obtaining rates and bandwidth for ${STREAM_CONFIG}-stream configuration"
     for stream in "${STREAMS[@]}"; do
         echo "Stream name: ${stream}"
-        time python $PRCONFIGROOT/python/MooreTests/line-and-stream-rates.py -c $CONFIG_FILE -n $N_EVTS -p $PROCESS -s $stream --stream-config $STREAM_CONFIG
+        time python $PRCONFIGROOT/python/MooreTests/line-and-stream-rates.py -d -c $CONFIG_FILE -n $N_EVTS -p $PROCESS -s $stream --stream-config $STREAM_CONFIG
         STORE_ERR_CODE
     done
 done
@@ -232,6 +231,9 @@ if [ $PROCESS = "spruce" ] && [ $INPUTDATA = "nominal" ]
 then
     echo 'Testing downloads of Hlt2 output for the future'
     time python -m MooreTests.download_hlt2_output
+elif [ $PROCESS = "hlt2" ] && [ $INPUTDATA = "nominal" ]
+    echo 'Generating metadata.yaml for potential upload to eos'
+    time python -m MooreTests.generate_metadata -c $CONFIG_FILE -n $N_EVTS
 fi
 STORE_ERR_CODE