cmt/requirements

@@ -3,7 +3,7 @@
 # Maintainer : Ben Couturier
 #============================================================================
 package           PRConfig
-version           v1r64
+version           v1r65
 
 #============================================================================
 # Structure, i.e. directories to process.

doc/release.notes

@@ -4,6 +4,59 @@
 ! Purpose     : App Configuration for performance and regression tests
 !-----------------------------------------------------------------------------
 
+========================= PRConfig v1r65 2023-12-07 =========================
+
+! 2023-12-05 - commit 2fd9c22
+
+ - Merge branch 'lugrazet-BW-hlt1-fix-loggingerror' into 'master'
+
+   [RTA BW Tests] quick fix for Hlt1 BW test. len(inputs) < 2 breaks logging
+   string
+
+   See merge request lhcb-datapkg/PRConfig!363
+
+! 2023-12-05 - commit ee5e840
+
+ - Merge branch 'spruce-bw-input-Dec23' into 'master'
+
+   Add new Dec2023 samples for Sprucing PR tests
+
+   See merge request lhcb-datapkg/PRConfig!364
+
+! 2023-11-29 - commit f4c873b
+
+ - Merge branch 'rjhunter-bwtest-cleanup-copies' into 'master'
+
+   [RTA BW tests] Small cleanups after !359
+
+   See merge request lhcb-datapkg/PRConfig!362
+
+! 2023-11-28 - commit 6f6c963
+
+ - Merge branch 'rjhunter-reduce-moore-threads-in-bw-test' into 'master'
+
+   [Bandwidth tests] Use LBN_BUILD_JOBS to properly set n_threads on Moore in BW
+   test
+
+   See merge request lhcb-datapkg/PRConfig!356
+
+! 2023-11-16 - commit 11208b4
+
+ - Merge branch 'rjhunter-chained-test-feasibility' into 'master'
+
+   [RTA BW tests] Test feasibility of copying HLT2 output to read into sprucing
+   test
+
+   See merge request lhcb-datapkg/PRConfig!359
+
+! 2023-11-08 - commit ac9460c
+
+ - Merge branch 'lugrazet-BW-hlt1testpage-cleanup' into 'master'
+
+   [RTA BW Tests] BW test page clean-ups
+
+   See merge request lhcb-datapkg/PRConfig!355
+
 ========================= PRConfig v1r64 2023-11-01 =========================
 
 ! 2023-11-01 - commit f7f0f10

python/MooreTests/download_hlt2_output.py

@@ -27,7 +27,7 @@ DEFAULT_CACHE_DIRS = {'default': '.'}
 # prefer XDG_RUNTIME_DIR which should be on tmpfs
 FALLBACK_CACHE_DIR = os.getenv('XDG_RUNTIME_DIR', tempfile.gettempdir())
 
-FILE_TO_COPY = "mdf:root://eoslhcb.cern.ch//eos/lhcb/storage/lhcbpr/www/UpgradeRateTest/hlt2_bw_testing__production__full.mdf"
+FILE_TO_COPY = "mdf:root://eoslhcb.cern.ch//eos/lhcb/storage/lhcbpr/www/UpgradeRateTest/current_hlt2_output/hlt2_bw_testing__production__full.mdf"
 
 
 def default_cache_dirs():

python/MooreTests/run_bandwidth_test_jobs.py

@@ -27,6 +27,7 @@ import tempfile
 import atexit
 import shutil
 import yaml
+from datetime import datetime
 
 # Default cache dir is the current working directory as this is most convenient for the machine
 # that the test runs on periodically. It assumes the working directory is not cleaned up often,
@@ -36,9 +37,6 @@ DEFAULT_CACHE_DIRS = {'default': ['.']}
 # prefer XDG_RUNTIME_DIR which should be on tmpfs
 FALLBACK_CACHE_DIR = os.getenv('XDG_RUNTIME_DIR', tempfile.gettempdir())
 
-# Limit size of output log if many options files
-MAX_NFILES_TO_PRINT_TO_LOG = 10
-
 
 def default_cache_dirs():
     hostname = socket.getfqdn()
@@ -170,7 +168,6 @@ if __name__ == '__main__':
     job_inputs = [
         inputs_fns
     ]  # This is a list to allow for possible NUMA extension: see discussion on !316.
-    logging.info(inputs_fns[:MAX_NFILES_TO_PRINT_TO_LOG])
 
     # Set up local directories where inputs are cached
     if args.download_input_files:
@@ -190,22 +187,25 @@ if __name__ == '__main__':
 
         # Now download files
         for i, inputs in enumerate(job_inputs):
-            logging.info(
-                f'Downloading input files {inputs[:MAX_NFILES_TO_PRINT_TO_LOG]}'
-            )
             if all(is_remote(url) for url in inputs):
                 from Moore.qmtest.context import download_mdf_inputs_locally
                 # download_mdf_inputs_locally only downloads if files
                 # are not already available locally on the machine
+                before_copy = datetime.now()
+                logging.info(
+                    f'Downloading inputs for bandwidth job to {args.cache_dirs[i]}'
+                )
                 logging.info(
-                    'Downloading inputs for bandwidth job to {}'.format(
-                        args.cache_dirs[i]))
+                    f'There are {len(inputs)} input files: [{inputs[0]} ' +
+                    ']' if len(inputs) < 2 else '{inputs[1]}, ... ]')
                 kB_to_GB = 1e3
                 job_inputs[i] = download_mdf_inputs_locally(
                     inputs,
                     args.cache_dirs[i],
                     max_size=args.avg_evt_size * kB_to_GB * args.events)
-                logging.info(inputs)
+                logging.info(
+                    f"Finished file downloads. This took: {datetime.now() - before_copy}"
+                )
             elif any(is_remote(url) for url in inputs_fns):
                 parser.error('inputs must either be all xrootd or all local')
             else:

python/PRConfig/TestFileDB.py

@@ -13483,3 +13483,27 @@ testfiles(
     },
     comment='Real data selected by HLT1 from run 269939.',
     test_file_db=test_file_db)
+
+testfiles(
+    myname='upgrade-minbias-hlt2-full-output-Dec2023',
+    filenames=[
+        "mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/hlt2_full_stream_Dec2023/MagDown/hlt2_full_stream_{0}.mdf"
+        .format(i) for i in list(range(35)) + list(range(36, 240))
+    ] + [
+        "mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/dpa/wp1/hlt2_full_stream_Dec2023/MagUp/hlt2_full_stream_{0}.mdf"
+        .format(i) for i in list(range(117)) + list(range(118, 135))
+    ],
+    qualifiers={
+        'Author': 'Shunan Zhang',
+        'Format': 'MDF',
+        'DataType': 'Upgrade',
+        'Date': '2023-12-01',
+        'Simulation': True,
+        "CondDB": "sim-20171127-vc-md100",
+        "GeometryVersion": "run3/trunk",
+        "ConditionsVersion": "master",
+        "DDDB": "dddb-20171126",
+    },
+    comment=
+    'Hlt2 Full stream output as of December 2023, used as inputs for Sprucing tests, HLT2 output rate around 128 kHz',
+    test_file_db=test_file_db)

scripts/benchmark-scripts/Moore_bandwidth_test.sh

@@ -119,7 +119,7 @@ case $PROCESS in
     ;;
     hlt2)
     N_EVTS=1e5
-    MOORE_THREADS=$(nproc)
+    MOORE_THREADS=${LBN_BUILD_JOBS:-1} # Default to single-threaded
     TEST_PATH_PREFIX='$HLT2CONFROOT/tests/options/bandwidth/'
     EVENT_SIZE_UPPER_LIMIT=200
     GAUDIRUN_INPUT_PROCESS="Hlt2"
@@ -141,7 +141,7 @@ case $PROCESS in
     ;;
     spruce)
     N_EVTS=1e5
-    MOORE_THREADS=$(nproc)
+    MOORE_THREADS=${LBN_BUILD_JOBS:-1} # Default to single-threaded
     TEST_PATH_PREFIX='$HLT2CONFROOT/tests/options/bandwidth/'
     EVENT_SIZE_UPPER_LIMIT=300
     GAUDIRUN_INPUT_PROCESS="Spruce"

scripts/benchmark-scripts/Moore_spruce_all_lines.sh

@@ -22,9 +22,9 @@ fi
 export THOR_JIT_N_SPLITS=8
 export THOR_JIT_N_JOBS=8
 
-python -m MooreTests.run_throughput_jobs -n=2e4 --avg-event-size=300000 --test-file-db-key=upgrade-minbias-hlt2-full-output-Aug2023 '$MOOREROOT/tests/options/disable-git-metadata-write.py' '$HLT2CONFROOT/options/sprucing/spruce_all_lines.py' "${cache_dirs[@]}"
+python -m MooreTests.run_throughput_jobs -n=2e4 --avg-event-size=300000 --test-file-db-key=upgrade-minbias-hlt2-full-output-Dec2023 '$MOOREROOT/tests/options/disable-git-metadata-write.py' '$HLT2CONFROOT/options/sprucing/spruce_all_lines.py' "${cache_dirs[@]}"
 
-python -m MooreTests.run_throughput_jobs -n=-1 -j 1 --profile --avg-event-size=300000 --test-file-db-key=upgrade-minbias-hlt2-full-output-Aug2023 '$MOOREROOT/tests/options/disable-git-metadata-write.py' '$HLT2CONFROOT/options/sprucing/spruce_all_lines.py' "${cache_dirs[@]}"
+python -m MooreTests.run_throughput_jobs -n=-1 -j 1 --profile --avg-event-size=300000 --test-file-db-key=upgrade-minbias-hlt2-full-output-Dec2023 '$MOOREROOT/tests/options/disable-git-metadata-write.py' '$HLT2CONFROOT/options/sprucing/spruce_all_lines.py' "${cache_dirs[@]}"
 
 # force 0 return code so the handler runs even for failed jobs
 exit 0