From 32a926bf5affee171324ae9b3473a4b56210187d Mon Sep 17 00:00:00 2001
From: Giovanna Lazzari Miotto <giovanna.lazzari.miotto@cern.ch>
Date: Mon, 19 Feb 2024 17:34:46 +0100
Subject: [PATCH] fix: test: Amend config file regex
JSON5 configuration files follow a slightly different syntax and were
silently not being pattern-matched to grep the output file name.
CI jobs still worked because the .dat suffix was hardcoded in the regex,
and the only .dat files were output files for the current execution.
The bug became evident now that suffixes are configurable in feature
branches.
---
scripts/test_filedma.sh | 14 +++++++-------
test/config/filedma-calo.json5 | 2 +-
test/config/filedma-gmt.json5 | 2 +-
test/config/filedma-passthrough.json5 | 2 +-
test/config/filedma-passthrough1.json5 | 2 +-
test/config/filedma-passthrough2.json5 | 2 +-
test/config/scdaq-calo.conf | 2 +-
test/config/scdaq-gmt-cmssw.conf | 2 +-
test/config/scdaq-gmt.conf | 2 +-
test/config/scdaq-micron-bril.conf | 2 +-
test/config/scdaq-passthrough.conf | 2 +-
test/config/scdaq-passthrough1.conf | 2 +-
test/config/scdaq-passthrough2.conf | 2 +-
test/config/tcp-test.conf | 2 +-
test/config/tcp-test1.conf | 2 +-
test/config/tcp-test2.conf | 2 +-
16 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/scripts/test_filedma.sh b/scripts/test_filedma.sh
index a3b72142..1fc54dc8 100755
--- a/scripts/test_filedma.sh
+++ b/scripts/test_filedma.sh
@@ -1,12 +1,10 @@
#!/bin/bash
-##
## File: test_filedma.sh
-## Author: glazzari
+## Author: @glazzari
## Description: runs file-based regression tests for GMT, Calo and Passthrough processors.
## Style: Google Shell Guide (https://google.github.io/styleguide/shellguide.html).
## Gold standard for regression: scdaq @ 67ddf197 from January 9th, 2024.
-##
readonly DATA_DIR=test/data
readonly CONFIG_DIR=test/config
@@ -89,9 +87,9 @@ function run_filedma_test() {
setup_test
- output_dir=$(awk -F":" '/^output_filename_base:/{printf $2}' $config_file)
- output_prefix=$(awk -F":" '/^output_filename_prefix:/{printf $2}' $config_file)
- output_suffix=$(awk -F":" '/^output_filename_suffix:/{printf $2}' $config_file)
+ output_dir=$(awk -F ':' '/^ *output_filename_base:/{printf $2}' ${config_file} | tr -d "\"',[:blank:]")
+ output_prefix=$(awk -F ':' '/^ *output_filename_prefix:/{printf $2}' ${config_file} | tr -d "\"',[:blank:]")
+ output_suffix=.dat
local test_failed=1
echo "${FUNCNAME}: starting SCDAQ test with ${test_id} P5 capture file"
@@ -107,7 +105,9 @@ function run_filedma_test() {
elif [[ ${expects_output} = true ]]; then
# Find file matching name pattern under the output directory, and excluding input data files
# In case of multiple files, sorts alphanumerically so the lower-numbered one comes first
- local output_files=$(find ${output_dir} -iname "*${output_prefix}*.${output_suffix}*" ! -iname "*testfile.dat" | sort -rV)
+ local pattern=".*${output_prefix}.*${output_suffix}[_0-9]*"
+ echo "${FUNCNAME}: searching for filename matching pattern: '${pattern}'"
+ local output_files=$(find ${output_dir} -type f -iregex "${pattern}" ! -iname "*testfile.dat" | sort -rV)
if [ -z "${output_files}" ]; then
echo "${FUNCNAME}: SCDAQ test for ${test_id} FAILED: output file not found!"
else
diff --git a/test/config/filedma-calo.json5 b/test/config/filedma-calo.json5
index c14c7562..9236dba6 100644
--- a/test/config/filedma-calo.json5
+++ b/test/config/filedma-calo.json5
@@ -34,7 +34,7 @@
// Output settings /////////////////////////////////////////////////
output_filename_prefix: "scout_CALO",
output_filename_base: "test/data",
- output_filename_suffix: "dat",
+ output_filename_suffix: ".raw",
max_file_size: 8589934592,
// Always write data to a file regardless of the run status
output_force_write: "yes",
diff --git a/test/config/filedma-gmt.json5 b/test/config/filedma-gmt.json5
index 287fe894..15866a02 100644
--- a/test/config/filedma-gmt.json5
+++ b/test/config/filedma-gmt.json5
@@ -34,7 +34,7 @@
// Output settings /////////////////////////////////////////////////
output_filename_prefix: "scout_GMT",
output_filename_base: "test/data",
- output_filename_suffix: "raw",
+ output_filename_suffix: ".raw",
max_file_size: 8589934592,
// Always write data to a file regardless of the run status
output_force_write: "yes",
diff --git a/test/config/filedma-passthrough.json5 b/test/config/filedma-passthrough.json5
index 38c0f9cd..54cd6e8c 100644
--- a/test/config/filedma-passthrough.json5
+++ b/test/config/filedma-passthrough.json5
@@ -34,7 +34,7 @@
// Output settings /////////////////////////////////////////////////
output_filename_prefix: "scout_PASS0_GMT",
output_filename_base: "test/data/run000000",
- output_filename_suffix: "dat",
+ output_filename_suffix: ".dat",
max_file_size: 8589934592,
// Always write data to a file regardless of the run status
output_force_write: "yes",
diff --git a/test/config/filedma-passthrough1.json5 b/test/config/filedma-passthrough1.json5
index cfb22da2..9000901b 100644
--- a/test/config/filedma-passthrough1.json5
+++ b/test/config/filedma-passthrough1.json5
@@ -34,7 +34,7 @@
// Output settings /////////////////////////////////////////////////
output_filename_prefix: "scout_PASS1_CALO",
output_filename_base: "test/data/run000000",
- output_filename_suffix: "dat",
+ output_filename_suffix: ".dat",
max_file_size: 8589934592,
// Always write data to a file regardless of the run status
output_force_write: "yes",
diff --git a/test/config/filedma-passthrough2.json5 b/test/config/filedma-passthrough2.json5
index 504d4323..256fd7a7 100644
--- a/test/config/filedma-passthrough2.json5
+++ b/test/config/filedma-passthrough2.json5
@@ -34,7 +34,7 @@
// Output settings /////////////////////////////////////////////////
output_filename_prefix: "scout_PASS2_GMT",
output_filename_base: "test/data/run000000",
- output_filename_suffix: "dat",
+ output_filename_suffix: ".dat",
max_file_size: 8589934592,
// Always write data to a file regardless of the run status
output_force_write: "yes",
diff --git a/test/config/scdaq-calo.conf b/test/config/scdaq-calo.conf
index f730a6e6..3772b06f 100644
--- a/test/config/scdaq-calo.conf
+++ b/test/config/scdaq-calo.conf
@@ -66,7 +66,7 @@ doZS:yes
output_filename_prefix:scout_CALO
output_filename_base:test/data
-output_filename_suffix:dat
+output_filename_suffix:.dat
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
diff --git a/test/config/scdaq-gmt-cmssw.conf b/test/config/scdaq-gmt-cmssw.conf
index c024dddd..975b3ad1 100644
--- a/test/config/scdaq-gmt-cmssw.conf
+++ b/test/config/scdaq-gmt-cmssw.conf
@@ -68,7 +68,7 @@ doZS:yes
output_filename_prefix:scout_GMT
output_filename_base:test/data
-output_filename_suffix:raw
+output_filename_suffix:.raw
max_file_size:8589934592
diff --git a/test/config/scdaq-gmt.conf b/test/config/scdaq-gmt.conf
index 6de088d3..bf01b830 100644
--- a/test/config/scdaq-gmt.conf
+++ b/test/config/scdaq-gmt.conf
@@ -68,7 +68,7 @@ doZS:yes
output_filename_prefix:scout_GMT
output_filename_base:test/data
-output_filename_suffix:dat
+output_filename_suffix:.dat
max_file_size:8589934592
diff --git a/test/config/scdaq-micron-bril.conf b/test/config/scdaq-micron-bril.conf
index a4a2157f..39967a79 100644
--- a/test/config/scdaq-micron-bril.conf
+++ b/test/config/scdaq-micron-bril.conf
@@ -68,7 +68,7 @@ doZS:yes
output_filename_prefix:scout_BRIL
output_filename_base:test/data
-output_filename_suffix:dat
+output_filename_suffix:.dat
max_file_size:8589934592
diff --git a/test/config/scdaq-passthrough.conf b/test/config/scdaq-passthrough.conf
index b946af7a..36300e76 100644
--- a/test/config/scdaq-passthrough.conf
+++ b/test/config/scdaq-passthrough.conf
@@ -67,7 +67,7 @@ doZS:yes
output_filename_prefix:scout_PASS0_GMT
output_filename_base:test/data/run000000
-output_filename_suffix:dat
+output_filename_suffix:.dat
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
diff --git a/test/config/scdaq-passthrough1.conf b/test/config/scdaq-passthrough1.conf
index 73a7f821..5dd57098 100644
--- a/test/config/scdaq-passthrough1.conf
+++ b/test/config/scdaq-passthrough1.conf
@@ -67,7 +67,7 @@ doZS:yes
output_filename_prefix:scout_PASS1_CALO
output_filename_base:test/data/run000000
-output_filename_suffix:dat
+output_filename_suffix:.dat
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
diff --git a/test/config/scdaq-passthrough2.conf b/test/config/scdaq-passthrough2.conf
index ab8a6bec..8c10c645 100644
--- a/test/config/scdaq-passthrough2.conf
+++ b/test/config/scdaq-passthrough2.conf
@@ -68,7 +68,7 @@ doZS:yes
output_filename_prefix:scout_PASS2_GMT
output_filename_base:test/data/run000000
-output_filename_suffix:dat
+output_filename_suffix:.dat
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
diff --git a/test/config/tcp-test.conf b/test/config/tcp-test.conf
index 38b935bf..1ef75aee 100644
--- a/test/config/tcp-test.conf
+++ b/test/config/tcp-test.conf
@@ -67,7 +67,7 @@ doZS:yes
output_filename_prefix:tcp-single-scouting_gmt_1
output_filename_base:test/in_progress
-output_filename_suffix:raw
+output_filename_suffix:.raw
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
diff --git a/test/config/tcp-test1.conf b/test/config/tcp-test1.conf
index 29220fc0..92f79289 100644
--- a/test/config/tcp-test1.conf
+++ b/test/config/tcp-test1.conf
@@ -67,7 +67,7 @@ doZS:yes
output_filename_prefix:tcp-mp1-scouting_calo_1
output_filename_base:test/in_progress
-output_filename_suffix:raw
+output_filename_suffix:.raw
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
diff --git a/test/config/tcp-test2.conf b/test/config/tcp-test2.conf
index 07530071..e109dace 100644
--- a/test/config/tcp-test2.conf
+++ b/test/config/tcp-test2.conf
@@ -67,7 +67,7 @@ doZS:yes
output_filename_prefix:tcp-mp2-scouting_gmt
output_filename_base:test/in_progress
-output_filename_suffix:raw_1
+output_filename_suffix:.raw_1
max_file_size:8589934592
# Always write data to a file regardless of the run status, useful for debugging
--
GitLab