From b18359a88fcddd7e508f084da2e587ce89f98b51 Mon Sep 17 00:00:00 2001
From: Giovanna Lazzari Miotto <giovanna.lazzari.miotto@cern.ch>
Date: Mon, 19 Feb 2024 12:49:47 +0100
Subject: [PATCH] ref: ci: Invert test outcome default to fail
---
scripts/test_filedma.sh | 69 +++++++++++++++++++++++++++++++----------
1 file changed, 52 insertions(+), 17 deletions(-)
diff --git a/scripts/test_filedma.sh b/scripts/test_filedma.sh
index deb0dde0..0e2ec99e 100755
--- a/scripts/test_filedma.sh
+++ b/scripts/test_filedma.sh
@@ -92,7 +92,7 @@ function run_filedma_test() {
output_dir=$(awk -F":" '/^output_filename_base:/{printf $2}' $config_file)
output_prefix=$(awk -F":" '/^output_filename_prefix:/{printf $2}' $config_file)
- local test_failed=0
+ local test_failed=1
echo "${FUNCNAME}: starting SCDAQ test with ${test_id} P5 capture file"
echo "${FUNCNAME}: timeout ${timeout_secs}s ./${SCDAQ_PATH} --config ${config_file} --nstreams ${num_streams}"
timeout ${timeout_secs}s ./"${SCDAQ_PATH}" --config ${config_file} --nstreams ${num_streams} | uniq -uc
@@ -102,7 +102,6 @@ function run_filedma_test() {
# We expect a TIMEOUT failure because `scdaq` is not normally supposed to return.
if [[ "${ret_status}" -ne 124 ]]; then
echo "${FUNCNAME}: SCDAQ test for ${test_id} FAILED with exit code ${ret_status}!"
- test_failed=1
# Check if output file was correctly generated
elif [[ ${expects_output} = true ]]; then
# Find file matching name pattern under the output directory, and excluding input data files
@@ -125,25 +124,42 @@ function run_filedma_test() {
# Hash-verify contents
if check_hashes ${bytes_to_hash} ${output_file} ${ref_hash_file} ; then
echo "${FUNCNAME}: SCDAQ test PASSED for ${test_id}"
+ test_failed=0
else
echo "${FUNCNAME}: SCDAQ test for ${test_id} FAILED"
- test_failed=1
fi
fi
+ else
+ # Execution ended by timeout, and not checking for output files
+ test_failed=0
fi
return ${test_failed}
}
+function echo_bool () {
+ var=$1
+ if [[ -z $var || $var -eq 1 ]]; then
+ echo "X"
+ else
+ echo "Ü"
+ fi
+}
+
function run_all_tests () {
local test_description=$1
local expects_output=${EXPECTS_OUTPUT}
declare -i timeout_secs=${MAX_RUNTIME_SECS}
readonly test_description timeout_secs expects_output
- local gmt_conf_failed gmt_json5_failed calo_conf_failed calo_json5_failed
- local passthrough_conf_failed passthrough_json5_failed multistream_conf_failed
+ local gmt_conf_failed=1 gmt_json5_failed=1
+ local calo_conf_failed=1 calo_json5_failed=1
+ local passthrough_conf_failed=1 passthrough_json5_failed=1
+ local multistream_conf_failed=1 multistream_json5_failed=1
echo "${FUNCNAME}: running ${test_description}"
+ failure_statuses=(gmt_conf_failed calo_conf_failed passthrough_conf_failed)
+
+ # Legacy configuration files
run_filedma_test "GMT" ${timeout_secs} 0 "${CONFIG_DIR}/scdaq-gmt.conf"
gmt_conf_failed=$?
run_filedma_test "CALO" ${timeout_secs} 0 "${CONFIG_DIR}/scdaq-calo.conf"
@@ -151,6 +167,9 @@ function run_all_tests () {
run_filedma_test "PASS0_GMT" ${timeout_secs} 0 "${CONFIG_DIR}/scdaq-passthrough.conf"
passthrough_conf_failed=$?
+ # JSON5 configuration files
+ failure_statuses+=(gmt_json5_failed calo_json5_failed passthrough_json5_failed)
+
run_filedma_test "GMT" ${timeout_secs} 0 "${CONFIG_DIR}/filedma-gmt.json5"
gmt_json5_failed=$?
run_filedma_test "CALO" ${timeout_secs} 0 "${CONFIG_DIR}/filedma-calo.json5"
@@ -159,6 +178,7 @@ function run_all_tests () {
passthrough_json5_failed=$?
if [[ ${expects_output} = false ]]; then
+ failure_statuses+=(multistream_conf_failed multistream_json5_failed)
# We evaluate multi-stream operation with two parallel pipelines
run_filedma_test "MULTI_PASS" ${timeout_secs} 2 "${CONFIG_DIR}/scdaq-passthrough.conf"
multistream_conf_failed=$?
@@ -166,24 +186,37 @@ function run_all_tests () {
multistream_json5_failed=$?
fi
- local failed_any=0
- echo "${FUNCNAME}: error report for ${test_description} (0 = SUCCESS)"
+ declare -i some_failed=0
+ for status in "${failure_statuses[@]}"; do
+ if [[ ! ${!status} =~ ^[01]$ ]]; then
+ echo "${FUNCNAME}: ${test_description} ===> ERROR: Invalid return value for $status"
+ some_failed=1
+ fi
+ if (( $status != 0 )); then
+ echo "${FUNCNAME}: ${test_description} ===> FAILED ($status)!"
+ some_failed=1
+ fi
+ done
+
+ echo "${FUNCNAME}: error report for ${test_description} (Ü = SÜCCESS)"
echo " conf | json5"
- echo "-- GMT : ${gmt_conf_failed} | ${gmt_json5_failed}"
- echo "-- Calo : ${calo_conf_failed} | ${calo_json5_failed}"
- echo "-- Passthrough : ${passthrough_conf_failed} | ${passthrough_json5_failed}"
+ echo "-- GMT : $(echo_bool ${gmt_conf_failed}) | $(echo_bool ${gmt_json5_failed} )"
+ echo "-- Calo : $(echo_bool ${calo_conf_failed}) | $(echo_bool ${calo_json5_failed})"
+ echo "-- Passthrough : $(echo_bool ${passthrough_conf_failed}) | $(echo_bool ${passthrough_json5_failed})"
if [[ ${expects_output} = false ]]; then
- echo "-- Multi-stream: ${multistream_conf_failed} | ${multistream_json5_failed}"
- failed_any=${multistream_conf_failed}+${multistream_json5_failed}
+ echo "-- Multi-stream: $(echo_bool ${multistream_conf_failed}) | $(echo_bool ${multistream_json5_failed})"
fi
- failed_any=${failed_any}+${gmt_json5_failed}+${gmt_conf_failed}+${calo_json5_failed}+${calo_conf_failed}+${passthrough_json5_failed}+${passthrough_conf_failed}
- if [[ $failed_any -ne 0 ]]; then
- echo "${FUNCNAME}: ${test_description} ===> FAILED!"
- exit 1
- else
+ if (( $some_failed == 0 )); then
echo "${FUNCNAME}: ${test_description} ===> SUCCESS"
fi
+ return $some_failed
+}
+
+function exit_on_failure {
+ if [[ $? -ne 0 ]]; then
+ exit 1
+ fi
}
function main {
@@ -193,9 +226,11 @@ function main {
# Test 1: time-bound operation, checks for crashes
MODIFY_CONFIG=false EXPECTS_OUTPUT=false run_all_tests "Test 1 (operation) - ${MAX_RUNTIME_SECS}-second runs, no output files"
+ exit_on_failure $?
# Test 2: validates file output for basic runs
MODIFY_CONFIG=true EXPECTS_OUTPUT=true run_all_tests "Test 2 (regression) - ${MAX_RUNTIME_SECS}-second runs, check output files"
+ exit_on_failure $?
echo "${FUNCNAME}: all tests finished"
}
--
GitLab