Improve post-processing of log files

Trigger/TrigValidation/TrigUpgradeTest/share/newJO_build.ref

+Storing config in the config newJOtest.pkl
 Py:Athena            INFO now loading newJOtest.pkl  ... 
 TrigSignatureMo...   INFO Chains passing step (1st row events & 2nd row decision counts
 TrigSignatureMo...   INFO Chain name                   L1,      AfterPS, [... steps ...], Output

Trigger/TrigValidation/TrigUpgradeTest/test/exec_TrigUpgradeTest_art_athenaMT.sh

@@ -54,16 +54,18 @@ else
   exit 1
 fi
 
-trap 'PREVIOUS_COMMAND=$THIS_COMMAND; THIS_COMMAND=$BASH_COMMAND' DEBUG
-
 ######################################
 
 if [[ ${FROMPICKLE} == "1" ]]; then
-  echo "Running athena from pickle file ${JOBOPTION}"
+  echo "Running athena from pickle file ${JOBOPTION} with the command:"
+  (set -x
   athena.py \
   ${MATHLIBOPT} \
   ${JOBOPTION} &> ${JOB_LOG}
+  )
 else
+  echo "Running athena command:"
+  (set -x
   athena.py \
   ${MATHLIBOPT} \
   --threads ${THREADS} \
@@ -73,18 +75,12 @@ else
   --skipEvents ${SKIPEVENTS} \
   -c "${EXTRA}" \
   ${JOBOPTION} &> ${JOB_LOG}
+  )
 fi
 
 ######################################
 
-COMMAND=$PREVIOUS_COMMAND ATH_RETURN=$?
-echo ${COMMAND} > command.txt
-echo "Command to reproduce:"
-envsubst < command.txt
+export ATH_RETURN=$?
 echo "art-result: ${ATH_RETURN} ${JOB_LOG%%.*}"
 echo  $(date "+%FT%H:%M %Z")"     Done executing Athena test ${NAME} with result ${ATH_RETURN}"
-if [ "${ATH_RETURN}" -ne "0" ]; then
-  exit ${ATH_RETURN}
-fi
-
 

Trigger/TrigValidation/TrigUpgradeTest/test/exec_TrigUpgradeTest_art_post.sh

@@ -7,8 +7,22 @@
 
 echo  $(date "+%FT%H:%M %Z")"     Execute TrigUpgradeTest post processing for test ${NAME}"
 
+### Find the branch name
+if [ -n "${AtlasBuildBranch}" ]; then
+  export BRANCH=${AtlasBuildBranch} # available after asetup
+elif [ -n "${gitlabTargetBranch}" ]; then
+  export BRANCH=${gitlabTargetBranch} # available in CI
+else
+  echo "WARNING Cannot determine the branch name, both variables AtlasBuildBranch and gitlabTargetBranch are empty"
+fi
+
 ### DEFAULTS
 
+if [ -z ${ATH_RETURN} ]; then
+  echo "WARNING The env variable ATH_RETURN is not set, assuming 0"
+  export ATH_RETURN=0
+fi
+
 if [ -z ${JOB_LOG} ]; then
   export JOB_LOG="athena.log"
 fi
@@ -19,12 +33,12 @@ fi
 
 if [ -z ${REF_FOLDER} ]; then
   # Try eos first
-  export REF_FOLDER="/eos/atlas/atlascerngroupdisk/data-art/grid-input/${TEST}/ref/${AtlasBuildBranch}/test_${NAME}"
+  export REF_FOLDER="/eos/atlas/atlascerngroupdisk/data-art/grid-input/${TEST}/ref/${BRANCH}/test_${NAME}"
   # If not available, try cvmfs
   if [ -d ${REF_FOLDER} ]; then
     echo "Using reference directory from eos: ${REF_FOLDER}"
   else
-    export REF_FOLDER="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/${TEST}/ref/${AtlasBuildBranch}/test_${NAME}"
+    export REF_FOLDER="/cvmfs/atlas-nightlies.cern.ch/repo/data/data-art/${TEST}/ref/${BRANCH}/test_${NAME}"
     if [ -d ${REF_FOLDER} ]; then
       echo "Reference from eos unavailable, using one from cvmfs: ${REF_FOLDER}"
     else
@@ -43,19 +57,25 @@ if [ -z "${REGTESTREF}" ]; then
   export REGTESTREF=${REF_FOLDER}/athena.regtest
 fi
 
-###
+### CHECKLOG
+
+# if athena failed and we are running in CI, print the full log to stdout
+if [ "${ATH_RETURN}" -ne "0" ] && [ -n "${gitlabTargetBranch}" ]; then
+  echo "Printing the full ${JOB_LOG}"
+  cat ${JOB_LOG}
+fi
 
 echo $(date "+%FT%H:%M %Z")"     Running checklog"
 timeout 1m check_log.pl --config checklogTrigUpgradeTest.conf --showexcludestats ${JOB_LOG} | tee checklog.log
 
 echo "art-result: ${PIPESTATUS[0]} CheckLog"
 
-###
+### MAKE LOG TAIL FILE
 
 export JOB_LOG_TAIL=${JOB_LOG%%.*}.tail.${JOB_LOG#*.}
 tail -10000  ${JOB_LOG} > ${JOB_LOG_TAIL}
 
-###
+### REGTEST
 
 REGTESTREF_BASENAME=$(basename -- "${REGTESTREF}")
 grep -E "${REGTESTEXP}" ${JOB_LOG} > "${REGTESTREF_BASENAME}"
@@ -80,7 +100,7 @@ else
   echo "art-result:  999 RootComp"
 fi
 
-###
+### SUMMARY
 
 echo  $(date "+%FT%H:%M %Z")"     Files in directory:"
 ls -lh

Trigger/TrigValidation/TrigUpgradeTest/test/test_newJO_build.sh

@@ -27,12 +27,23 @@ status=$?
 if [ ${status} -ne 0 ]; then
     echo "ERROR in configuration generation stage, stopping"
     echo "art-result: 1 Configuration"
-    exit 1
+    export ATH_RETURN=1
+    export JOB_LOG="athena.pickle.log"
 else
     echo
     echo "JOs reading stage finished, launching Athena from pickle file"
     echo "art-result: 0 Configuration"
     echo 
     source exec_TrigUpgradeTest_art_athenaMT.sh
-    source exec_TrigUpgradeTest_art_post.sh
+
+    # Merge log files for post-processing
+    JOB_LOG_MERGED="athena.merged.log"
+    echo "### athena.pickle.log ###" > ${JOB_LOG_MERGED}
+    cat athena.pickle.log >> ${JOB_LOG_MERGED}
+    echo "### ${JOB_LOG} ###"
+    cat ${JOB_LOG} >> ${JOB_LOG_MERGED}
+    export JOB_LOG=${JOB_LOG_MERGED}
 fi
+
+source exec_TrigUpgradeTest_art_post.sh
+