diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index b52c16653bdda677e4a945fbb58a93f24fefe503..2c514088974c9706640854c47999b3728fbfcbda 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -135,7 +135,7 @@ if [ "$stage" == "build" ]; then # Clean up: clear old docker images echo -e "\n\n--------------\n Clean up docker images\n--------------\n" - $CI_PROJECT_DIR/test/clear_images.sh + $CI_PROJECT_DIR/scripts/clear_images.sh # Exit the CI with the return code of docker run echo -e "\n\nTerminating with exit status $status" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f3da018bd6b4bb724f0967fd35129034b01eba9..770991c8fb372ca98a548f70e692a3730c28baf7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,11 +46,26 @@ job_build-hepwlbuilder: when: always ### -### When a new build-image is created run the following test to verify that hep-workload images can be built +### When a new build-image is created run the following tests to verify that hep-workload images can be built +### Run the build of a dummy CI test image ### Run the build of a kv standalone image ### -job_test-hepwlbuilder: +job_test-hepwlbuilder-ci: + variables: + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/test/ci/test-ci.spec + only: + refs: + - qa + changes: + - build-executor/* + - .gitlab-ci.* + - test/ci/* + - test/ci/*/* + <<: *build-hepwl-anchor + stage: test-hepwlbuilder + +job_test-hepwlbuilder-kv: variables: CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv/atlas-kv.spec only: @@ -58,7 +73,6 @@ job_test-hepwlbuilder: - qa changes: - build-executor/* - - build-executor/singularity/* - .gitlab-ci.* <<: *build-hepwl-anchor stage: test-hepwlbuilder diff --git a/lhcb/gen-sim/lhcb-gen-sim.spec b/lhcb/gen-sim/lhcb-gen-sim.spec index 0491fe7ec89266fe0a07ae6ebe66e77f10d2bca0..799152f1ef1c57fe675a4b3df461eddc92574836 100644 --- a/lhcb/gen-sim/lhcb-gen-sim.spec +++ b/lhcb/gen-sim/lhcb-gen-sim.spec @@ -1,12 +1,7 @@ -# These variables are needed in Dockerfile.template HEPWL_BMKEXE=lhcb-gen-sim-bmk.sh HEPWL_BMKDIR=lhcb-gen-sim HEPWL_BMKDESCRIPTION="LHCb GEN-SIM benchmark" - -# These variables are needed in main.sh HEPWL_BMKOPTS="-n 1 -e 2" -###HEPWL_BMKOPTS="-n 1 -e DUMMY" # DUMMY TEST FOR GITLAB CI DEVELOPMENTS HEPWL_DOCKERIMAGENAME=lhcb-gen-sim-bmk HEPWL_DOCKERIMAGETAG=testv0.6 -###HEPWL_DOCKERIMAGETAG=DUMMY # DUMMY TEST FOR GITLAB CI DEVELOPMENTS HEPWL_CVMFSREPOS=lhcb.cern.ch diff --git a/lhcb/gen-sim/lhcb-gen-sim/lhcb-gen-sim-bmk.sh b/lhcb/gen-sim/lhcb-gen-sim/lhcb-gen-sim-bmk.sh index 66b26e4a3c771b614c60dfbdfedd8609503a3e8d..a4e3dafa01bb0246327058af5c58c605e0068146 100755 --- a/lhcb/gen-sim/lhcb-gen-sim/lhcb-gen-sim-bmk.sh +++ b/lhcb/gen-sim/lhcb-gen-sim/lhcb-gen-sim-bmk.sh @@ -17,12 +17,7 @@ function doOne(){ date strace="" ###strace="strace -tt -f -o out_$1_strace0.txt" # optionally produce a strace output - if [ "$nevents" == "DUMMY" ]; then - ###${strace} lb-run --use-sp -c x86_64-slc6-gcc48-opt --use="AppConfig v3r335" --use="DecFiles v30r11" --use="ProdConf" Gauss/v49r9 echo "DUMMY TEST" 2>&1 >out_$1.log # DUMMY TEST FOR GITLAB CI DEVELOPMENTS - echo "FASTER DUMMY TEST" 2>&1 >out_$1.log # FASTER DUMMY TEST FOR GITLAB CI DEVELOPMENTS - else - ${strace} lb-run --use-sp -c x86_64-slc6-gcc48-opt --use="AppConfig v3r335" --use="DecFiles v30r11" --use="ProdConf" Gauss/v49r9 gaudirun.py -T '$APPCONFIGOPTS/Gauss/Beam6500GeV-md100-2016-nu1.6.py' '$APPCONFIGOPTS/Gauss/EnableSpillover-25ns.py' '$APPCONFIGOPTS/Gauss/DataType-2016.py' '$APPCONFIGOPTS/Gauss/RICHRandomHits.py' '$DECFILESROOT/options/27163076.py' '$LBPYTHIA8ROOT/options/Pythia8.py' '$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py' '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py' "$WDIR/prodConf_Gauss_00071400_00000089_1.py" 2>&1 >out_$1.log - fi + ${strace} lb-run --use-sp -c x86_64-slc6-gcc48-opt --use="AppConfig v3r335" --use="DecFiles v30r11" --use="ProdConf" Gauss/v49r9 gaudirun.py -T '$APPCONFIGOPTS/Gauss/Beam6500GeV-md100-2016-nu1.6.py' '$APPCONFIGOPTS/Gauss/EnableSpillover-25ns.py' '$APPCONFIGOPTS/Gauss/DataType-2016.py' '$APPCONFIGOPTS/Gauss/RICHRandomHits.py' '$DECFILESROOT/options/27163076.py' '$LBPYTHIA8ROOT/options/Pythia8.py' '$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py' '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py' "$WDIR/prodConf_Gauss_00071400_00000089_1.py" 2>&1 >out_$1.log status=${?} ###if [ "$strace" != "" ]; then cat out_$1_strace0.txt | grep '"/cvmfs' | awk '{p=$0; while(1){ i1=index(p,"\"/cvmfs"); if (i1<=0) break; p=substr(p,i1+1); i2=index(p,"\""); if (substr(p,i2+1,3)!="...") {print substr(p,0,i2-1)}; p=substr(p,i2+1)} }' | sort -u > out_$1_strace1.txt; fi date @@ -66,9 +61,7 @@ while getopts "w:n:e:dh" o; do fi ;; e) - if [ "$OPTARG" == "DUMMY" ]; then - nevents=$OPTARG - elif [ $OPTARG -gt 0 ]; then + if [ $OPTARG -gt 0 ]; then nevents=$OPTARG else echo "ERROR! Invalid argument '-e $OPTARG'" @@ -79,7 +72,7 @@ while getopts "w:n:e:dh" o; do DEBUG=1 ;; h) - echo "Usage: $0 [-n <num_copies [$ncopies]>] [-e <num_events|DUMMY [$nevents]>] [-w <results_dir [$RESULTS_DIR]>] [-d (enable debug printouts)]" + echo "Usage: $0 [-n <num_copies [$ncopies]>] [-e <num_events [$nevents]>] [-w <results_dir [$RESULTS_DIR]>] [-d (enable debug printouts)]" exit 0 ;; esac @@ -104,12 +97,10 @@ done if [ $FAIL -gt 0 ]; then s_msg="[lhcb-bmk.sh] ERROR: $FAIL subprocess(es) failed"; echo $s_msg; generateSummary; exit $FAIL; fi -if [ "$nevents" != "DUMMY" ]; then - parseResults - status=$? - if [ "$status" != "0" ]; then s_msg="[lhcb-bmk.sh] ERROR: parseResults failed"; echo $s_msg; fi - generateSummary - exit $status -fi +parseResults +status=$? +if [ "$status" != "0" ]; then s_msg="[lhcb-bmk.sh] ERROR: parseResults failed"; echo $s_msg; fi +generateSummary +exit $status echo "[lhcb-bmk.sh] finished at $(date)" diff --git a/test/ci/Dockerfile.append b/test/ci/Dockerfile.append new file mode 100644 index 0000000000000000000000000000000000000000..26b0f71e683f735459bfa69dc674cf660e71fbea --- /dev/null +++ b/test/ci/Dockerfile.append @@ -0,0 +1,3 @@ +# Add here any workload-specific Dockerfile instructions. +# They will be appended to the Dockerfile generated from a common template. + diff --git a/test/ci/test-ci.spec b/test/ci/test-ci.spec new file mode 100644 index 0000000000000000000000000000000000000000..bb94cdae1aecb251e70c07854649a708009d553e --- /dev/null +++ b/test/ci/test-ci.spec @@ -0,0 +1,12 @@ +# These variables are needed in Dockerfile.template +HEPWL_BMKEXE=test-ci-bmk.sh +HEPWL_BMKDIR=test-ci +HEPWL_BMKDESCRIPTION="DUMMY benchmark for CI tests (based on LHCb setup)" + +# These variables are needed in main.sh +###HEPWL_BMKOPTS="-n 1 -e 1" # DUMMY HalloWorld TEST (FASTER) +HEPWL_BMKOPTS="-n 1 -e 2" # DUMMY LHCb setup TEST (DEFAULT) +###HEPWL_BMKOPTS="-n 1 -e 3" # DUMMY LHCb Gauss setup TEST (SLOWER) +HEPWL_DOCKERIMAGENAME=test-ci-bmk +HEPWL_DOCKERIMAGETAG=ci0.0 +HEPWL_CVMFSREPOS=lhcb.cern.ch diff --git a/test/ci/test-ci/parseResults.sh b/test/ci/test-ci/parseResults.sh new file mode 100644 index 0000000000000000000000000000000000000000..ca4c067f0f0385b3049199ae4ad96f86ef70d78c --- /dev/null +++ b/test/ci/test-ci/parseResults.sh @@ -0,0 +1,11 @@ +function generateSummary(){ + echo "[${FUNCNAME[0]}] Generate json summary" + echo -e "{ \"copies\": $ncopies , \"threads_x_copy\": 1 , \"events_x_thread\" : $nevents , \"througput_score\" : 1 , \"log\": \"OK\", \"app\":`cat $BMKDIR/version.json` }" > $BASE_WDIR/test-ci_summary.json + cat $BASE_WDIR/test-ci_summary.json +} + +function parseResults(){ + echo "[${FUNCNAME[0]}] BASE_WDIR=$BASE_WDIR" + echo "[${FUNCNAME[0]}] Available log files:" $BASE_WDIR/proc_*/out_*.log + for f in $BASE_WDIR/proc_*/out_*.log; do echo "[${FUNCNAME[0]}] --> Parsing results from $f"; cat $f; done +} diff --git a/test/ci/test-ci/test-ci-bmk.sh b/test/ci/test-ci/test-ci-bmk.sh new file mode 100755 index 0000000000000000000000000000000000000000..39c6f5a18831812dbe17b552dabec1eb18b5e6ed --- /dev/null +++ b/test/ci/test-ci/test-ci-bmk.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +#set -x # enable debug printouts + +#set -e # immediate exit on error + +function doOne(){ + [ $DEBUG -gt 0 ] && echo "`date` : start process $i"; + sleep 5 + WDIR=$BASE_WDIR/proc_$1 + [ $DEBUG -gt 0 ] && echo $WDIR + [ -e $WDIR ] && rm -f $WDIR + mkdir -p $WDIR + cd $WDIR + date + if [ "$nevents" == "1" ]; then + echo "Hallo World! (FASTER DUMMY TEST)" 2>&1 >out_$1.log + elif [ "$nevents" == "2" ]; then + source /cvmfs/lhcb.cern.ch/lib/LbLogin.sh 2>&1 >out_$1.log + echo "LHCb setup (DEFAULT DUMMY TEST)" 2>&1 >>out_$1.log + else + source /cvmfs/lhcb.cern.ch/lib/LbLogin.sh 2>&1 >out_$1.log + lb-run --use-sp -c x86_64-slc6-gcc48-opt --use="AppConfig v3r335" --use="DecFiles v30r11" --use="ProdConf" Gauss/v49r9 echo "LHCb Gauss setup (SLOWER DUMMY TEST)" 2>&1 >>out_$1.log + fi + status=${?} + date + if [ "$status" != "0" ]; then + echo "Process $i failed" + return $status + else + [ $DEBUG -gt 0 ] && echo "Process $i succesfully completed" + return 0 + fi +} + +echo "[test-ci-bmk.sh] starting at $(date)" + +cd `dirname $0` +export BMKDIR=`pwd` +. $BMKDIR/parseResults.sh + +export APP=`basename $0 -bmk.sh` +export RESULTS_DIR=/results + +export DATETAG=`date +%s` +export NTHREADS=1 #Num Threads per copy + +export DEBUG=0 +export ncopies=`nproc` +export nevents=2 # Num Events per thread + +function usage(){ + echo "Usage: $0 [-n <num_copies [1]>] [-e <num_events [5]>] [-w <results_dir [$RESULTS_DIR]>] [-d (enable debug printouts)]" + echo " num_events=1: dummy HalloWorld test (faster)" + echo " num_events=2: dummy LHCb setup test (default)" + echo " num_events>2: dummy LHCb Gauss setup test (slower)" + exit 1 +} + +while getopts "w:n:e:d" o; do + case ${o} in + w) + RESULTS_DIR=$OPTARG + ;; + n) + if [ $OPTARG -gt 0 ]; then + ncopies=$OPTARG + else + echo "ERROR! Invalid argument '-n $OPTARG'" + exit 1 + fi + ;; + e) + if [ $OPTARG -gt 0 ]; then + nevents=$OPTARG + else + echo "ERROR! Invalid argument '-e $OPTARG'" + exit 1 + fi + ;; + d) + DEBUG=1 + ;; + *) + usage + ;; + esac +done + +export BASE_WDIR=${RESULTS_DIR}/${APP}-n${ncopies}-e${nevents}-${DATETAG}_$(((RANDOM%9000)+1000)) +echo "BASE WORKING DIR=$BASE_WDIR" + +for i in `seq 1 $ncopies`; +do + ( doOne $i; )& +done + +status=0 +FAIL=0 +for job in `jobs -p` +do + [ $DEBUG -gt 0 ] && echo "...waiting forked process " $job + wait $job || let "FAIL+=1" +done +[ $DEBUG -gt 0 ] && echo "FAILS " $FAIL + +if [ $FAIL -gt 0 ]; then s_msg="[test-ci-bmk.sh] ERROR: $FAIL subprocess(es) failed"; echo $s_msg; generateSummary; exit $FAIL; fi + +if [ "$nevents" != "DUMMY" ]; then + parseResults + status=$? + if [ "$status" != "0" ]; then s_msg="[test-ci-bmk.sh] ERROR: parseResults failed"; echo $s_msg; fi + generateSummary + exit $status +fi + +echo "[test-ci-bmk.sh] finished at $(date)"