diff --git a/alice/gen-sim/alice-dpgsim-bmk/alice-dpgsim-bmk.sh b/alice/gen-sim/alice-dpgsim-bmk/alice-dpgsim-bmk.sh index 900f7bdb235314d56d19ffee7048cf86de864e05..8720672c3dfae00a4036ddfb6c4daba4f312b946 100755 --- a/alice/gen-sim/alice-dpgsim-bmk/alice-dpgsim-bmk.sh +++ b/alice/gen-sim/alice-dpgsim-bmk/alice-dpgsim-bmk.sh @@ -12,36 +12,15 @@ function doOne(){ ln -s $BMKDIR/environment environment ln -s $BMKDIR/GeneratorCustom.C GeneratorCustom.C ln -s $BMKDIR/GenParamCustomSingle.C GenParamCustomSingle.C - ln -s $BMKDIR/OCDBrec.root OCDBrec.root - ln -s $BMKDIR/OCDBsim.root OCDBsim.root + ln -s $BMKDIR/data/OCDBrec.root OCDBrec.root + ln -s $BMKDIR/data/OCDBsim.root OCDBsim.root - ./dpgsim.sh --run 264035 --mode sim --uid 1 --nevents $nevents --generator Custom --system 2>&1 > out_$1.log + ./dpgsim.sh --run 292839 --mode sim --uid 1 --nevents $nevents --generator Custom --system 2>&1 > out_$1.log [ $DEBUG -gt 0 ] && echo -e "\n`date` : done process $i\n" } -function parseResults(){ - STATUS=0 - echo "Parsing results from " $BASE_WDIR/proc_*/time.useronly - - #Parsing Event Throughput: xxxx ev/s - res=` cat $BASE_WDIR/proc_*/time.useronly | awk -v nevents=$nevents ' - BEGIN{amin=1000000;amax=0;count=0;} - { if ($1>0) {val=( int(nevents*1.) / int($1*1.) ); a[count]=val; count+=1; sum+=val; if(amax<val) amax=val; if(amin>val) amin=val} } - END{sep=sprintf("%*s", 120, "");gsub(/ /, "*", sep);n=asort(a); - if (n % 2) { - median=a[(n + 1) / 2]; - } else { - median=(a[(n / 2)] + a[(n / 2) + 1]) / 2.0; - }; printf "{\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum/count, median, amin, amax - }' || (STATUS=1; echo "\"[ERROR] Something went wrong in parsing the CPU score\"")` - - echo -e "{\"copies\":$ncopies , \"threads_x_copy\":$NTHREADS , \"events_x_thread\" : $nevents , \"CPU_score\": $res , \"app\": \"ALICE_pp_dpgsim\"}" > $BASE_WDIR/${APP}_summary.json - cat $BASE_WDIR/${APP}_summary.json - - return $STATUS -} export APP=`basename $0 -bmk.sh` export RESULTS_DIR=/results @@ -69,7 +48,7 @@ while getopts "w:n:e:dh" o; do DEBUG=1 ;; h) - echo "usage: $0 [-e <num_events_x_thread [$nevents]>] -n <num_copies [$ncopies]> [-d (for debug verbosity)] -w" + echo "usage: $0 [-e <num_events_x_thread [$nevents]>] -n <num_copies [$ncopies]> [-d (for debug verbosity)] [-w result_dir]" exit 0 ;; esac @@ -89,11 +68,12 @@ echo "nevents x thread=$nevents" cd `dirname $0` export BMKDIR=`pwd` -export BASE_WDIR=${RESULTS_DIR}/test_${APP}-n${ncopies}-e${nevents}-${DATETAG}_$(((RANDOM%9000)+1000)) +export BASE_WDIR=${RESULTS_DIR}/${APP}-n${ncopies}-e${nevents}-${DATETAG}_$(((RANDOM%9000)+1000)) echo "BASE WORKING DIR=$BASE_WDIR" -/cvmfs/alice.cern.ch/bin/alienv printenv VO_ALICE@AliDPG::v5-09-XX-15,VO_ALICE@AliPhysics::v5-09-09-01-1 &>environment +/cvmfs/alice.cern.ch/bin/alienv printenv VO_ALICE@AliDPG::prod-201901-02-1,VO_ALICE@AliPhysics::v5-09-45a-01-1 &>environment + source environment @@ -113,5 +93,6 @@ do done [ $DEBUG -gt 0 ] && echo "FAILS " $FAIL +. ./parseResults.sh parseResults exit $? diff --git a/alice/gen-sim/alice-dpgsim-bmk/OCDBsim.root b/alice/gen-sim/alice-dpgsim-bmk/data/OCDBrec.root similarity index 68% rename from alice/gen-sim/alice-dpgsim-bmk/OCDBsim.root rename to alice/gen-sim/alice-dpgsim-bmk/data/OCDBrec.root index 4a4ef5dd41f4516e791c9c0a6a5c8d0d85eb1ebc..93d59be821f1728d03e20ed977d84fb6a037af79 100644 Binary files a/alice/gen-sim/alice-dpgsim-bmk/OCDBsim.root and b/alice/gen-sim/alice-dpgsim-bmk/data/OCDBrec.root differ diff --git a/alice/gen-sim/alice-dpgsim-bmk/OCDBrec.root b/alice/gen-sim/alice-dpgsim-bmk/data/OCDBsim.root similarity index 68% rename from alice/gen-sim/alice-dpgsim-bmk/OCDBrec.root rename to alice/gen-sim/alice-dpgsim-bmk/data/OCDBsim.root index cb7b16b7dc2bdd741a05200bd0f87e57072ea305..bafcb3dd6df2f51cb0c499099f946b1f280dfba3 100644 Binary files a/alice/gen-sim/alice-dpgsim-bmk/OCDBrec.root and b/alice/gen-sim/alice-dpgsim-bmk/data/OCDBsim.root differ diff --git a/alice/gen-sim/alice-dpgsim-bmk/parseResults.sh b/alice/gen-sim/alice-dpgsim-bmk/parseResults.sh new file mode 100644 index 0000000000000000000000000000000000000000..da7669296983333bc27c106259754be6f32e5cb5 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/parseResults.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +function parseResults(){ + STATUS=0 + + echo "[parseResults] Parsing results from $BASE_WDIR/proc_*/time.useronly" + + #Parsing Event Throughput: xxxx ev/s + res=` cat $BASE_WDIR/proc_*/time.useronly | awk -v nevents=$nevents ' + BEGIN{amin=1000000;amax=0;count=0;} + { if ($1>0) {val=( int(nevents*1.) / int($1*1.) ); a[count]=val; count+=1; sum+=val; if(amax<val) amax=val; if(amin>val) amin=val} } + END{sep=sprintf("%*s", 120, "");gsub(/ /, "*", sep);n=asort(a); + if (n % 2) { + median=a[(n + 1) / 2]; + } else { + median=(a[(n / 2)] + a[(n / 2) + 1]) / 2.0; + }; printf "{\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum/count, median, amin, amax + }' || (STATUS=1; echo "\"[ERROR] Something went wrong in parsing the CPU score\"")` + + echo "[parseResults] Checking that all job are completed correctly $BASE_WDIR/proc_*/out_*.log" + + ALL_JOBS_SUCCEED=`grep -c -h -m1 "finished with the expected exit code" $BASE_WDIR/proc_*/out_*.log | awk 'BEGIN{sum=0;count=0}{sum+=$1;count+=1}END{print sum==count}'` + if [ "$ALL_JOBS_SUCCEED" != 1 ]; + then + STATUS=1 + res="\"[ERROR] There is at least one job not finished properly\"" + grep -c "finished with the expected exit code" $BASE_WDIR/proc_*/out_*.log + fi + + echo -e "{\"copies\":$ncopies , \"threads_x_copy\":$NTHREADS , \"events_x_thread\" : $nevents , \"CPU_score\": $res , \"app\": \"ALICE_pp_dpgsim\"}" > $BASE_WDIR/${APP}_summary.json + cat $BASE_WDIR/${APP}_summary.json + + return $STATUS +} diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser.sh b/alice/gen-sim/alice-dpgsim-bmk/test_parser.sh new file mode 100755 index 0000000000000000000000000000000000000000..5521eddc1356c31761e97d896d6de45bfe5bbe50 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser.sh @@ -0,0 +1,13 @@ +. ./parseResults.sh + +export APP=alice-dpgsim +export ncopies=2 +export nevents=1 +export NTHREADS=1 + +export BASE_WDIR=`pwd`/test_parser_data/succeeded_jobs +parseResults + +echo " " +export BASE_WDIR=`pwd`/test_parser_data/failed_job +parseResults diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/alice-dpgsim_summary.json b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/alice-dpgsim_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..400ef428455c7a390fcb9975f660903b52bb82e7 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/alice-dpgsim_summary.json @@ -0,0 +1 @@ +{"copies":2 , "threads_x_copy":1 , "events_x_thread" : 1 , "CPU_score": "[ERROR] There is at least one job not finished properly" , "app": "ALICE_pp_dpgsim"} diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_1/out_1.log b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_1/out_1.log new file mode 100644 index 0000000000000000000000000000000000000000..00c862eedb1a6f3d1a7643f23569a8d24eb4c192 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_1/out_1.log @@ -0,0 +1,56 @@ +* Sourcing G4 environment from /cvmfs/alice.cern.ch/el6-x86_64/Packages/GEANT4/v10.4.2-1/bin/geant4.sh + +============================================ + DPGSIM +============================================ +Run.............. 292839 +Mode............. sim +QA train......... +AOD train........ +============================================ +Year............. 2018 +Period........... LHC18m +Beam type........ p-p +Energy........... 12998 +Detector mask.... 0x40277fbf +============================================ +Generator........ Custom +Process.......... +No. Events....... 1 +Unique-ID........ 1 +MC seed.......... 283900001 +============================================ +Background....... +No. Events....... +============================================ +Detector......... +GEANT4........... +Material Budget.. +Simulation....... +Reconstruction... +System........... p-p +Trigger.......... p-p +OCDB............. snapshot +HLT.............. +============================================ +b-min............ +b-max............ +y-min............ +y-max............ +============================================ +pT-hard bin...... +pT-hard min...... +pT-hard max...... +pT-trigger bin... +pT-trigger min... +pT-trigger max... +quenching........ +q-hat............ +============================================ + + + +* SIMULATION : /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C +* SIMULATION : output log in sim.log +SIMULATION TIME: 61.01 +* /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C finished with the expected exit code (0), moving on diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_1/time.useronly b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_1/time.useronly new file mode 100644 index 0000000000000000000000000000000000000000..a851f074db4d5fbf9b7e594b9c30f6eae85ae3e1 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_1/time.useronly @@ -0,0 +1 @@ +61.01 diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_2/out_2.log b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_2/out_2.log new file mode 100644 index 0000000000000000000000000000000000000000..138fd28ba9403c2fd5330ade3ae526825a566e6a --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_2/out_2.log @@ -0,0 +1,55 @@ +* Sourcing G4 environment from /cvmfs/alice.cern.ch/el6-x86_64/Packages/GEANT4/v10.4.2-1/bin/geant4.sh + +============================================ + DPGSIM +============================================ +Run.............. 292839 +Mode............. sim +QA train......... +AOD train........ +============================================ +Year............. 2018 +Period........... LHC18m +Beam type........ p-p +Energy........... 12998 +Detector mask.... 0x40277fbf +============================================ +Generator........ Custom +Process.......... +No. Events....... 1 +Unique-ID........ 1 +MC seed.......... 283900001 +============================================ +Background....... +No. Events....... +============================================ +Detector......... +GEANT4........... +Material Budget.. +Simulation....... +Reconstruction... +System........... p-p +Trigger.......... p-p +OCDB............. snapshot +HLT.............. +============================================ +b-min............ +b-max............ +y-min............ +y-max............ +============================================ +pT-hard bin...... +pT-hard min...... +pT-hard max...... +pT-trigger bin... +pT-trigger min... +pT-trigger max... +quenching........ +q-hat............ +============================================ + + + +* SIMULATION : /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C +* SIMULATION : output log in sim.log +SIMULATION TIME: 61.01 diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_2/time.useronly b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_2/time.useronly new file mode 100644 index 0000000000000000000000000000000000000000..93f73566ce3686164ae524269ef42525774974c0 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/failed_job/proc_2/time.useronly @@ -0,0 +1 @@ +60.5 diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/alice-dpgsim_summary.json b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/alice-dpgsim_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6635acf5253884eedc48644c6fc671b0c097eb99 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/alice-dpgsim_summary.json @@ -0,0 +1 @@ +{"copies":2 , "threads_x_copy":1 , "events_x_thread" : 1 , "CPU_score": {"score": 0.0331, "avg": 0.0165, "median": 0.0165, "min": 0.0164, "max": 0.0167} , "app": "ALICE_pp_dpgsim"} diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_1/out_1.log b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_1/out_1.log new file mode 100644 index 0000000000000000000000000000000000000000..00c862eedb1a6f3d1a7643f23569a8d24eb4c192 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_1/out_1.log @@ -0,0 +1,56 @@ +* Sourcing G4 environment from /cvmfs/alice.cern.ch/el6-x86_64/Packages/GEANT4/v10.4.2-1/bin/geant4.sh + +============================================ + DPGSIM +============================================ +Run.............. 292839 +Mode............. sim +QA train......... +AOD train........ +============================================ +Year............. 2018 +Period........... LHC18m +Beam type........ p-p +Energy........... 12998 +Detector mask.... 0x40277fbf +============================================ +Generator........ Custom +Process.......... +No. Events....... 1 +Unique-ID........ 1 +MC seed.......... 283900001 +============================================ +Background....... +No. Events....... +============================================ +Detector......... +GEANT4........... +Material Budget.. +Simulation....... +Reconstruction... +System........... p-p +Trigger.......... p-p +OCDB............. snapshot +HLT.............. +============================================ +b-min............ +b-max............ +y-min............ +y-max............ +============================================ +pT-hard bin...... +pT-hard min...... +pT-hard max...... +pT-trigger bin... +pT-trigger min... +pT-trigger max... +quenching........ +q-hat............ +============================================ + + + +* SIMULATION : /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C +* SIMULATION : output log in sim.log +SIMULATION TIME: 61.01 +* /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C finished with the expected exit code (0), moving on diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_1/time.useronly b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_1/time.useronly new file mode 100644 index 0000000000000000000000000000000000000000..a851f074db4d5fbf9b7e594b9c30f6eae85ae3e1 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_1/time.useronly @@ -0,0 +1 @@ +61.01 diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_2/out_2.log b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_2/out_2.log new file mode 100644 index 0000000000000000000000000000000000000000..00c862eedb1a6f3d1a7643f23569a8d24eb4c192 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_2/out_2.log @@ -0,0 +1,56 @@ +* Sourcing G4 environment from /cvmfs/alice.cern.ch/el6-x86_64/Packages/GEANT4/v10.4.2-1/bin/geant4.sh + +============================================ + DPGSIM +============================================ +Run.............. 292839 +Mode............. sim +QA train......... +AOD train........ +============================================ +Year............. 2018 +Period........... LHC18m +Beam type........ p-p +Energy........... 12998 +Detector mask.... 0x40277fbf +============================================ +Generator........ Custom +Process.......... +No. Events....... 1 +Unique-ID........ 1 +MC seed.......... 283900001 +============================================ +Background....... +No. Events....... +============================================ +Detector......... +GEANT4........... +Material Budget.. +Simulation....... +Reconstruction... +System........... p-p +Trigger.......... p-p +OCDB............. snapshot +HLT.............. +============================================ +b-min............ +b-max............ +y-min............ +y-max............ +============================================ +pT-hard bin...... +pT-hard min...... +pT-hard max...... +pT-trigger bin... +pT-trigger min... +pT-trigger max... +quenching........ +q-hat............ +============================================ + + + +* SIMULATION : /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C +* SIMULATION : output log in sim.log +SIMULATION TIME: 61.01 +* /cvmfs/alice.cern.ch/el6-x86_64/Packages/AliDPG/prod-201901-02-1/MC/sim.C finished with the expected exit code (0), moving on diff --git a/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_2/time.useronly b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_2/time.useronly new file mode 100644 index 0000000000000000000000000000000000000000..93f73566ce3686164ae524269ef42525774974c0 --- /dev/null +++ b/alice/gen-sim/alice-dpgsim-bmk/test_parser_data/succeeded_jobs/proc_2/time.useronly @@ -0,0 +1 @@ +60.5 diff --git a/build-executor/main.sh b/build-executor/main.sh index 0b932b0d011a24e10e461dcbff0f49e36f8b74ae..99f0088091d7b9d57c80571f7e2a872f0bea28f8 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -1,7 +1,5 @@ #!/bin/bash -set -x # enable debug printouts - #set -e # immediate exit on error @@ -327,7 +325,7 @@ export DOCKER_BASE_IMAGE='gitlab-registry.cern.ch/cloud-infrastructure/cloud-ben # 1: SPEC file path # 2: optional export DOCKER_REGISTRY gitlab-registry.cern.ch/giordano/hep-workloads -while getopts "hs:r:e:m" o; do +while getopts "hds:r:e:m" o; do case ${o} in s) [ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG" @@ -341,8 +339,11 @@ while getopts "hs:r:e:m" o; do m) MOUNT_CVMFS=1 ;; + d) + set -x # enable debug printouts + ;; h) - echo "usage: $0 -s <full path to HEP Workload Spec file> [-r <docker registry>] [-e <event from which to start (all|shrink|build|sleep|none): default is all>]" + echo "usage: $0 -s <full path to HEP Workload Spec file> [-r <docker registry>] [-e <event from which to start (all|shrink|build|sleep|none): default is all>] [-d (enable debug printouts)]" exit 1 ;; esac diff --git a/build-executor/run_build.sh b/build-executor/run_build.sh index a04d7747d375e959b97ecda04c898e7f4d06fd21..33d3a053d3c95e69db76bc9796b303f2e37be282 100755 --- a/build-executor/run_build.sh +++ b/build-executor/run_build.sh @@ -1,7 +1,6 @@ #!/bin/bash -set -x # enable debug printouts - +#set -x # enable debug printouts #set -e # immediate exit on error SCRIPT_DIR=`dirname $0` @@ -30,18 +29,41 @@ DOCK_CVMFS="/var/lib/cvmfs" WL_SPEC="${WL_DIR}/cms/gen-sim/cms-ttbar.spec" BUILD_EVENT="all" -[[ "$1" != "" ]] && WL_SPEC="$1" -[[ "$2" != "" ]] && BUILD_EVENT="$2" +while getopts "hs:e:d" o; do + case ${o} in + s) + [ "$OPTARG" != "" ] && export WL_SPEC="$OPTARG" + ;; + e) + [ "$OPTARG" != "" ] && export BUILD_EVENT="$OPTARG" + ;; + d) + set -x # enable debug printouts + ;; + h) + echo "usage: $0 -s <full path to HEP Workload Spec file> [-e <event from which to start (all|shrink|build|sleep|none): default is all>]" + exit 1 + ;; + esac +done + UUID=$(date +"%Y-%m-%d-%H-%M-%S")_$(((RANDOM%100)+1)) CVMFS_VOLUME=/scratch/cvmfs_hep/cvmfs_hep_$UUID -umount $CVMFS_VOLUME/* -rm -rf $CVMFS_VOLUME/* + +if [ -e "$CVMFS_VOLUME" ]; then + umount $CVMFS_VOLUME/* + rm -rf $CVMFS_VOLUME/* +fi JOB_DIR=/scratch/logs/hep_builder_$UUID mkdir -p $JOB_DIR + +echo -e "\n\n\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n" echo JOB_DIR is $JOB_DIR echo CVMFS_VOLUME is $CVMFS_VOLUME +echo -e "\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n\n\n" + echo "running privileged docker with fixed entrypoint" docker run --rm --privileged --entrypoint=/root/main.sh \