Merge branch 'BMK-192' into 'qa'

including parsing tests, changing json format

See merge request !243

.gitlab-ci.yml

 stages:
 - build-hepwlbuilder
 - test-hepwlbuilder
+- test-parsers
 - build-hepwl
       
 # Build the hep-workload-builder image (which is used to build and test WLs using docker)
@@ -220,3 +221,20 @@ job_build-lhcb-gen-sim:
     #- common/bmk-driver.sh
     #- common/Dockerfile.template
   <<: *build-hepwl-anchor
+
+###
+### Template for all the build jobs of hep-workloads containers
+###
+job_test-parsers:
+  image: $CI_REGISTRY_IMAGE/hep-workload-builder:latest
+  stage: test-parsers
+  script:
+   - aline=`printf '=%.0s' {1..100}`; 
+   - for aparser in `find . -name "test_parser.sh"`; do echo -e "$aline\nRunning $aparser\n$aline"; jobdir=$(dirname $(readlink -f $aparser))/jobs; $aparser || (tar -cvzf archive.tgz $jobdir; exit 1); done
+#  after_script:
+#   - tar -cvzf archive.tgz .
+  artifacts:
+    paths:
+      - $CI_PROJECT_DIR/archive.tgz
+    expire_in: 1 week
+    when: on_failure

alice/gen-sim/alice-gen-sim/jobs/bad_1/alice-gen-sim_summary.json

-{"copies":2 , "threads_per_copy":1 , "events_per_thread" : 1 , "CPU_score": "" , "log": "[ERROR] There is at least one job not finished properly", "app":  }
+{"copies":2 , "threads_per_copy":1 , "events_per_thread" : 1 , "CPU_score": "" , "log": "[ERROR] There is at least one job not finished properly", "app": {"version":"v0.15","description":"ALICE pp GEN-SIM","cvmfs_checksum":"9986a9a83745b416a510788e1f35d807","bmkdata_checksum":"b94489e16c0a77f2a0a9aaf333260a96","bmk_checksum":"6c5c2fd1379388c9b1cbd3888e5ecbd3"} }

alice/gen-sim/alice-gen-sim/jobs/bad_1/version.json

+{"version":"v0.15","description":"ALICE pp GEN-SIM","cvmfs_checksum":"9986a9a83745b416a510788e1f35d807","bmkdata_checksum":"b94489e16c0a77f2a0a9aaf333260a96","bmk_checksum":"6c5c2fd1379388c9b1cbd3888e5ecbd3"}

atlas/gen/atlas-gen/jobs/good_1/atlas-gen_summary.json

-{"copies":4 , "threads_per_copy": 1 , "events_per_thread" : 5 , "CPU_score": {"score": 89.2575, "avg": 22.3144, "median": 22.3242, "min": 21.8818, "max": 22.7273} , "log": "ok", "app":{"version":"v0.15","description":"ATLAS Event Generation based on athena version 19.2.5.5","cvmfs_checksum":"512aae44a881c0d4f9bc690488653241","bmkdata_checksum":"9a62d4782b086a9c74ab6edd38dc0ee3","bmk_checksum":"e93678ff532a3233f94f3e3890a8f39c"} }
+{"copies":4 , "threads_per_copy": 1 , "events_per_thread" : 5 , "wl-scores": {"sim": 89.2575} , "wl-stats": {"score": 89.2575, "avg": 22.3144, "median": 22.3242, "min": 21.8818, "max": 22.7273} , "log": "ok", "app":{"version":"v0.15","description":"ATLAS Event Generation based on athena version 19.2.5.5","cvmfs_checksum":"512aae44a881c0d4f9bc690488653241","bmkdata_checksum":"9a62d4782b086a9c74ab6edd38dc0ee3","bmk_checksum":"e93678ff532a3233f94f3e3890a8f39c"} }

atlas/gen/atlas-gen/parseResults.sh

 parseResultsDir=$(cd $(dirname ${BASH_SOURCE}); pwd) # needed to locate parseResults.py
 
 function generateSummary(){
-  echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\": $NTHREADS , \"events_per_thread\" : $NEVENTS_THREAD , \"CPU_score\": $resJSON , \"log\": \"${s_msg}\", \"app\":`cat $BMKDIR/version.json` }" > ${APP}_summary.json
+  echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\": $NTHREADS , \"events_per_thread\" : $NEVENTS_THREAD , $resJSON , \"log\": \"${s_msg}\", \"app\":`cat $BMKDIR/version.json` }" > ${APP}_summary.json
   cat ${APP}_summary.json
 }
 
@@ -26,11 +26,11 @@ function parseResults(){
   #-----------------------
   # Parse results (python)
   #-----------------------
-  echo "[parseResults] python parser starting"
+  #echo "[parseResults] python parser starting"
   export BASE_WDIR=`pwd`
-  python ${parseResultsDir}/parseResults.py # same directory as parseResults.sh
+  #python ${parseResultsDir}/parseResults.py # same directory as parseResults.sh
   pystatus=$?
-  echo "[parseResults] python parser completed (status=$pystatus)"
+  #echo "[parseResults] python parser completed (status=$pystatus)"
   #-----------------------
   # Parse results (bash)
   #-----------------------
@@ -45,8 +45,9 @@ function parseResults(){
         median=a[(n + 1) / 2];
       } else {
         median=(a[(n / 2)] + a[(n / 2) + 1]) / 2.0;
-      }; printf "{\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum/count, median, amin, amax
-    }' || (echo "\"[ERROR] Something went wrong in parsing the CPU score\""; exit 1)`
+      }; 
+printf "\"wl-scores\": {\"sim\": %.4f} , \"wl-stats\": {\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum, sum/count, median, amin, amax
+    }' || (echo "\"wl-scores\":{}"; exit 1)`
   shstatus=$?
   [ "$shstatus" != "0" ] && s_msg="ERROR"
   #-----------------------

atlas/kv/atlas-kv/jobs/good_1/atlas-kv_summary.json

-{"copies":1 , "threads_per_copy":1 , "events_per_thread" : 100 , "wl-scores": {"sim": 1.3514} , "CPU_score": {"score": 1.3514, "avg": 1.3514, "median": 1.3514, "min": 1.3514, "max": 1.3514} , "app": "KV_17.8.0.9_SingleMuon"}
+{"copies":1 , "threads_per_copy":1 , "events_per_thread" : 100 , "wl-scores": {"sim": 1.2723} , "wl-stats": {"score": 1.2723, "avg": 1.2723, "median": 1.2723, "min": 1.2723, "max": 1.2723} , "app":{"version":"ci1.1","description":"ATLAS KV: GEANT4 simulation of 100 single muon events, based on Athena version v17.8.0.9","cvmfs_checksum":"512aae44a881c0d4f9bc690488653241","bmkdata_checksum":"9a62d4782b086a9c74ab6edd38dc0ee3","bmk_checksum":"d441df8a450c4adeda688924bcd64e17"}}

atlas/kv/atlas-kv/parseResults.sh

@@ -21,15 +21,15 @@ function parseResults(){
   echo "[parseResults] parsing results from" proc_*/out_*.log
   local STATUS=0
   # Parse 'Event Throughput: xxxx ev/s'
-  res=`grep -A1 "INFO Statistics for 'evt'" proc_*/AtlasG4_trf.log | grep "<cpu>" | sed -e "s@[^(]*([[:blank:]]*\([ 0-9\.]*\) +/-.*@\1@" | awk 'BEGIN{amin=1000000;amax=0;count=0;}
+  res=`grep -A2 "INFO Statistics for 'evt'" proc_*/AtlasG4_trf.log | grep "<real>" | sed -e "s@[^(]*([[:blank:]]*\([ 0-9\.]*\) +/-.*@\1@" | awk 'BEGIN{amin=1000000;amax=0;count=0;}
 { val=1./(int($1)/1000.); a[count]=val; count+=1; sum+=val; if(amax<val) amax=val; if(amin>val) amin=val}
 END{sep=sprintf("%*s", 120, "");gsub(/ /, "*", sep);
 n = asort(a); if (n % 2) {
        median=a[(n + 1) / 2];
     } else {
         median=(a[(n / 2)] + a[(n / 2) + 1]) / 2.0;};
-printf "\"wl-scores\": {\"sim\": %.4f} , \"CPU_score\": {\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum, sum/count, median, amin, amax
-}' || (STATUS=1; echo "\"[ERROR] Something went wrong in parsing the CPU score\"")`
+printf "\"wl-scores\": {\"sim\": %.4f} , \"wl-stats\": {\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum, sum/count, median, amin, amax
+}' || (STATUS=1; echo "\"wl-scores\": {}")`
   #-----------------------
   # Generate json summary
   #-----------------------
@@ -38,7 +38,6 @@ printf "\"wl-scores\": {\"sim\": %.4f} , \"CPU_score\": {\"score\": %.4f, \"avg\
   local OUTPUT=${APP}_summary.json
   echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\":1 , \"events_per_thread\" : $NEVENTS_THREAD , $res , \"app\":`cat $BMKDIR/version.json`}" > $OUTPUT
   cat $OUTPUT
-  ln -s $OUTPUT report.json
   #-----------------------
   # Return status
   #-----------------------

atlas/sim/atlas-sim/jobs/bad_1/atlas-sim_summary.json

-{"copies":2 , "threads_per_copy":2 , "events_per_thread" : 3 , "CPU_score": "[ERROR] Something went wrong in parsing the CPU score" , "log": "ok", "app":{"version":"v0.11","description":"ATLAS Sim (GEANT4) based on athena version 21.0.15","checksum":"7d9492ceadf490c1933f491ba610b610"} }
+{"copies":2 , "threads_per_copy":2 , "events_per_thread" : 3 , "wl-scores":{} , "log": "ERROR in parsing", "app":{"version":"v0.11","description":"ATLAS Sim (GEANT4) based on athena version 21.0.15","checksum":"7d9492ceadf490c1933f491ba610b610"} }

atlas/sim/atlas-sim/jobs/good_1/atlas-sim_summary.json

-{"copies":2 , "threads_per_copy":2 , "events_per_thread" : 3 , "CPU_score": {"score": 0.0114, "avg": 0.0029, "median": 0.0029, "min": 0.0025, "max": 0.0032, "count": 4} , "log": "ok", "app":{"version":"v0.18","description":"ATLAS Sim (GEANT4) based on athena version 21.0.15","cvmfs_checksum":"5a371565e9f5abf81686a551c91703ba","bmkdata_checksum":"b94489e16c0a77f2a0a9aaf333260a96","bmk_checksum":"97142ce8dcd5ebf2dfe5f1d578c4decb"} }
+{"copies":2 , "threads_per_copy":2 , "events_per_thread" : 3 , "wl-scores": {"sim": 0.0114} , "wl-stats": {"score": 0.0114, "avg": 0.0029, "median": 0.0029, "min": 0.0025, "max": 0.0032} , "log": "ok", "app":{"version":"v0.18","description":"ATLAS Sim (GEANT4) based on athena version 21.0.15","cvmfs_checksum":"5a371565e9f5abf81686a551c91703ba","bmkdata_checksum":"b94489e16c0a77f2a0a9aaf333260a96","bmk_checksum":"97142ce8dcd5ebf2dfe5f1d578c4decb"} }

atlas/sim/atlas-sim/parseResults.sh

 parseResultsDir=$(cd $(dirname ${BASH_SOURCE}); pwd) # needed to locate parseResults.py
 
 function generateSummary(){
-  echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\":$NTHREADS , \"events_per_thread\" : $NEVENTS_THREAD , \"CPU_score\": $resJSON , \"log\": \"${s_msg}\", \"app\":`cat $BMKDIR/version.json` }" > ${APP}_summary.json
+  echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\":$NTHREADS , \"events_per_thread\" : $NEVENTS_THREAD , $resJSON , \"log\": \"${s_msg}\", \"app\":`cat $BMKDIR/version.json` }" > ${APP}_summary.json
   cat ${APP}_summary.json
 }
 
@@ -32,11 +32,11 @@ function parseResults(){
     #-----------------------
     # Parse results (python)
     #-----------------------
-    echo "[parseResults] python parser starting"
+    #echo "[parseResults] python parser starting"
     export BASE_WDIR=`pwd`
-    python ${parseResultsDir}/parseResults.py # same directory as parseResults.sh
+    #python ${parseResultsDir}/parseResults.py # same directory as parseResults.sh
     pystatus=$?
-    echo "[parseResults] python parser completed (status=$pystatus)"
+    #echo "[parseResults] python parser completed (status=$pystatus)"
     #-----------------------
     # Parse results (bash)
     #-----------------------
@@ -51,8 +51,9 @@ function parseResults(){
           median=a[(n + 1) / 2];
         } else {
           median=(a[(n / 2)] + a[(n / 2) + 1]) / 2.0;
-        }; printf "{\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f, \"count\": %d}", sum, sum/count, median, amin, amax , count
-      }' || (echo "\"[ERROR] Something went wrong in parsing the CPU score\""; exit 1)`
+        }; 
+printf "\"wl-scores\": {\"sim\": %.4f} , \"wl-stats\": {\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum, sum/count, median, amin, amax
+       }' || (echo "\"wl-scores\":{}"; exit 1)`
     shstatus=$?
     [ "$shstatus" != "0" ] && s_msg="ERROR in parsing"
     echo "[parseResults] bash parser completed (status=$shstatus)"

build-executor/image/Dockerfile

@@ -24,7 +24,7 @@ RUN yum install -y docker-ce \
     ShellCheck \
     http://ecsft.cern.ch/dist/cvmfs/cvmfs-2.6.0/cvmfs-shrinkwrap-2.6.0-1.el7.x86_64.rpm \
     http://ecsft.cern.ch/dist/cvmfs/cvmfs-2.6.0/cvmfs-2.6.0-1.el7.x86_64.rpm \
-    python-pip \
+    python-pip numpy \
     && yum clean all
 
 RUN sed -e 's@inet_interfaces = localhost@#inet_interfaces = localhost@' -e 's@inet_protocols = all@inet_protocols = ipv4@' -i /etc/postfix/main.cf

cms/reco/cms-reco/jobs/bad_1/cms-reco_summary.json

-{"copies":1 , "threads_per_copy":4 , "events_per_thread" : 3 , "throughput_score": "[ERROR] Something went wrong in parsing the Event Throughput" , "CPU_score": "[ERROR] Something went wrong in parsing the CPU score" , "log": "ERROR", "app":  }
+{"copies":1 , "threads_per_copy":4 , "events_per_thread" : 3 , "throughput_score": "[ERROR] Something went wrong in parsing the Event Throughput" , "CPU_score": "[ERROR] Something went wrong in parsing the CPU score" , "log": "ERROR", "app": {"version":"v0.11","description":"CMS RECO of ttbar events, based on CMSSW_10_2_9","cvmfs_checksum":"7709cc1dd71cc16f1e82ed0bef0002a2","bmkdata_checksum":"85537134eef4c02127b3b638bd6f766d","bmk_checksum":"ff2071f17349425463b4e8e59e23f846"}  }

cms/reco/cms-reco/jobs/bad_1/version.json

+{"version":"v0.11","description":"CMS RECO of ttbar events, based on CMSSW_10_2_9","cvmfs_checksum":"7709cc1dd71cc16f1e82ed0bef0002a2","bmkdata_checksum":"85537134eef4c02127b3b638bd6f766d","bmk_checksum":"ff2071f17349425463b4e8e59e23f846"}

common/json-differ.py

@@ -4,10 +4,13 @@ import sys
 import json
 from dictdiffer import diff
 
-json_a = json.load(open(sys.argv[1]))
-json_b = json.load(open(sys.argv[2]))
+json_list = []
+for ajson in sys.argv[1:3]:
+    print("Reading file %s" % ajson)
+    json_list.append( json.load(open(ajson)) )
+        
 
-result = list(diff(json_a, json_b))
+result = list(diff(json_list[0], json_list[1]))
 
 for entry in result:
     if len(entry[2]) == 1:

common/parsertest.sh

@@ -105,7 +105,7 @@ for job in $jobs; do
   done
 
   # Clean up and dump test results
-  \rm -f *TEST*json
+  #\rm -f *TEST*json
   echo -e "\nParser test completed for '$job' (parse exit code=$status1; test exit code=$status2)"
 
 done

lhcb/gen-sim/lhcb-gen-sim/jobs/good_1/lhcb-gen-sim_summary_old.json

-{"copies":4 , "threads_per_copy":1 , "events_per_thread" : 5 , "throughput_score": {"score": 18.8494, "avg": 4.7124, "median": 4.7115, "min": 4.6994, "max": 4.7271} , "log": "ok", "app":{"version":"v0.12","description":"LHCb GEN-SIM benchmark","cvmfs_checksum":"dc9fea953411cf41c03f9086f355e5fc","bmkdata_checksum":"199a9d5cb218c303eafd74076bb7a3c0","bmk_checksum":"8b45d0b7de247fea287383312204f5d8"} }
+{"copies":4 , "threads_per_copy":1 , "events_per_thread" : 5 , "wl-scores": {"sim": 18.8494} , "wl-stats": {"score": 18.8494, "avg": 4.7124, "median": 4.7115, "min": 4.6994, "max": 4.7271} , "log": "ok", "app":{"version":"v0.12","description":"LHCb GEN-SIM benchmark","cvmfs_checksum":"dc9fea953411cf41c03f9086f355e5fc","bmkdata_checksum":"199a9d5cb218c303eafd74076bb7a3c0","bmk_checksum":"8b45d0b7de247fea287383312204f5d8"} }

lhcb/gen-sim/lhcb-gen-sim/parseResults.sh

@@ -26,8 +26,9 @@ function parseResults(){
   local resJSON=`grep "EVENT LOOP" proc_*/out_*.log  | \
                    awk -F"|" 'BEGIN{amin=1000000;amax=0;count=0;}  \
                               { val=1000.*$5/$6; a[count]=val; count+=1; sum+=val; if(amax<val) amax=val; if(amin>val) amin=val} \
-                             END{n = asort(a); if (n % 2) {   median=a[(n + 1) / 2]; } else {median=(a[(n / 2)] + a[(n / 2) + 1])/ 2.0;}; printf "{\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum/count, median, amin, amax
-}' || (echo "\"[ERROR] Something went wrong in parsing the Event Throughput score\""; exit 1)`
+                             END{n = asort(a); if (n % 2) {   median=a[(n + 1) / 2]; } else {median=(a[(n / 2)] + a[(n / 2) + 1])/ 2.0;}; 
+printf "\"wl-scores\": {\"sim\": %.4f} , \"wl-stats\": {\"score\": %.4f, \"avg\": %.4f, \"median\": %.4f, \"min\": %.4f, \"max\": %.4f}", sum, sum, sum/count, median, amin, amax
+}' || (echo "\"wl-scores\":{}"; exit 1)`
   shstatus=$?
   [ "$shstatus" != "0" ] && s_msg="ERROR in bash parsing"
   echo $resJSON
@@ -40,14 +41,14 @@ function parseResults(){
   local app="\"UNKNOWN\""
   if [ -f $BMKDIR/version.json ]; then app=$(cat $BMKDIR/version.json); fi
   local OUTPUT=${APP}_summary_old.json
-  echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\":1 , \"events_per_thread\" : $NEVENTS_THREAD , \"throughput_score\": $resJSON , \"log\": \"${s_msg}\", \"app\":${app} }" > $OUTPUT
+  echo -e "{\"copies\":$NCOPIES , \"threads_per_copy\":1 , \"events_per_thread\" : $NEVENTS_THREAD , $resJSON , \"log\": \"${s_msg}\", \"app\":${app} }" > $OUTPUT
   cat $OUTPUT
   #-----------------------
   # Parse results (python)
   #-----------------------
   echo -e "\n[parseResults] python parser starting using $(python3 -V &> /dev/stdout)"
   local resJSON2 # declare 'local' separately to avoid masking $? (https://stackoverflow.com/a/4421282)
-  resJSON2=$(PYTHONPATH=${parseResultsDir} python3 -c "from parseResults import *; parseBmkDir('.')") # same directory as parseResults.sh
+  resJSON2=$(PYTHONPATH=${parseResultsDir} python -c "from parseResults import *; parseBmkDir('.')") # same directory as parseResults.sh
   pystatus=$?
   [ "$pystatus" != "0" ] && s_msg="ERROR in python parsing"
   echo $resJSON2