From 67542b81e8855051a066f12c46bf00eb3465e337 Mon Sep 17 00:00:00 2001
From: Domenico Giordano <domenico.giordano@cern.ch>
Date: Tue, 1 Sep 2020 15:08:21 +0000
Subject: [PATCH] how to promote images

---
 .gitlab-ci.yml                                | 62 ++++++++++++-
 ci_promote_image.sh                           | 22 +++++
 cms/cms-patatrack-ci.yml                      | 66 +++++++------
 ...ia.Dockerfile.0 => Dockerfile.nvidia.base} |  6 +-
 ...ia.Dockerfile.1 => Dockerfile.nvidia.data} |  2 +-
 ...ia.Dockerfile.2 => Dockerfile.nvidia.full} |  4 +-
 .../ci-scripts/{build_2.sh => build_final.sh} |  2 +-
 cms/patatrack/ci-scripts/snapshot_cvmfs.sh    | 24 +++--
 cms/patatrack/cms-patatrack/bmk-driver.sh     |  1 -
 .../cms-patatrack/cms-patatrack-bmk.sh        | 17 ++--
 cms/patatrack/cms-patatrack/cms-reco-bmk.sh   | 93 -------------------
 .../traces/cms.cern.ch_spec_custom.txt        |  6 +-
 lhc/simpletrack/.simpletrack-ci.yml           | 30 ++++--
 13 files changed, 173 insertions(+), 162 deletions(-)
 create mode 100755 ci_promote_image.sh
 rename cms/patatrack/ci-scripts/{nvidia.Dockerfile.0 => Dockerfile.nvidia.base} (59%)
 rename cms/patatrack/ci-scripts/{nvidia.Dockerfile.1 => Dockerfile.nvidia.data} (96%)
 rename cms/patatrack/ci-scripts/{nvidia.Dockerfile.2 => Dockerfile.nvidia.full} (69%)
 rename cms/patatrack/ci-scripts/{build_2.sh => build_final.sh} (90%)
 delete mode 100755 cms/patatrack/cms-patatrack/cms-reco-bmk.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f219562..21bfbf3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,41 @@
 stages:
 - triggers
 - test
+- promote-image
+- announce-promoted-image
+
+
+############################################################
+#####              PROMOTE PROD IMAGES                 #####
+############################################################
+
+.definition_promote_prod_image: &template_promote_prod_image
+  stage: promote-image
+  tags:
+    - hep-workload-gpu-docker-builder
+  image: 
+    name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
+    entrypoint: [""]
+  after_script:
+       - docker images
+       - docker ps -a
+  script: 
+    - ${CI_PROJECT_DIR}/ci_promote_image.sh
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^.*-v[0-9].*$/
+
+.defnition_announce_prod_image: &template_announce_prod_image
+  stage: announce-promoted-image
+  image: $CI_REGISTRY_IMAGE/announcement:latest
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^.*-v[0-9].*$/
+  script: 
+    - /announce.sh $IMAGE
+  before_script:
+    - export IMAGE=$CI_REGISTRY_IMAGE/something
+
 
 #####################################################
 ### ATLAS KV (a test of cvmfs functionality)
@@ -40,7 +75,7 @@ stages:
 ### CMS PATATRACK 
 #####################################################
 
-patatrack:
+build_patatrack:
   stage: triggers
   trigger:
     include: 
@@ -49,18 +84,26 @@ patatrack:
   only:
     variables:
       - $CI_COMMIT_BRANCH =~ /^qa.*$/
-      - $CI_COMMIT_TAG =~ /^v.*$/
     changes:
       - cms/patatrack/*
       - cms/patatrack/ci-scripts/*
       - cms/patatrack/cms-patatrack/*
       - cms/patatrack/cms-patatrack/utility_scripts/*
-      
+
+promote_patatrack:
+  <<: *template_promote_prod_image
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^cms-patatrack-v.*$/
+  before_script:
+    - export IMAGE_NAMES=$CI_REGISTRY_IMAGE/cms/cms-patatrack-nvidia-bmk
+    - export IMAGE_TAG:ci-qa-${CI_COMMIT_SHA:0:8}
+
 #####################################################
 ### LHC Simple Track
 #####################################################
 
-simpletrack:
+build_simpletrack:
   stage: triggers
   trigger:
     include: lhc/simpletrack/.simpletrack-ci.yml
@@ -68,8 +111,17 @@ simpletrack:
   only:
     variables:
     - $CI_COMMIT_BRANCH =~ /^qa.*$/
-    - $CI_COMMIT_TAG =~ /^v.*$/
     changes:
     - lhc/simpletrack/Dockerfile.*
     - lhc/simpletrack/lhc-simpletrack.*
     - lhc/simpletrack/.simpletrack-ci.yml
+
+
+promote_simpletrack:
+  <<: *template_promote_prod_image
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^simpletrack-v.*$/
+  before_script:
+    - export IMAGE_NAMES="$CI_REGISTRY_IMAGE/simpletrack/intel|$CI_REGISTRY_IMAGE/simpletrack/nvidia|$CI_REGISTRY_IMAGE/simpletrack/pocl|$CI_REGISTRY_IMAGE/simpletrack/rocm"
+    - export IMAGE_TAG=ci-qa-latest
diff --git a/ci_promote_image.sh b/ci_promote_image.sh
new file mode 100755
index 0000000..45c2dd8
--- /dev/null
+++ b/ci_promote_image.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+ echo "current CI_COMMIT_BRANCH is ${CI_COMMIT_BRANCH}"
+ echo "current CI_COMMIT_REF_NAME is $CI_COMMIT_REF_NAME"
+ echo "current CI_COMMIT_SHA is ${CI_COMMIT_SHA:0:8}"
+ echo "current CI_COMMIT_TAG is ${CI_COMMIT_TAG}"
+ if [[ -z $IMAGE_NAMES ]]; then echo "ERROR variable IMAGE_NAMES is not defined "; exit 1; fi
+ if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi
+ if [[ -z $CI_COMMIT_TAG ]]; then echo "ERROR variable CI_COMMIT_TAG is not defined "; exit 1; fi 
+ NEW_IMAGE_TAG=`echo ${CI_COMMIT_TAG} | sed -e 's@.*-\(v[0-9\.]*\)@\1@'`
+ echo "new IMAGE TAG is ${NEW_IMAGE_TAG}"
+ docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN gitlab-registry.cern.ch
+ for IMAGE_NAME in `echo $IMAGE_NAMES | tr '|' '\n'`;
+ do
+    echo "IMAGE to tag is ${IMAGE_NAME}:${IMAGE_TAG}"
+    docker pull ${IMAGE_NAME}:${IMAGE_TAG}
+    docker tag ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${NEW_IMAGE_TAG}
+    docker tag ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:latest
+    docker push ${IMAGE_NAME}:${NEW_IMAGE_TAG}
+    docker push ${IMAGE_NAME}:latest
+    docker rmi ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${NEW_IMAGE_TAG} ${IMAGE_NAME}:latest
+done
\ No newline at end of file
diff --git a/cms/cms-patatrack-ci.yml b/cms/cms-patatrack-ci.yml
index ab394ab..fa62a2d 100644
--- a/cms/cms-patatrack-ci.yml
+++ b/cms/cms-patatrack-ci.yml
@@ -1,10 +1,10 @@
 ---
 stages:
-    - build_0
-    - build_1
-    - build_2
-    - snapshot
-    - build_standalone
+    - build_base_image
+    - build_data_image
+    - build_interim_image
+    - snapshot_cvmfs
+    - build_standalone_image
     - test
 #- publish
 #- announce
@@ -61,48 +61,53 @@ stages:
 # docker in docker image: to trigger other docker runs
 ###########################################################
 
-job_build_image_step0: 
-    stage: build_0
+job_build_base_image: 
+    stage: build_base_image
     before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.base
         - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
-        - export IMAGE_NAME=cms/cms-patatrack-nvidia-0
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-base
         - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
     <<: *template_build_image
     only:
         changes:
-          - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
-
+          - cms/patatrack/ci-scripts/Dockerfile.nvidia.base
+        variables:
+          - $CI_COMMIT_BRANCH =~ /^qa.*$/
 
-job_build_image_step1: 
-    stage: build_1
+job_build_data_image: 
+    stage: build_data_image
     before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.data
         - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
-        - export IMAGE_NAME=cms/cms-patatrack-nvidia-1
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-data
         - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
     <<: *template_build_image
     only:
         changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.base
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.data
+        variables:
+            - $CI_COMMIT_BRANCH =~ /^qa.*$/
 
-job_build_image_step2: 
-    stage: build_2
+job_build_interim_image: 
+    stage: build_interim_image
     before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.full
         - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
-        - export IMAGE_NAME=cms/cms-patatrack-nvidia-2
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-interim
         - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
     <<: *template_build_image
     only:
         changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.*
             - cms/patatrack/cms-patatrack/*
             - cms/patatrack/cms-patatrack/utility_scripts/*
+        variables:
+            - $CI_COMMIT_BRANCH =~ /^qa.*$/
 
 job_snapshot_cvmfs: 
-    stage: snapshot
+    stage: snapshot_cvmfs
     tags:
         - hep-workload-gpu-docker-builder
     image:
@@ -119,35 +124,36 @@ job_snapshot_cvmfs:
     only:
         variables:
             - $CI_COMMIT_BRANCH =~ /^qa.*$/
-            - $CI_COMMIT_TAG =~ /^v.*$/
         changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.*
             - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
             - cms/patatrack/cms-patatrack/*
             - cms/patatrack/cms-patatrack/utility_scripts/*
     artifacts:
         paths:
             - ${CI_PROJECT_DIR}/traces
-            - ${CI_PROJECT_DIR}/cvmfs_export_dir_content
+            - ${CI_PROJECT_DIR}/cvmfs_export_dir_content.txt
             - ${CI_PROJECT_DIR}/cvmfs_export_py2-scipy_content
             - ${CI_PROJECT_DIR}/cms/patatrack/cvmfs
         expire_in: 1 week
         when: always
 
 job_build_standalone_image: 
-    stage: build_standalone
+    stage: build_standalone_image
     before_script:
-        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/Dockerfile.nvidia.full
         - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
         - export IMAGE_NAME=cms/cms-patatrack-nvidia-bmk
         - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
     <<: *template_build_image
     only:
         changes:
-            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/Dockerfile.nvidia.*
             - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
             - cms/patatrack/cms-patatrack/*
             - cms/patatrack/cms-patatrack/utility_scripts/*
+        variables:
+            - $CI_COMMIT_BRANCH =~ /^qa.*$/
 
 job_test_standalone_image: 
     stage: test
@@ -170,6 +176,6 @@ job_test_standalone_image:
             - cms/patatrack/cms-patatrack/utility_scripts/*
     artifacts:
         paths:
-            - ${CI_PROJECT_DIR}/${RESULTS_DIR}
+            - ${CI_PROJECT_DIR}/CI-JOB-${CI_JOB_ID}
         expire_in: 1 week
         when: always
\ No newline at end of file
diff --git a/cms/patatrack/ci-scripts/nvidia.Dockerfile.0 b/cms/patatrack/ci-scripts/Dockerfile.nvidia.base
similarity index 59%
rename from cms/patatrack/ci-scripts/nvidia.Dockerfile.0
rename to cms/patatrack/ci-scripts/Dockerfile.nvidia.base
index 6d66195..f3a7dec 100644
--- a/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+++ b/cms/patatrack/ci-scripts/Dockerfile.nvidia.base
@@ -1,5 +1,7 @@
-# FIXME: need to build in gitlab this base image. Was done by hand
-FROM nvidia/cuda:10.1-devel-centos7 
+# This dockerfile generates a base image for 
+# later importing bmk code and cvmfs.
+# This image is not frequently updated.
+FROM nvidia/cuda:10.1-base-centos7
 
 RUN yum install -y \
     which \
diff --git a/cms/patatrack/ci-scripts/nvidia.Dockerfile.1 b/cms/patatrack/ci-scripts/Dockerfile.nvidia.data
similarity index 96%
rename from cms/patatrack/ci-scripts/nvidia.Dockerfile.1
rename to cms/patatrack/ci-scripts/Dockerfile.nvidia.data
index b63ab1a..87691b1 100644
--- a/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+++ b/cms/patatrack/ci-scripts/Dockerfile.nvidia.data
@@ -1,4 +1,4 @@
-FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-0:qa
+FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-base:qa
 
 
 # Prepare a data directory for downloading large files that should normally be cacheable (BMK-159)
diff --git a/cms/patatrack/ci-scripts/nvidia.Dockerfile.2 b/cms/patatrack/ci-scripts/Dockerfile.nvidia.full
similarity index 69%
rename from cms/patatrack/ci-scripts/nvidia.Dockerfile.2
rename to cms/patatrack/ci-scripts/Dockerfile.nvidia.full
index ecc4be0..c422efc 100644
--- a/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+++ b/cms/patatrack/ci-scripts/Dockerfile.nvidia.full
@@ -1,7 +1,7 @@
-FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-1:qa
+FROM gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-data:qa
 
 COPY . /stage/
 RUN ls -la /stage/*
-RUN /stage/ci-scripts/build_2.sh
+RUN /stage/ci-scripts/build_final.sh
 
 ENTRYPOINT ["/bmk/./cms-patatrack/cms-patatrack-bmk.sh"]
diff --git a/cms/patatrack/ci-scripts/build_2.sh b/cms/patatrack/ci-scripts/build_final.sh
similarity index 90%
rename from cms/patatrack/ci-scripts/build_2.sh
rename to cms/patatrack/ci-scripts/build_final.sh
index 8a78e56..afef513 100755
--- a/cms/patatrack/ci-scripts/build_2.sh
+++ b/cms/patatrack/ci-scripts/build_final.sh
@@ -22,7 +22,7 @@ chmod -R 555 /cvmfs
 # This MUST happen before linking the data dir
 # otherwise will take a lot of time to tar
 date
-tar -cf /tmp/bmk_checksum.tar /bmk && md5sum /tmp/bmk_checksum.tar | cut -f1 -d" " >/tmp/bmk_checksum && rm /tmp/bmk_checksum.tar #FIXME
+tar -cf /tmp/bmk_checksum.tar /bmk --exclude='data' && md5sum /tmp/bmk_checksum.tar | cut -f1 -d" " >/tmp/bmk_checksum && rm /tmp/bmk_checksum.tar #FIXME
 
 # The data dir has already a checksum in /tmp/bmkdata_checksum
 # generated in nvidia.Dockerfile.1
diff --git a/cms/patatrack/ci-scripts/snapshot_cvmfs.sh b/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
index 4cf6533..b885ab1 100644
--- a/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+++ b/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # script used in gitlab CI
-# for job job_snapshot_cvmfs
+# for job snapshot_cvmfs
 # in file cms/cms-patatrack-ci.yml
 
 function _before_script() {
@@ -15,25 +15,35 @@ function _script() {
     # check cvmfs is running
     docker exec cvmfs_${CI_JOB_ID} cvmfs_config probe
     # Here comes the dry run of the CMS Patatrack container. Arguments are for the time being defaults/hardcoded FIXME
-    docker pull gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-2:qa
-    docker run --name patatrack_container --gpus '"device=0"' -v ${CIENV_CVMFSVOLUME}:/cvmfs gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-2:qa -e 100 -t 8 -c 1
+    docker pull ${INTERIM_IMAGE}
+    docker run --name patatrack_container --gpus '"device=0"' -v ${CIENV_CVMFSVOLUME}:/cvmfs ${INTERIM_IMAGE} -e 100 -t 8 -c 1
     # run shrinkwrapper
-    docker exec cvmfs_${CI_JOB_ID} /root/shrinkwrap.sh -t /tmp/traces/ -e ${CVMFS_EXPORT_DIR}
+    docker exec cvmfs_${CI_JOB_ID} /root/shrinkwrap.sh -t /tmp/traces -e ${CVMFS_EXPORT_DIR}
+    # get the generated traces from the container
+    docker cp cvmfs_${CI_JOB_ID}:/tmp/traces ${CI_PROJECT_DIR}/traces
+    # TODO: check if fixed in CI via traces/cms.cern.ch_spec_custom.txt
     # FIXME this is a dirty patch needed to make scipy running. cvmfs shrinkwrapper alone does not copy all files of that dir. To be investigated why
     ls -lR ${CVMFS_EXPORT_DIR}/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/py2-scipy/1.2.3-bcolbf/lib/python2.7 >${CI_PROJECT_DIR}/cvmfs_export_py2-scipy_content
     rm -fr ${CVMFS_EXPORT_DIR}/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/py2-scipy/1.2.3-bcolbf/lib/python2.7/site-packages
     docker cp patatrack_container:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/py2-scipy/1.2.3-bcolbf/lib/python2.7/site-packages ${CVMFS_EXPORT_DIR}/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/py2-scipy/1.2.3-bcolbf/lib/python2.7
-    # remove duplicated data
+    # TODO: include sha5checksum of exported CVMFS?
+    # tar -cf /tmp/cvmfs_checksum.tar ${CVMFS_EXPORT_DIR}/cvmfs && md5sum /tmp/cvmfs_checksum.tar | cut -f1 -d" " > /tmp/cvmfs_checksum.txt && rm /tmp/cvmfs_checksum.tar
+    # mv /tmp/cvmfs_checksum.txt ${CVMFS_EXPORT_DIR}/cms/patatrack/cvmfs
+    # remove duplicated data & empty dirs
+    # find ${CVMFS_EXPORT_DIR} -type d -empty -delete
     rm -rf ${CVMFS_EXPORT_DIR}/cvmfs/.data
-    ls -R ${CVMFS_EXPORT_DIR} >${CI_PROJECT_DIR}/cvmfs_export_dir_content
+    ls -R ${CVMFS_EXPORT_DIR} >${CI_PROJECT_DIR}/cvmfs_export_dir_content.txt
 }
 
 function _after_script() {
+    #sigkill and remove containers, images remain.
     docker rm -f cvmfs_${CI_JOB_ID}
     docker rm -f patatrack_container
 }
 
+# TODO: clean up $CIENV_CVMFSVOLUME, clean up docker image cache
 export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
 export CVMFS_EXPORT_DIR=${CI_PROJECT_DIR}/cms/patatrack
 export CIENV_CVMFSREPO=cms.cern.ch
-export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
\ No newline at end of file
+export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:qa
+export INTERIM_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-gpu/cms/cms-patatrack-nvidia-interim:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
\ No newline at end of file
diff --git a/cms/patatrack/cms-patatrack/bmk-driver.sh b/cms/patatrack/cms-patatrack/bmk-driver.sh
index e711aeb..6c1215b 100644
--- a/cms/patatrack/cms-patatrack/bmk-driver.sh
+++ b/cms/patatrack/cms-patatrack/bmk-driver.sh
@@ -324,7 +324,6 @@ function doOneWrapper(){
     #   local status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee!
     #   chown -R root:root $workDir 2>&1 | tee -a $log
     # else
-      myecho "[doOneWrapper ($1)] not inside docker - run doOne as $(whoami)\n" 2>&1 | tee -a $log
       doOne $1 2>&1 | tee -a $log
       local status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee!
     # fi
diff --git a/cms/patatrack/cms-patatrack/cms-patatrack-bmk.sh b/cms/patatrack/cms-patatrack/cms-patatrack-bmk.sh
index eed2c5c..71d2706 100755
--- a/cms/patatrack/cms-patatrack/cms-patatrack-bmk.sh
+++ b/cms/patatrack/cms-patatrack/cms-patatrack-bmk.sh
@@ -52,21 +52,21 @@ function validateInputArguments(){
 
   export CMSSW_RELEASE=CMSSW_11_1_0_pre8_Patatrack
   export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
-  export LC_ALL=en_US.UTF-8
 
+  # TODO: These functions are *bad* in anything non-privileged! we can only r/w in mounted DIRs!
+  cd ${resultsDir}
   source $VO_CMS_SW_DIR/cmsset_default.sh
   [[ ! -e ${CMSSW_RELEASE} ]] && scram project CMSSW ${CMSSW_RELEASE}
   cd ${CMSSW_RELEASE}/src; 
+  # sets all paths to point to CMVFS; ignores system binaries
   eval `scramv1 runtime -sh`;
   cd -
 
   env | grep LD_LIBRARY_PATH
   env | grep SRT_LD_LIBRARY_PATH_SCRAMRT
 
-  # FIXME: so far, after having snapshotted cvmfs the LD_LIBRARY_PATH
-  # FIXME: does not contain all path needed as when cvmfs is bind mounted
-  # FIXME: therefore I'm forcing it to be as the correct one
-  export LD_LIBRARY_PATH=/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/biglib/slc7_amd64_gcc820:/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/lib/slc7_amd64_gcc820:/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/external/slc7_amd64_gcc820/lib:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/biglib/slc7_amd64_gcc820:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/lib/slc7_amd64_gcc820:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/external/slc7_amd64_gcc820/lib:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/llvm/9.0.1-pfdnen/lib64:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/gcc/8.2.0-bcolbf/lib64:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/gcc/8.2.0-bcolbf/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64
+  # TODO: This error comes from making CVMFS 555 - paths are set via <pkg>/<version>/etc/init.d/*
+  export LD_LIBRARY_PATH=/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/biglib/slc7_amd64_gcc820:/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/lib/slc7_amd64_gcc820:/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/external/slc7_amd64_gcc820/lib:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/biglib/slc7_amd64_gcc820:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/lib/slc7_amd64_gcc820:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/external/slc7_amd64_gcc820/lib:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/llvm/9.0.1-pfdnen/lib64:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/gcc/8.2.0-bcolbf/lib64:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/gcc/8.2.0-bcolbf/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/.singularity.d/libs
   
   export SRT_LD_LIBRARY_PATH_SCRAMRT=/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/biglib/slc7_amd64_gcc820:/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/lib/slc7_amd64_gcc820:/bmk/cms-patatrack/CMSSW_11_1_0_pre8_Patatrack/external/slc7_amd64_gcc820/lib:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/biglib/slc7_amd64_gcc820:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/lib/slc7_amd64_gcc820:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/cms/cmssw/CMSSW_11_1_0_pre8_Patatrack/external/slc7_amd64_gcc820/lib:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/llvm/9.0.1-pfdnen/lib64:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/gcc/8.2.0-bcolbf/lib64:/cvmfs/cms.cern.ch/slc7_amd64_gcc820/external/gcc/8.2.0-bcolbf/lib
 
@@ -91,9 +91,12 @@ fi
 
 export LC_ALL=en_US.UTF-8
 
-# Source the common benchmark driver
+# Execute common benchmark driver
 if [ -f $(dirname $0)/bmk-driver.sh ]; then
   . $(dirname $0)/bmk-driver.sh
-else
+elif [ -f $(dirname $0)/../../../common/bmk-driver.sh ]; then
   . $(dirname $0)/../../../common/bmk-driver.sh
+else
+  echo "Something went wrong with the default work dir. Could not find bmk-driver.sh";
+  exit 1;
 fi
diff --git a/cms/patatrack/cms-patatrack/cms-reco-bmk.sh b/cms/patatrack/cms-patatrack/cms-reco-bmk.sh
deleted file mode 100755
index 6a91e8f..0000000
--- a/cms/patatrack/cms-patatrack/cms-reco-bmk.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/bin/bash
-
-#set -x # enable debug printouts
-
-#set -e # immediate exit on error
-
-# Function doOne must be defined in each benchmark
-# Input argument $1: process index (between 1 and $NCOPIES)
-# Return value: please return 0 if this workload copy was successful, 1 otherwise
-# The following variables are guaranteed to be defined and exported: NCOPIES, NTHREADS, NEVENTS_THREAD, BMKDIR, DEBUG
-# The function is started in process-specific working directory <basewdir>/proc_$1:
-# please store here the individual log files for each of the NCOPIES processes
-function doOne(){
-  if [ "$1" == "" ] || [ "$2" != "" ]; then echo "[doOne] ERROR! Invalid arguments '$@' to doOne"; return 1; fi
-  echo "[doOne ($1)] $(date) starting in $(pwd)"
-  # Extra CMS-RECO-specific setup
-  export CMSSW_RELEASE=CMSSW_10_2_9
-  export VO_CMS_SW_DIR=/cvmfs/cms.cern.ch
-  source $VO_CMS_SW_DIR/cmsset_default.sh
-  export SCRAM_ARCH=slc6_amd64_gcc700
-  [[ ! -e ${CMSSW_RELEASE} ]] && scram project CMSSW ${CMSSW_RELEASE}
-  pushd ${CMSSW_RELEASE}; eval `scramv1 runtime -sh`; popd
-  # Configure WL copy
-  ln -s ${BMKDIR}/data/GlobalTag.db ./GlobalTag.db
-  ln -s ${BMKDIR}/data/*.root .
-  CMSSW_CONF=step3_RAW2DIGI_L1Reco_RECO_EI_PAT_DQM.py
-  JOB_EVENTS=$(( NEVENTS_THREAD * NTHREADS )) # bash shell arithmetic, may use var instead of $var
-  cp ${BMKDIR}/${CMSSW_CONF}_template ./${CMSSW_CONF}
-  sed -e "s@_NEVENTS_@${JOB_EVENTS}@g" -e "s@_NTHREADS_@$NTHREADS@g" -i ./${CMSSW_CONF}
-  # Execute WL copy
-  LOG=out_$1.log
-  cmsRun ./${CMSSW_CONF} >>$LOG 2>&1 3>&1
-  status=${?}
-  echo "[doOne ($1)] $(date) completed (status=$status)"
-  # Return 0 if this workload copy was successful, 1 otherwise
-  return $status
-}
-
-# Optional function validateInputArguments may be defined in each benchmark
-# If it exists, it is expected to set NCOPIES, NTHREADS, NEVENTS_THREAD
-# (based on previous defaults and on user inputs USER_NCOPIES, USER_NTHREADS, USER_NEVENTS_THREADS)
-# Input arguments: none
-# Return value: please return 0 if input arguments are valid, 1 otherwise
-# The following variables are guaranteed to be defined: NCOPIES, NTHREADS, NEVENTS_THREAD
-# (benchmark defaults) and USER_NCOPIES, USER_NTHREADS, USER_NEVENTS_THREADS (user inputs)
-function validateInputArguments(){
-  if [ "$1" != "" ]; then echo "[validateInputArguments] ERROR! Invalid arguments '$@' to validateInputArguments"; return 1; fi
-  echo "[validateInputArguments] validate input arguments"
-  # Number of copies and number of threads per copy
-  if [ "$USER_NTHREADS" != "" ] && [ "$USER_NCOPIES" != "" ]; then
-    NCOPIES=$USER_NCOPIES
-    NTHREADS=$USER_NTHREADS
-  elif [ "$USER_NTHREADS" != "" ]; then
-    NTHREADS=$USER_NTHREADS
-    NCOPIES=$((`nproc`/$NTHREADS))
-  elif [ "$USER_NCOPIES" != "" ]; then
-    NCOPIES=$USER_NCOPIES
-    NTHREADS=$((`nproc`/$NCOPIES))
-  fi
-  # Number of events per thread
-  if [ "$USER_NEVENTS_THREAD" != "" ]; then NEVENTS_THREAD=$USER_NEVENTS_THREAD; fi
-  # Return 0 if input arguments are valid, 1 otherwise
-  # Report any issues to parseResults via s_msg
-  export s_msg="ok"
-  tot_load=$(($NCOPIES*$NTHREADS))
-  if [ $tot_load -gt `nproc` ]; then
-    s_msg="[ERROR] NCOPIES*NTHREADS=$NCOPIES*$NTHREADS=$tot_load > number of available cores (`nproc`)"
-    return 1
-  elif [ $tot_load -eq 0 ]; then
-    s_msg="[ERROR] NCOPIES*NTHREADS=$NCOPIES*$NTHREADS=$tot_load. Please fix it"
-    return 1
-  elif [ $tot_load -ne `nproc` ];
-    then s_msg="[WARNING] NCOPIES*NTHREADS ($NCOPIES*$NTHREADS=$tot_load) != `nproc` (number of available cores nproc)"
-    echo $s_msg
-  fi
-  return 0
-}
-
-# Default values for NCOPIES, NTHREADS, NEVENTS_THREAD must be set in each benchmark
-NTHREADS=4
-NCOPIES=$(( `nproc` / $NTHREADS ))
-NEVENTS_THREAD=100
-if [ "$NCOPIES" -lt 1 ]; then # when $NTHREADS > nproc
-  NCOPIES=1
-  NTHREADS=`nproc`
-fi
-
-# Source the common benchmark driver
-if [ -f $(dirname $0)/bmk-driver.sh ]; then
-  . $(dirname $0)/bmk-driver.sh
-else
-  . $(dirname $0)/../../../common/bmk-driver.sh
-fi
diff --git a/cms/patatrack/traces/cms.cern.ch_spec_custom.txt b/cms/patatrack/traces/cms.cern.ch_spec_custom.txt
index e1badbc..d7a8f40 100644
--- a/cms/patatrack/traces/cms.cern.ch_spec_custom.txt
+++ b/cms/patatrack/traces/cms.cern.ch_spec_custom.txt
@@ -1,3 +1,3 @@
-/slc7_amd64_gcc820/external/py2-future/*
-/slc7_amd64_gcc820/external/py2-numpy/*
-/slc7_amd64_gcc820/external/py2-scipy/*
\ No newline at end of file
+/slc7_amd64_gcc820/external/py2-future/0.18.2-bcolbf/*
+/slc7_amd64_gcc820/external/py2-numpy/1.16.6-bcolbf/*
+/slc7_amd64_gcc820/external/py2-scipy/1.2.3-bcolbf/*
diff --git a/lhc/simpletrack/.simpletrack-ci.yml b/lhc/simpletrack/.simpletrack-ci.yml
index eb43ed8..0c4c78c 100644
--- a/lhc/simpletrack/.simpletrack-ci.yml
+++ b/lhc/simpletrack/.simpletrack-ci.yml
@@ -14,9 +14,15 @@ stages:
     entrypoint: [""]
   environment:
     name: build/$CI_COMMIT_REF_NAME
-  before_script:
+  script:
   - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
-  - export HEP_LATEST="latest"; export HEP_TAG=$CI_COMMIT_TAG; if [ "$CI_COMMIT_REF_NAME" == "qa" ]; then export HEP_LATEST=`echo ci-${CI_COMMIT_REF_NAME}-latest` ; export HEP_TAG=`echo "ci"-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}`; fi
+  - export LATEST="latest"; 
+  - export IMAGE_TAG=`echo ${CI_COMMIT_TAG} | sed -e 's@.*-\(v[0-9\.]*\)@\1@'`; 
+  - if [ "$CI_COMMIT_REF_NAME" == "qa" ]; then export LATEST=`echo ci-${CI_COMMIT_REF_NAME}-latest` ; export IMAGE_TAG=`echo "ci"-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}`; fi
+  - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi 
+  - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi 
+  - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi 
+  - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile ${DOCKERFILE} --destination ${IMAGE_NAME}:$IMAGE_TAG --destination ${IMAGE_NAME}:$LATEST
 
 .job_template: &job_test
   stage: test
@@ -48,23 +54,27 @@ stages:
 
 build:nvidia:
   <<: *job_build
-  script:
-  - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.nvidia --destination ${CI_REGISTRY_IMAGE}/simpletrack/nvidia:$HEP_TAG --destination ${CI_REGISTRY_IMAGE}/simpletrack/nvidia:$HEP_LATEST
+  before_script:
+    - export DOCKERFILE=$CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.nvidia
+    - export IMAGE_NAME=${CI_REGISTRY_IMAGE}/simpletrack/nvidia
 
 build:intel:
   <<: *job_build
-  script:
-  - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.intel  --destination ${CI_REGISTRY_IMAGE}/simpletrack/intel:$HEP_TAG --destination ${CI_REGISTRY_IMAGE}/simpletrack/intel:$HEP_LATEST
+  before_script:
+    - export DOCKERFILE=$CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.intel
+    - export IMAGE_NAME=${CI_REGISTRY_IMAGE}/simpletrack/intel
 
 build:rocm:
   <<: *job_build
-  script:
-  - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.rocm --destination $CI_REGISTRY_IMAGE/simpletrack/rocm:$HEP_TAG --destination $CI_REGISTRY_IMAGE/simpletrack/rocm:$HEP_LATEST
+  before_script:
+    - export DOCKERFILE=$CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.rocm
+    - export IMAGE_NAME=${CI_REGISTRY_IMAGE}/simpletrack/rocm
 
 build:pocl:
   <<: *job_build
-  script:
-  - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.pocl  --destination ${CI_REGISTRY_IMAGE}/simpletrack/pocl:$HEP_TAG --destination ${CI_REGISTRY_IMAGE}/simpletrack/pocl:$HEP_LATEST
+  before_script:
+    - export DOCKERFILE=$CI_PROJECT_DIR/lhc/simpletrack/Dockerfile.pocl
+    - export IMAGE_NAME=${CI_REGISTRY_IMAGE}/simpletrack/pocl
 
 ##########################
 ### Test #################
-- 
GitLab