From 6320e1acd7141419dfdbd4c6115f03249ceddd1d Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 13:32:34 +0200
Subject: [PATCH 01/48] Remove main.sh from the hepwl builder image (BMK-145)

---
 .gitlab-ci.yml                        | 10 +++++-----
 build-executor/{ => image}/Dockerfile | 14 +-------------
 2 files changed, 6 insertions(+), 18 deletions(-)
 rename build-executor/{ => image}/Dockerfile (81%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 770991c8..c02bbffd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -18,16 +18,14 @@ job_build-hepwlbuilder:
     refs:
     - qa
     changes:
-    - build-executor/*
-    - .gitlab-ci.*
+    - build-executor/image/*
   script:
     # Prepare Kaniko configuration file
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
     # Build and push the image from the Dockerfile at the root of the project.
     # To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
     # See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
-    # Build arg CACHEBUST=$CI_BUILD_ID partially ignores the docker cache (adding --cache=false would fully ignore it?)
-    - /kaniko/executor --context $CI_PROJECT_DIR/build-executor --dockerfile $CI_PROJECT_DIR/build-executor/Dockerfile --destination $CI_REGISTRY_IMAGE/hep-workload-builder:latest --build-arg CACHEBUST=$CI_BUILD_ID
+    - /kaniko/executor --context $CI_PROJECT_DIR/build-executor/image --dockerfile $CI_PROJECT_DIR/build-executor/image/Dockerfile --destination $CI_REGISTRY_IMAGE/hep-workload-builder:latest
 
 ###
 ### Template for all the build jobs of hep-workloads containers
@@ -59,6 +57,7 @@ job_test-hepwlbuilder-ci:
     - qa
     changes:
     - build-executor/*
+    - build-executor/*/*
     - .gitlab-ci.*
     - test/ci/*
     - test/ci/*/*
@@ -73,6 +72,7 @@ job_test-hepwlbuilder-kv:
     - qa
     changes:
     - build-executor/*
+    - build-executor/*/*
     - .gitlab-ci.*
   <<: *build-hepwl-anchor
   stage: test-hepwlbuilder
@@ -108,7 +108,7 @@ job_build-atlas-kv:
   except: # prevent unnecessary KV rebuild if already done in test-hepwlbuilder stage (BMK-119)
     changes:
     - build-executor/*
-    - build-executor/singularity/*
+    - build-executor/*/*
     - .gitlab-ci.*
   <<: *build-hepwl-anchor
 
diff --git a/build-executor/Dockerfile b/build-executor/image/Dockerfile
similarity index 81%
rename from build-executor/Dockerfile
rename to build-executor/image/Dockerfile
index 5a6704e1..b9c8aba5 100644
--- a/build-executor/Dockerfile
+++ b/build-executor/image/Dockerfile
@@ -36,16 +36,4 @@ RUN yum install -y singularity-runtime && yum clean all
 RUN sed -i -e 's/underlay = no/underlay = yes/g' /etc/singularity/singularity.conf
 RUN sed -i -e 's/overlay = try/overlay = no/g' /etc/singularity/singularity.conf
 
-# Allow disabling the cache only from this point onwards if using
-#   docker build -t your-image --build-arg CACHEBUST=$(date +%s) .
-# See https://github.com/moby/moby/issues/1996#issuecomment-185872769
-
-ARG CACHEBUST=1
-
-RUN echo CACHEBUST=$CACHEBUST
-
-COPY ./main.sh /root/
-
-COPY ./generate_Dockerfile.sh /root/
-
-#ENTRYPOINT ["/root/main.sh"]
+ENTRYPOINT ["/root/main.sh"]
-- 
GitLab


From b783dabe4a548986dc092ccad084803ad54aad07 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 13:37:08 +0200
Subject: [PATCH 02/48] Test the ci

---
 .gitlab-ci.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index 2c514088..f5b5963d 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -29,6 +29,8 @@ echo -e "\n[gitlab-ci.sh] $stage starting at $(date)\n"
 echo -e "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
 echo "Current directory is $(pwd)"
 
+exit 0 # TEST
+
 # Sanity check: this script assumes the gitlab CI runner's toml is configured
 # with builds_dir="/scratch/builds" and volumes=["/scratch:/scratch",...], so
 # that the gitlab container's $PWD can be seen by the docker-in-docker main.sh
-- 
GitLab


From 6bae52ca8612a9e4e700449dac5435b0271e9178 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 13:52:25 +0200
Subject: [PATCH 03/48] Fix entrypoint

---
 build-executor/image/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-executor/image/Dockerfile b/build-executor/image/Dockerfile
index b9c8aba5..59847475 100644
--- a/build-executor/image/Dockerfile
+++ b/build-executor/image/Dockerfile
@@ -36,4 +36,4 @@ RUN yum install -y singularity-runtime && yum clean all
 RUN sed -i -e 's/underlay = no/underlay = yes/g' /etc/singularity/singularity.conf
 RUN sed -i -e 's/overlay = try/overlay = no/g' /etc/singularity/singularity.conf
 
-ENTRYPOINT ["/root/main.sh"]
+ENTRYPOINT ["/bin/bash"]
-- 
GitLab


From 0bba945f8ee1acac68fffc713700f9687d5ba1b1 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 14:12:03 +0200
Subject: [PATCH 04/48] Try no ENTRYPOINT

---
 build-executor/image/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-executor/image/Dockerfile b/build-executor/image/Dockerfile
index 59847475..953dd87d 100644
--- a/build-executor/image/Dockerfile
+++ b/build-executor/image/Dockerfile
@@ -36,4 +36,4 @@ RUN yum install -y singularity-runtime && yum clean all
 RUN sed -i -e 's/underlay = no/underlay = yes/g' /etc/singularity/singularity.conf
 RUN sed -i -e 's/overlay = try/overlay = no/g' /etc/singularity/singularity.conf
 
-ENTRYPOINT ["/bin/bash"]
+#ENTRYPOINT ["/bin/bash"]
-- 
GitLab


From 3aac2c3153ee7b6f91156b2645cb3741f56a6036 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 14:29:52 +0200
Subject: [PATCH 05/48] Remove ENTRYPOINT and add some commands

---
 build-executor/image/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/build-executor/image/Dockerfile b/build-executor/image/Dockerfile
index 953dd87d..c33a3581 100644
--- a/build-executor/image/Dockerfile
+++ b/build-executor/image/Dockerfile
@@ -36,4 +36,6 @@ RUN yum install -y singularity-runtime && yum clean all
 RUN sed -i -e 's/underlay = no/underlay = yes/g' /etc/singularity/singularity.conf
 RUN sed -i -e 's/overlay = try/overlay = no/g' /etc/singularity/singularity.conf
 
-#ENTRYPOINT ["/bin/bash"]
+# No ENTRYPOINT
+# See https://gitlab.com/gitlab-org/gitlab-runner/issues/2109
+# See https://stackoverflow.com/a/33219131/1831046
-- 
GitLab


From b142720f230c12792c3272d4a57a6e4c2df205d5 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 14:31:47 +0200
Subject: [PATCH 06/48] CI test

---
 .gitlab-ci.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index f5b5963d..c2f7d8b5 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -29,8 +29,6 @@ echo -e "\n[gitlab-ci.sh] $stage starting at $(date)\n"
 echo -e "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"
 echo "Current directory is $(pwd)"
 
-exit 0 # TEST
-
 # Sanity check: this script assumes the gitlab CI runner's toml is configured
 # with builds_dir="/scratch/builds" and volumes=["/scratch:/scratch",...], so
 # that the gitlab container's $PWD can be seen by the docker-in-docker main.sh
@@ -58,6 +56,8 @@ if [ "$stage" == "build" ]; then
     echo "$var=${!var}"
   done
 
+  exit 0 # TEST
+
   # Set the default CIENV_BUILDEVENT if not yet set by the gitlab CI or in run_build.sh
   if [ "$CIENV_BUILDEVENT" == "" ]; then CIENV_BUILDEVENT=all; fi
   echo "CIENV_BUILDEVENT=$CIENV_BUILDEVENT"
-- 
GitLab


From 052e4aaa376179c2b078937b8c5e82ec3c66fbf8 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 14:50:31 +0200
Subject: [PATCH 07/48] CI test

---
 .gitlab-ci.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index c2f7d8b5..f04b5086 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -56,8 +56,6 @@ if [ "$stage" == "build" ]; then
     echo "$var=${!var}"
   done
 
-  exit 0 # TEST
-
   # Set the default CIENV_BUILDEVENT if not yet set by the gitlab CI or in run_build.sh
   if [ "$CIENV_BUILDEVENT" == "" ]; then CIENV_BUILDEVENT=all; fi
   echo "CIENV_BUILDEVENT=$CIENV_BUILDEVENT"
@@ -80,6 +78,9 @@ if [ "$stage" == "build" ]; then
     export CIENV_CVMFSVOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID}
     export CIENV_DOCKERREGISTRY=""
   fi
+  echo "CIENV_JOBDIR=$CIENV_JOBDIR"
+  echo "CIENV_CVMFSVOLUME=$CIENV_CVMFSVOLUME"
+  echo "CIENV_DOCKERREGISTRY=$CIENV_DOCKERREGISTRY"
 
   # Prepare the environment for the docker-in-docker main.sh and execute it
   # NB: "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared" would mkdir $CIENV_CVMFSVOLUME if missing, but with owner root!
@@ -88,10 +89,14 @@ if [ "$stage" == "build" ]; then
   mkdir -p $CIENV_CVMFSVOLUME
   if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_CVMFSVOLUME' failed"; exit 1; fi
   set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR?
+
+  exit 0
+
   DOCSOCK="/var/run/docker.sock"
   BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest"
   echo "running privileged docker with fixed entrypoint"
   set -x # verbose on
+
   docker run --rm --privileged --entrypoint=/root/main.sh \
     --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
     -v $CIENV_JOBDIR:$CIENV_JOBDIR \
-- 
GitLab


From 2518800c1a1dd6688c3f0841069b97a1587e9d24 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 14:56:29 +0200
Subject: [PATCH 08/48] CI test

---
 .gitlab-ci.sh  | 27 ++++++++++++++-------------
 .gitlab-ci.yml | 24 ++++++++++++------------
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index f04b5086..a797be3a 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -90,24 +90,25 @@ if [ "$stage" == "build" ]; then
   if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_CVMFSVOLUME' failed"; exit 1; fi
   set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR?
 
-  exit 0
-
-  DOCSOCK="/var/run/docker.sock"
+  # TEST HERE
+  #DOCSOCK="/var/run/docker.sock"
   BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest"
-  echo "running privileged docker with fixed entrypoint"
+  #echo "running privileged docker with fixed entrypoint"
   set -x # verbose on
-
-  docker run --rm --privileged --entrypoint=/root/main.sh \
-    --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
-    -v $CIENV_JOBDIR:$CIENV_JOBDIR \
-    -v $DOCSOCK:$DOCSOCK \
-    -v $CIENV_CVMFSVOLUME:/cvmfs:shared \
-    -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \
-    $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log
-  status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
+  ${pwd}/build-executor/main.sh
+  #docker run --rm --privileged --entrypoint=/root/main.sh \
+  #  --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
+  #  -v $CIENV_JOBDIR:$CIENV_JOBDIR \
+  #  -v $DOCSOCK:$DOCSOCK \
+  #  -v $CIENV_CVMFSVOLUME:/cvmfs:shared \
+  #  -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \
+  #  $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log
+  #status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
   set +x # verbose off
   echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)"
   echo -e "[gitlab-ci.sh] status from main.sh: $status\n"
+  # EXIT
+  exit 0
 
   #Create tarball of JOBDIR results
   echo -e "\n\n--------------\nCreating tarball of json and log results for artifacts\n--------------\n"
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c02bbffd..ca6dac93 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -64,18 +64,18 @@ job_test-hepwlbuilder-ci:
   <<: *build-hepwl-anchor
   stage: test-hepwlbuilder
 
-job_test-hepwlbuilder-kv:
-  variables:
-    CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv/atlas-kv.spec
-  only:
-    refs:
-    - qa
-    changes:
-    - build-executor/*
-    - build-executor/*/*
-    - .gitlab-ci.*
-  <<: *build-hepwl-anchor
-  stage: test-hepwlbuilder
+#job_test-hepwlbuilder-kv:
+#  variables:
+#    CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv/atlas-kv.spec
+#  only:
+#    refs:
+#    - qa
+#    changes:
+#    - build-executor/*
+#    - build-executor/*/*
+#    - .gitlab-ci.*
+#  <<: *build-hepwl-anchor
+#  stage: test-hepwlbuilder
 
 ###
 ### ALICE
-- 
GitLab


From b002e59753a878673908c919d7a2dcba74c557e7 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 14:58:21 +0200
Subject: [PATCH 09/48] CI test

---
 .gitlab-ci.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ca6dac93..c02bbffd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -64,18 +64,18 @@ job_test-hepwlbuilder-ci:
   <<: *build-hepwl-anchor
   stage: test-hepwlbuilder
 
-#job_test-hepwlbuilder-kv:
-#  variables:
-#    CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv/atlas-kv.spec
-#  only:
-#    refs:
-#    - qa
-#    changes:
-#    - build-executor/*
-#    - build-executor/*/*
-#    - .gitlab-ci.*
-#  <<: *build-hepwl-anchor
-#  stage: test-hepwlbuilder
+job_test-hepwlbuilder-kv:
+  variables:
+    CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv/atlas-kv.spec
+  only:
+    refs:
+    - qa
+    changes:
+    - build-executor/*
+    - build-executor/*/*
+    - .gitlab-ci.*
+  <<: *build-hepwl-anchor
+  stage: test-hepwlbuilder
 
 ###
 ### ALICE
-- 
GitLab


From 7568e2f1345359db4b6faed296182050efed909e Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:01:14 +0200
Subject: [PATCH 10/48] CI test

---
 .gitlab-ci.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index a797be3a..a5b304a0 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -95,7 +95,7 @@ if [ "$stage" == "build" ]; then
   BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest"
   #echo "running privileged docker with fixed entrypoint"
   set -x # verbose on
-  ${pwd}/build-executor/main.sh
+  $(pwd)/build-executor/main.sh
   #docker run --rm --privileged --entrypoint=/root/main.sh \
   #  --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
   #  -v $CIENV_JOBDIR:$CIENV_JOBDIR \
-- 
GitLab


From f02867dc61e3604b8e1ab359ce5a6a57ed536ea7 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:13:55 +0200
Subject: [PATCH 11/48] CI test

---
 build-executor/main.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 4693f91e..893f3861 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -640,12 +640,16 @@ if ! load_and_validate_specfile "$CIENV_HEPWL_SPECFILE"; then fail "invalid SPEC
 
 if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then
   mount_cvmfs || fail '[main.sh] mount_cvmfs'
+else
+  echo -e "\n[main.sh] WARNING: cvmfs will not be mounted\n"
 fi
 
 execute_procedure "$CIENV_BUILDEVENT"
 
 if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then
   unmount_cvmfs || fail '[main.sh] unmount_cvmfs'
+else
+  echo -e "\n[main.sh] WARNING: cvmfs will not be unmounted\n"
 fi
 
 echo -e "\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
-- 
GitLab


From 04adf0707cb3a3519c93610ad3d2771ed0dde9d5 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:15:47 +0200
Subject: [PATCH 12/48] CI test

---
 .gitlab-ci.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index a5b304a0..161a07ff 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -95,7 +95,6 @@ if [ "$stage" == "build" ]; then
   BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest"
   #echo "running privileged docker with fixed entrypoint"
   set -x # verbose on
-  $(pwd)/build-executor/main.sh
   #docker run --rm --privileged --entrypoint=/root/main.sh \
   #  --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
   #  -v $CIENV_JOBDIR:$CIENV_JOBDIR \
@@ -103,7 +102,8 @@ if [ "$stage" == "build" ]; then
   #  -v $CIENV_CVMFSVOLUME:/cvmfs:shared \
   #  -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \
   #  $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log
-  #status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
+  $(pwd)/build-executor/main.sh | tee -a $CIENV_JOBDIR/main.log
+  status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
   set +x # verbose off
   echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)"
   echo -e "[gitlab-ci.sh] status from main.sh: $status\n"
-- 
GitLab


From 299a8fe593e5afa13ad1033f8fa9c67afd60030d Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:26:47 +0200
Subject: [PATCH 13/48] CI test

---
 .gitlab-ci.sh          | 24 ++++++++++++++++--------
 build-executor/main.sh | 21 +++++++++++++++------
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index 161a07ff..801c809f 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -48,6 +48,7 @@ if [ "$stage" == "build" ]; then
   # Check that the spec file exists and pass it to main.sh as a fully qualified path
   if [ ! -e $CIENV_HEPWL_SPECFILE ]; then echo "ERROR! $CIENV_HEPWL_SPECFILE does not exist"; exit 1; fi
   CIENV_HEPWL_SPECFILE=$(cd $(dirname $CIENV_HEPWL_SPECFILE); pwd)/$(basename $CIENV_HEPWL_SPECFILE)
+  echo "CIENV_HEPWL_SPECFILE=$CIENV_HEPWL_SPECFILE"
 
   # Sanity check: the following CI_ variables must have been set by the gitlab CI or in run_build.sh
   for var in CI_JOB_ID CI_JOB_NAME CI_PROJECT_DIR CI_REGISTRY_IMAGE; do
@@ -57,7 +58,7 @@ if [ "$stage" == "build" ]; then
   done
 
   # Set the default CIENV_BUILDEVENT if not yet set by the gitlab CI or in run_build.sh
-  if [ "$CIENV_BUILDEVENT" == "" ]; then CIENV_BUILDEVENT=all; fi
+  if [ "$CIENV_BUILDEVENT" == "" ]; then export CIENV_BUILDEVENT=all; fi
   echo "CIENV_BUILDEVENT=$CIENV_BUILDEVENT"
 
   # Set CIENV_MOUNTCVMFS=1 in the gitlab CI as it is not yet set
@@ -70,13 +71,13 @@ if [ "$stage" == "build" ]; then
   # Set other environment variables which were previously in the runner's toml
   # Skip image publishing and use a different naming convention in noCI mode
   if [ "$CI_JOB_NAME" != "noCI" ]; then
-    export CIENV_JOBDIR=/scratch/logs/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml
-    export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml
-    export CIENV_DOCKERREGISTRY=${CI_REGISTRY_IMAGE} # previously in runner's toml
+    CIENV_JOBDIR=/scratch/logs/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml
+    CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml
+    CIENV_DOCKERREGISTRY=${CI_REGISTRY_IMAGE} # previously in runner's toml
   else
-    export CIENV_JOBDIR=/tmp/${USER}/logs/noCI-${CI_JOB_ID}
-    export CIENV_CVMFSVOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID}
-    export CIENV_DOCKERREGISTRY=""
+    CIENV_JOBDIR=/tmp/${USER}/logs/noCI-${CI_JOB_ID}
+    CIENV_CVMFSVOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID}
+    CIENV_DOCKERREGISTRY=""
   fi
   echo "CIENV_JOBDIR=$CIENV_JOBDIR"
   echo "CIENV_CVMFSVOLUME=$CIENV_CVMFSVOLUME"
@@ -88,7 +89,14 @@ if [ "$stage" == "build" ]; then
   if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_JOBDIR' failed"; exit 1; fi
   mkdir -p $CIENV_CVMFSVOLUME
   if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_CVMFSVOLUME' failed"; exit 1; fi
-  set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR?
+
+  # These environment variables were previsouly passed to main.sh within docker via the file .env.file
+  # They are now simply exported to the main.sh subprocess (BMK-145)
+  # [NB: it is assumed here that all CI_ variables have been exported by the gitlab-runner CI]
+  export CIENV_JOBDIR
+  export CIENV_CVMFSVOLUME
+  export CIENV_MOUNTCVMFS
+  export CIENV_DOCKERREGISTRY
 
   # TEST HERE
   #DOCSOCK="/var/run/docker.sock"
diff --git a/build-executor/main.sh b/build-executor/main.sh
index 893f3861..36561047 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -570,12 +570,21 @@ echo "scriptDir is ${scriptDir}"
 export MAIN_CVMFSTRACESDIR=$CIENV_JOBDIR/cvmfs-traces/
 export MAIN_CVMFSEXPORTDIR=$CIENV_JOBDIR/cvmfs-export/
 
-if [ -z $CIENV_HEPWL_SPECFILE ]; then CIENV_HEPWL_SPECFILE=''; fi
-if [ -z $CIENV_BUILDEVENT ]; then CIENV_BUILDEVENT="all"; fi
-if [ -z $CIENV_DOCKERREGISTRY ]; then CIENV_DOCKERREGISTRY=""; fi
-if [ -z $CIENV_MOUNTCVMFS ]; then CIENV_MOUNTCVMFS=0; fi
-if [ -z $CIENV_JOBDIR ]; then CIENV_JOBDIR=/tmp; fi
-if [ -z $MAIN_NETWORKCONN ]; then MAIN_NETWORKCONN=0; fi
+# Previously these environment variables were reset to dafult values here
+# To make this cleaner (BMK-145), it is now expected that they must have been set in advance
+#if [ -z $CIENV_HEPWL_SPECFILE ]; then CIENV_HEPWL_SPECFILE=''; fi
+#if [ -z $CIENV_BUILDEVENT ]; then CIENV_BUILDEVENT="all"; fi
+#if [ -z $CIENV_DOCKERREGISTRY ]; then CIENV_DOCKERREGISTRY=""; fi
+#if [ -z $CIENV_MOUNTCVMFS ]; then CIENV_MOUNTCVMFS=0; fi
+#if [ -z $CIENV_JOBDIR ]; then CIENV_JOBDIR=/tmp; fi
+#if [ -z $MAIN_NETWORKCONN ]; then MAIN_NETWORKCONN=0; fi
+for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN abcdef; do
+  # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
+  if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
+  echo "$var=${!var}"
+done
+
+if [ "$CIENV_HEPWL_SPECFILE" == "" ]; then fail "SPEC file not specified"; fi
 
 # Parameters to be passed by command line
 # 1: SPEC file path
-- 
GitLab


From fc86d730aa6f50715406ef0762e9a140f6045ae2 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:33:14 +0200
Subject: [PATCH 14/48] CI test

---
 build-executor/main.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 36561047..24784d61 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -572,12 +572,8 @@ export MAIN_CVMFSEXPORTDIR=$CIENV_JOBDIR/cvmfs-export/
 
 # Previously these environment variables were reset to dafult values here
 # To make this cleaner (BMK-145), it is now expected that they must have been set in advance
-#if [ -z $CIENV_HEPWL_SPECFILE ]; then CIENV_HEPWL_SPECFILE=''; fi
-#if [ -z $CIENV_BUILDEVENT ]; then CIENV_BUILDEVENT="all"; fi
-#if [ -z $CIENV_DOCKERREGISTRY ]; then CIENV_DOCKERREGISTRY=""; fi
-#if [ -z $CIENV_MOUNTCVMFS ]; then CIENV_MOUNTCVMFS=0; fi
-#if [ -z $CIENV_JOBDIR ]; then CIENV_JOBDIR=/tmp; fi
-#if [ -z $MAIN_NETWORKCONN ]; then MAIN_NETWORKCONN=0; fi
+# [NB MAIN_NETWORKCONN, which was only used in main.sh, is now a local variable]
+MAIN_NETWORKCONN=0 # this can be overridden via the -n command line option
 for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN abcdef; do
   # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
   if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
-- 
GitLab


From 51a81ba7ff6d350d9f76d7a963d9ff8da3319642 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:36:52 +0200
Subject: [PATCH 15/48] CI test

---
 .gitlab-ci.sh          | 2 +-
 build-executor/main.sh | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index 801c809f..7d8a48c3 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -116,7 +116,7 @@ if [ "$stage" == "build" ]; then
   echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)"
   echo -e "[gitlab-ci.sh] status from main.sh: $status\n"
   # EXIT
-  exit 0
+  exit $status
 
   #Create tarball of JOBDIR results
   echo -e "\n\n--------------\nCreating tarball of json and log results for artifacts\n--------------\n"
diff --git a/build-executor/main.sh b/build-executor/main.sh
index 24784d61..3fc0e6a3 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -574,14 +574,12 @@ export MAIN_CVMFSEXPORTDIR=$CIENV_JOBDIR/cvmfs-export/
 # To make this cleaner (BMK-145), it is now expected that they must have been set in advance
 # [NB MAIN_NETWORKCONN, which was only used in main.sh, is now a local variable]
 MAIN_NETWORKCONN=0 # this can be overridden via the -n command line option
-for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN abcdef; do
+for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN; do
   # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
   if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
   echo "$var=${!var}"
 done
 
-if [ "$CIENV_HEPWL_SPECFILE" == "" ]; then fail "SPEC file not specified"; fi
-
 # Parameters to be passed by command line
 # 1: SPEC file path
 # 2: optional export CIENV_DOCKERREGISTRY gitlab-registry.cern.ch/giordano/hep-workloads
-- 
GitLab


From 3029360f798f53ccbaf0d969da8b1d98adf4dfc6 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:41:10 +0200
Subject: [PATCH 16/48] CI test

---
 build-executor/main.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 3fc0e6a3..62184639 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -100,6 +100,7 @@ EOF
     rm -rf /cvmfs/$repo
     mkdir /cvmfs/$repo # Assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
+    ls -l /cvmfs/$repo
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
-- 
GitLab


From e88f92a3f817ea2c6072c225413cecd3d6c8ffb9 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:46:20 +0200
Subject: [PATCH 17/48] CI test

---
 build-executor/main.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 62184639..e1ff2b7a 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -90,9 +90,9 @@ CVMFS_MOUNT_RW=yes
 CVMFS_HTTP_PROXY="http://squid.cern.ch:8060|http://ca-proxy.cern.ch:3128;DIRECT"
 CVMFS_TRACEFILE=${MAIN_CVMFSTRACESDIR}/cvmfs-@fqrn@.trace.log
 EOF
-  mkdir -p  /etc/cvmfs/config.d
+  mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
-  #if [[ `ls /cvmfs | wc -l` -gt 0 ]]; then fail "/cvmfs already contains files: remove them"; fi
+  if [[ `ls /cvmfs | wc -l` -gt 0 ]]; then fail "/cvmfs already contains files: remove them"; fi
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart"
   [ -d /cvmfs ] || fail "[mount_cvmfs] /cvmfs does not exist" # Assume that /cvmfs has been created by now (e.g. by docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared)
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
-- 
GitLab


From 8150c3ba3b95a6137ef405a7a4464eb63879b187 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:51:55 +0200
Subject: [PATCH 18/48] CI test

---
 build-executor/main.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index e1ff2b7a..3a354159 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -92,8 +92,10 @@ CVMFS_TRACEFILE=${MAIN_CVMFSTRACESDIR}/cvmfs-@fqrn@.trace.log
 EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
-  if [[ `ls /cvmfs | wc -l` -gt 0 ]]; then fail "/cvmfs already contains files: remove them"; fi
+  #if [[ `ls /cvmfs | wc -l` -gt 0 ]]; then fail "/cvmfs already contains files: remove them"; fi
+  if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart"
+  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi
   [ -d /cvmfs ] || fail "[mount_cvmfs] /cvmfs does not exist" # Assume that /cvmfs has been created by now (e.g. by docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared)
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
     umount /cvmfs/$repo
-- 
GitLab


From 14dbe86a95490d277412a3afe187c74c4f50cc4d Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 15:57:56 +0200
Subject: [PATCH 19/48] CI test

---
 build-executor/main.sh | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 3a354159..c9e4f9ec 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -92,17 +92,16 @@ CVMFS_TRACEFILE=${MAIN_CVMFSTRACESDIR}/cvmfs-@fqrn@.trace.log
 EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
-  #if [[ `ls /cvmfs | wc -l` -gt 0 ]]; then fail "/cvmfs already contains files: remove them"; fi
   if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
-  cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart"
-  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi
-  [ -d /cvmfs ] || fail "[mount_cvmfs] /cvmfs does not exist" # Assume that /cvmfs has been created by now (e.g. by docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared)
+  ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
+  cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
+  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
-    umount /cvmfs/$repo
-    rm -rf /cvmfs/$repo
-    mkdir /cvmfs/$repo # Assume that /cvmfs has been created by now
+    mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
     ls -l /cvmfs/$repo
+    sleep 200
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
-- 
GitLab


From a135614009af02f96d85d4e78c6edbade46d9a8b Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:13:46 +0200
Subject: [PATCH 20/48] CI test

---
 build-executor/main.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index c9e4f9ec..d0621e9b 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -93,8 +93,8 @@ EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
   if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
-  ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
-  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
+  #ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  #if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
@@ -177,7 +177,8 @@ function run_docker_wl(){
   strace=""
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
-  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  #execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
-- 
GitLab


From 7a838c33c9769f6f00f6871e71da8f1de9feb838 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:14:30 +0200
Subject: [PATCH 21/48] CI test

---
 build-executor/main.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index d0621e9b..dda82f9e 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -101,7 +101,6 @@ EOF
     mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
     ls -l /cvmfs/$repo
-    sleep 200
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
-- 
GitLab


From 1da4269580acd0b136cf198c1d6d4cc82e8623e3 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:29:42 +0200
Subject: [PATCH 22/48] CI test

---
 build-executor/main.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index dda82f9e..75d328c8 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -176,7 +176,7 @@ function run_docker_wl(){
   strace=""
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
-  #execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
@@ -276,7 +276,8 @@ function copy_cvmfs(){
     rm -rf $MAIN_HEPWLBUILDDIR/cvmfs/cms.cern.ch/SITECONF/local
   fi
   mkdir -p $MAIN_HEPWLBUILDDIR/cvmfs/cms.cern.ch/SITECONF/local/JobConfig # empty
-  ls $MAIN_HEPWLBUILDDIR/cvmfs
+  echo "[copy_cvmfs] /cvmfs contents copied to $MAIN_HEPWLBUILDDIR/cvmfs"
+  ls -l $MAIN_HEPWLBUILDDIR/cvmfs
   echo "[copy_cvmfs] finished at $(date)"
   return 0
 }
-- 
GitLab


From 75b8fc40e8f4c501ca045f3e9fbbdd06afeb83bb Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:30:26 +0200
Subject: [PATCH 23/48] CI test

---
 build-executor/main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 75d328c8..80dc922a 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -538,7 +538,7 @@ function execute_procedure(){
       run_shrinkwrap || fail "[execute_procedure] $1: run_shrinkwrap"
       copy_cvmfs || fail "[execute_procedure] $1: copy_cvmfs"
       build_standalone_image || fail "[execute_procedure] $1: build_standalone_image"
-      clean_cvmfs_copy || fail "[execute_procedure] $1: clean_cvmfs_copy"
+      #clean_cvmfs_copy || fail "[execute_procedure] $1: clean_cvmfs_copy"
       test_standalone_image_docker || fail "[execute_procedure] $1: test_standalone_image_docker"
       test_standalone_image_singularity || fail "[execute_procedure] $1: test_standalone_image_singularity"
       publish_standalone_image || fail "[execute_procedure] $1: publish_standalone_image"
-- 
GitLab


From 302e8e85741dce6338ce00d19d95371ab837c530 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:45:10 +0200
Subject: [PATCH 24/48] CI test

---
 build-executor/main.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 80dc922a..1fdc4491 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -93,14 +93,17 @@ EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
   if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
-  #ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
-  #if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
+  ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
     mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
+    echo "[mount_cvmfs] ls -l /cvmfs/$repo"
     ls -l /cvmfs/$repo
+    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
+    ls -l $CIENV_CVMFSVOLUME
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
@@ -239,7 +242,7 @@ EOF
       echo " " >> ${MAIN_CVMFSTRACESDIR}/$specname
       cat $MAIN_HEPWLBUILDDIR/${!spec_var} >> ${MAIN_CVMFSTRACESDIR}/$specname
     fi
-    echo "[run_shrinkwrap] cvmfs_shrinkwrap --repo $reponame --src-config ${cvmfs_shrink_conf} --spec-file  ${MAIN_CVMFSTRACESDIR}/$specname --dest-base ${MAIN_CVMFSEXPORTDIR}/cvmfs/ -j 4"
+    echo "[run_shrinkwrap] cvmfs_shrinkwrap --repo $reponame --src-config ${cvmfs_shrink_conf} --spec-file ${MAIN_CVMFSTRACESDIR}/$specname --dest-base ${MAIN_CVMFSEXPORTDIR}/cvmfs/ -j 4"
     cvmfs_shrinkwrap --repo $reponame --src-config ${cvmfs_shrink_conf} --spec-file  ${MAIN_CVMFSTRACESDIR}/$specname --dest-base ${MAIN_CVMFSEXPORTDIR}/cvmfs/ -j 4 || fail "[run_shrinkwrap] cvmfs_shrinkwrap failed"
     date
   done
-- 
GitLab


From 0cc01bece405d8d1e2783a697b0b43ba0a538aca Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:46:52 +0200
Subject: [PATCH 25/48] CI test

---
 build-executor/main.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 1fdc4491..e094f193 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -179,8 +179,8 @@ function run_docker_wl(){
   strace=""
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
-- 
GitLab


From c56ef544097e864f003e660b4266214f1865c62b Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 16:54:21 +0200
Subject: [PATCH 26/48] CI test

---
 build-executor/main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index e094f193..39be514d 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -180,7 +180,7 @@ function run_docker_wl(){
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
   execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  #execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
-- 
GitLab


From 93e0a68ce8304b9143f738bb04bfaa7302044ce5 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:07:23 +0200
Subject: [PATCH 27/48] CI test

---
 build-executor/main.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 39be514d..db9c64d2 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -179,8 +179,9 @@ function run_docker_wl(){
   strace=""
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
-  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  #execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
-- 
GitLab


From 8e3a1db71efd8f59c46f955aed44cd50519b2aec Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:08:22 +0200
Subject: [PATCH 28/48] CI test

---
 build-executor/main.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index db9c64d2..bed2248b 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -179,9 +179,13 @@ function run_docker_wl(){
   strace=""
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
+
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+
+  execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
-- 
GitLab


From 1b1dcb164bcfd0d0c4b3c2971dff86dc739cf921 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:20:29 +0200
Subject: [PATCH 29/48] CI test

---
 build-executor/main.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index bed2248b..d51528d3 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -104,6 +104,8 @@ EOF
     ls -l /cvmfs/$repo
     echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
     ls -l $CIENV_CVMFSVOLUME
+    echo "[mount_cvmfs] touch $CIENV_CVMFSVOLUME/pippo"
+    ls -l $CIENV_CVMFSVOLUME
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
-- 
GitLab


From 336d812aa501dc17bb89384b09594bb803d92261 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:25:04 +0200
Subject: [PATCH 30/48] CI test

---
 build-executor/main.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index d51528d3..6214443f 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -102,9 +102,9 @@ EOF
     mount -t cvmfs $repo /cvmfs/$repo
     echo "[mount_cvmfs] ls -l /cvmfs/$repo"
     ls -l /cvmfs/$repo
-    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
-    ls -l $CIENV_CVMFSVOLUME
     echo "[mount_cvmfs] touch $CIENV_CVMFSVOLUME/pippo"
+    touch $CIENV_CVMFSVOLUME/pippo
+    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
     ls -l $CIENV_CVMFSVOLUME
   done
   echo "[mount_cvmfs] finished at $(date)"
-- 
GitLab


From b0708514db3b93aefca6ae5eb963c3cfc7363b5b Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:28:30 +0200
Subject: [PATCH 31/48] CI test

---
 build-executor/main.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 6214443f..e500b1fe 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -106,6 +106,8 @@ EOF
     touch $CIENV_CVMFSVOLUME/pippo
     echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
     ls -l $CIENV_CVMFSVOLUME
+    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
+    ls -l $CIENV_CVMFSVOLUME/$repo
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
-- 
GitLab


From c875368068e1a46af72a5fcda8d99420d05502c6 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:31:26 +0200
Subject: [PATCH 32/48] CI test

---
 build-executor/main.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index e500b1fe..73a0cd5a 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -93,7 +93,8 @@ EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
   if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
-  ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  ###ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  ln -sf /cvmfs $CIENV_CVMFSVOLUME # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
-- 
GitLab


From dceea8c613a45a1d540dd150c4838efb338e289c Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:42:40 +0200
Subject: [PATCH 33/48] CI test

---
 build-executor/main.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 73a0cd5a..b0c7bdd0 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -94,13 +94,15 @@ EOF
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
   if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
   ###ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
-  ln -sf /cvmfs $CIENV_CVMFSVOLUME # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  ###ln -sf /cvmfs $CIENV_CVMFSVOLUME # NO... replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
     mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
+    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
+    ls -l $CIENV_CVMFSVOLUME/$repo
     echo "[mount_cvmfs] ls -l /cvmfs/$repo"
     ls -l /cvmfs/$repo
     echo "[mount_cvmfs] touch $CIENV_CVMFSVOLUME/pippo"
-- 
GitLab


From 6897d65584dd02a5807b9a83239f384da8808cd8 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:44:59 +0200
Subject: [PATCH 34/48] CI test

---
 build-executor/main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index b0c7bdd0..83123636 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -93,7 +93,7 @@ EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
   if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
-  ###ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
   ###ln -sf /cvmfs $CIENV_CVMFSVOLUME # NO... replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
-- 
GitLab


From f7f65ed4ddfc961a3746e1fc64ec5e0be409b99a Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:51:46 +0200
Subject: [PATCH 35/48] CI test

---
 build-executor/main.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 83123636..3edab93c 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -190,9 +190,12 @@ function run_docker_wl(){
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
-  execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ##execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
+  DOCSOCK="/var/run/docker.sock"
+  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
-- 
GitLab


From afb6627eab4aec87ac64ecbcfee19cc370ea152b Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 17:53:31 +0200
Subject: [PATCH 36/48] CI test

---
 build-executor/main.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 3edab93c..084c34bb 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -190,11 +190,12 @@ function run_docker_wl(){
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
-  ##execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
   DOCSOCK="/var/run/docker.sock"
-  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
-- 
GitLab


From 05524ea244ee122cbdbe0a358553265366bd1dbc Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 18:06:54 +0200
Subject: [PATCH 37/48] CI test

---
 build-executor/main.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 084c34bb..221abe91 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -193,8 +193,12 @@ function run_docker_wl(){
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
+  #DOCSOCK="/var/run/docker.sock"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+
   DOCSOCK="/var/run/docker.sock"
-  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
-- 
GitLab


From 358766ebba169fd77ee31325bc84e0887b559709 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 18:33:48 +0200
Subject: [PATCH 38/48] CI test

---
 build-executor/main.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 221abe91..d464b1ec 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -187,20 +187,24 @@ function run_docker_wl(){
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
 
+  ls -lRt ${MAIN_CVMFSTRACESDIR}
+
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
-  #DOCSOCK="/var/run/docker.sock"
+  ###DOCSOCK="/var/run/docker.sock"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
-  DOCSOCK="/var/run/docker.sock"
-  execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###DOCSOCK="/var/run/docker.sock"
+  ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
+  ls -lRt ${MAIN_CVMFSTRACESDIR}
+
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
     echo "[run_docker_wl] cvmfs flush trace for $acvmfs"
@@ -208,6 +212,9 @@ function run_docker_wl(){
       fail "[run_docker_wl] cvmfs flush $acvmfs" # fix BMK-136 (flush was silently failing, leading to missing cvmfs files)
     fi
   done
+
+  ls -lRt ${MAIN_CVMFSTRACESDIR}
+
   echo "[run_docker_wl] finished at $(date)"
   return 0
 }
-- 
GitLab


From dc7db3ddd3f6cebdfb69e343433116b7a82059b4 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 18:41:25 +0200
Subject: [PATCH 39/48] CI test

---
 build-executor/main.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index d464b1ec..9f923bab 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -203,6 +203,7 @@ function run_docker_wl(){
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
+
   ls -lRt ${MAIN_CVMFSTRACESDIR}
 
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
-- 
GitLab


From 99a1c803202401c32a6c500d22c9d2b44129b6e5 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 18:44:33 +0200
Subject: [PATCH 40/48] CI test

---
 build-executor/main.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 9f923bab..d464b1ec 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -203,7 +203,6 @@ function run_docker_wl(){
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
-
   ls -lRt ${MAIN_CVMFSTRACESDIR}
 
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
-- 
GitLab


From 08c17ddc3897403974655fbc766f5f38817df46e Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 19:10:39 +0200
Subject: [PATCH 41/48] CI test

---
 build-executor/main.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index d464b1ec..33e82834 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -112,6 +112,7 @@ EOF
     echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
     ls -l $CIENV_CVMFSVOLUME/$repo
   done
+  sleep 60
   echo "[mount_cvmfs] finished at $(date)"
   return 0
 }
-- 
GitLab


From e15b478ac4e1e6c4a377cc4cc9b4c34a2621a4af Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 19:35:49 +0200
Subject: [PATCH 42/48] CI test

---
 .gitlab-ci.sh | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index 7d8a48c3..a06f29a7 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -99,18 +99,19 @@ if [ "$stage" == "build" ]; then
   export CIENV_DOCKERREGISTRY
 
   # TEST HERE
-  #DOCSOCK="/var/run/docker.sock"
+  DOCSOCK="/var/run/docker.sock"
   BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest"
-  #echo "running privileged docker with fixed entrypoint"
+  echo "running privileged docker with fixed entrypoint"
   set -x # verbose on
-  #docker run --rm --privileged --entrypoint=/root/main.sh \
-  #  --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
-  #  -v $CIENV_JOBDIR:$CIENV_JOBDIR \
-  #  -v $DOCSOCK:$DOCSOCK \
-  #  -v $CIENV_CVMFSVOLUME:/cvmfs:shared \
-  #  -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \
-  #  $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log
-  $(pwd)/build-executor/main.sh | tee -a $CIENV_JOBDIR/main.log
+  ###docker run --rm --privileged --entrypoint=/root/main.sh \
+  docker run --rm --privileged --entrypoint=$CI_PROJECT_DIR/build-executor/main.sh \
+    --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
+    -v $CIENV_JOBDIR:$CIENV_JOBDIR \
+    -v $DOCSOCK:$DOCSOCK \
+    -v $CIENV_CVMFSVOLUME:/cvmfs:shared \
+    -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \
+    $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log
+  ###$(pwd)/build-executor/main.sh | tee -a $CIENV_JOBDIR/main.log
   status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
   set +x # verbose off
   echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)"
-- 
GitLab


From c87b250984dde26f44ca00421421674d6a66ea6d Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 19:38:10 +0200
Subject: [PATCH 43/48] CI test

---
 .gitlab-ci.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index a06f29a7..b12ccdf5 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -93,10 +93,11 @@ if [ "$stage" == "build" ]; then
   # These environment variables were previsouly passed to main.sh within docker via the file .env.file
   # They are now simply exported to the main.sh subprocess (BMK-145)
   # [NB: it is assumed here that all CI_ variables have been exported by the gitlab-runner CI]
-  export CIENV_JOBDIR
-  export CIENV_CVMFSVOLUME
-  export CIENV_MOUNTCVMFS
-  export CIENV_DOCKERREGISTRY
+  #export CIENV_JOBDIR
+  #export CIENV_CVMFSVOLUME
+  #export CIENV_MOUNTCVMFS
+  #export CIENV_DOCKERREGISTRY
+  set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file
 
   # TEST HERE
   DOCSOCK="/var/run/docker.sock"
-- 
GitLab


From 9f93be877191f90985412bcca65b1e34f46fd761 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 19:43:09 +0200
Subject: [PATCH 44/48] CI test

---
 build-executor/main.sh | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 33e82834..45f5abaa 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -608,11 +608,11 @@ export MAIN_CVMFSEXPORTDIR=$CIENV_JOBDIR/cvmfs-export/
 # To make this cleaner (BMK-145), it is now expected that they must have been set in advance
 # [NB MAIN_NETWORKCONN, which was only used in main.sh, is now a local variable]
 MAIN_NETWORKCONN=0 # this can be overridden via the -n command line option
-for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN; do
-  # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
-  if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
-  echo "$var=${!var}"
-done
+#for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN; do
+#  # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
+#  if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
+#  echo "$var=${!var}"
+#done
 
 # Parameters to be passed by command line
 # 1: SPEC file path
@@ -644,6 +644,13 @@ while getopts "hds:r:e:m" o; do
   esac
 done
 
+# BACK
+for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN; do
+  # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
+  if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
+  echo "$var=${!var}"
+done
+
 echo -e "\n-------------\n[main] Input parameters"
 echo CIENV_HEPWL_SPECFILE $CIENV_HEPWL_SPECFILE
 echo CIENV_BUILDEVENT $CIENV_BUILDEVENT
-- 
GitLab


From 7a57503f11bc04ffab8571158d43b237a590a697 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 19:45:12 +0200
Subject: [PATCH 45/48] CI test

---
 build-executor/main.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 45f5abaa..2305346b 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -92,12 +92,14 @@ CVMFS_TRACEFILE=${MAIN_CVMFSTRACESDIR}/cvmfs-@fqrn@.trace.log
 EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
-  if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi
-  ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+
+
+  ###if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi # 1-inception
+  ###ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
   ###ln -sf /cvmfs $CIENV_CVMFSVOLUME # NO... replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+
   if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
-  cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this creates /cvmfs if it does not exist yet
-  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
+  cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this would create /cvmfs if it did not exist yet
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
     mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
-- 
GitLab


From c8bffa6a6204850d34e3061962fca6906a0f5193 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 19:47:50 +0200
Subject: [PATCH 46/48] CI test

---
 build-executor/main.sh | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index 2305346b..f0db2e65 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -93,7 +93,6 @@ EOF
   mkdir -p /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
 
-
   ###if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi # 1-inception
   ###ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
   ###ln -sf /cvmfs $CIENV_CVMFSVOLUME # NO... replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
@@ -103,18 +102,17 @@ EOF
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
     mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
-    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
-    ls -l $CIENV_CVMFSVOLUME/$repo
+    ###echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
+    ###ls -l $CIENV_CVMFSVOLUME/$repo
     echo "[mount_cvmfs] ls -l /cvmfs/$repo"
     ls -l /cvmfs/$repo
-    echo "[mount_cvmfs] touch $CIENV_CVMFSVOLUME/pippo"
-    touch $CIENV_CVMFSVOLUME/pippo
-    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
-    ls -l $CIENV_CVMFSVOLUME
-    echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
-    ls -l $CIENV_CVMFSVOLUME/$repo
+    ###echo "[mount_cvmfs] touch $CIENV_CVMFSVOLUME/pippo"
+    ###touch $CIENV_CVMFSVOLUME/pippo
+    ###echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
+    ###ls -l $CIENV_CVMFSVOLUME
+    ###echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
+    ###ls -l $CIENV_CVMFSVOLUME/$repo
   done
-  sleep 60
   echo "[mount_cvmfs] finished at $(date)"
   return 0
 }
@@ -192,8 +190,8 @@ function run_docker_wl(){
 
   ls -lRt ${MAIN_CVMFSTRACESDIR}
 
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
+  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
   ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-- 
GitLab


From 6578e29c77bd69063c92f5e4c334f236acba7ed1 Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 20:05:16 +0200
Subject: [PATCH 47/48] CI test

---
 .gitlab-ci.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh
index b12ccdf5..2b7176cb 100755
--- a/.gitlab-ci.sh
+++ b/.gitlab-ci.sh
@@ -90,21 +90,23 @@ if [ "$stage" == "build" ]; then
   mkdir -p $CIENV_CVMFSVOLUME
   if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_CVMFSVOLUME' failed"; exit 1; fi
 
-  # These environment variables were previsouly passed to main.sh within docker via the file .env.file
+  # NEW OPTION 1 (BMK-145) - main.sh as a subprocess
+  # These environment variables were previously passed to main.sh within docker via the file .env.file
   # They are now simply exported to the main.sh subprocess (BMK-145)
   # [NB: it is assumed here that all CI_ variables have been exported by the gitlab-runner CI]
   #export CIENV_JOBDIR
   #export CIENV_CVMFSVOLUME
   #export CIENV_MOUNTCVMFS
   #export CIENV_DOCKERREGISTRY
-  set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file
+  #$(pwd)/build-executor/main.sh | tee -a $CIENV_JOBDIR/main.log
+  #status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
 
-  # TEST HERE
+  # OLD OPTION 2 (BMK-145) - main.sh as an entrypoint of docker run
+  set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR?
   DOCSOCK="/var/run/docker.sock"
   BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest"
   echo "running privileged docker with fixed entrypoint"
   set -x # verbose on
-  ###docker run --rm --privileged --entrypoint=/root/main.sh \
   docker run --rm --privileged --entrypoint=$CI_PROJECT_DIR/build-executor/main.sh \
     --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \
     -v $CIENV_JOBDIR:$CIENV_JOBDIR \
@@ -112,13 +114,11 @@ if [ "$stage" == "build" ]; then
     -v $CIENV_CVMFSVOLUME:/cvmfs:shared \
     -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \
     $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log
-  ###$(pwd)/build-executor/main.sh | tee -a $CIENV_JOBDIR/main.log
   status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee...
+
   set +x # verbose off
   echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)"
   echo -e "[gitlab-ci.sh] status from main.sh: $status\n"
-  # EXIT
-  exit $status
 
   #Create tarball of JOBDIR results
   echo -e "\n\n--------------\nCreating tarball of json and log results for artifacts\n--------------\n"
-- 
GitLab


From c01a5e61d9b29ee34da02e55bd4035a4c6b91aad Mon Sep 17 00:00:00 2001
From: Andrea Valassi <andrea.valassi@cern.ch>
Date: Wed, 17 Jul 2019 20:17:31 +0200
Subject: [PATCH 48/48] CI test

---
 build-executor/main.sh | 56 +++++++++++++++---------------------------
 1 file changed, 20 insertions(+), 36 deletions(-)

diff --git a/build-executor/main.sh b/build-executor/main.sh
index f0db2e65..6dc31767 100755
--- a/build-executor/main.sh
+++ b/build-executor/main.sh
@@ -90,28 +90,25 @@ CVMFS_MOUNT_RW=yes
 CVMFS_HTTP_PROXY="http://squid.cern.ch:8060|http://ca-proxy.cern.ch:3128;DIRECT"
 CVMFS_TRACEFILE=${MAIN_CVMFSTRACESDIR}/cvmfs-@fqrn@.trace.log
 EOF
-  mkdir -p /etc/cvmfs/config.d
+  mkdir -p  /etc/cvmfs/config.d
   echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
 
-  ###if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi # 1-inception
-  ###ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
-  ###ln -sf /cvmfs $CIENV_CVMFSVOLUME # NO... replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"
+  # NEW OPTION 1 (BMK-145) - main.sh as a subprocess
+  #if [ -e /cvmfs ]; then fail "/cvmfs already exists"; fi # 1-inception
+  #ln -sf $CIENV_CVMFSVOLUME /cvmfs # replaces "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared"? IT DOES NOT WORK...
+  #if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
+
+  # OLD OPTION 2 (BMK-145) - main.sh as an entrypoint of docker run
+  [ -d /cvmfs ] || fail "[mount_cvmfs] /cvmfs does not exist" # assume that /cvmfs has been created by now (e.g. by docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared)
 
-  if [ ! -d /cvmfs ]; then fail "/cvmfs does not exist"; fi # assume that /cvmfs has been created by now
   cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" # this would create /cvmfs if it did not exist yet
   for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do
-    mkdir /cvmfs/$repo # assume that /cvmfs has been created by now
+    umount /cvmfs/$repo # OPTION 2 ONLY - NEEDED AT ALL?
+    rm -rf /cvmfs/$repo # OPTION 2 ONLY - NEEDED AT ALL?
+    mkdir /cvmfs/$repo # Assume that /cvmfs has been created by now
     mount -t cvmfs $repo /cvmfs/$repo
-    ###echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
-    ###ls -l $CIENV_CVMFSVOLUME/$repo
     echo "[mount_cvmfs] ls -l /cvmfs/$repo"
     ls -l /cvmfs/$repo
-    ###echo "[mount_cvmfs] touch $CIENV_CVMFSVOLUME/pippo"
-    ###touch $CIENV_CVMFSVOLUME/pippo
-    ###echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME"
-    ###ls -l $CIENV_CVMFSVOLUME
-    ###echo "[mount_cvmfs] ls -l $CIENV_CVMFSVOLUME/$repo"
-    ###ls -l $CIENV_CVMFSVOLUME/$repo
   done
   echo "[mount_cvmfs] finished at $(date)"
   return 0
@@ -188,23 +185,11 @@ function run_docker_wl(){
   ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
   echo -e "\n[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - started"
 
-  ls -lRt ${MAIN_CVMFSTRACESDIR}
+  # NEW OPTION 1 (BMK-145) - main.sh as a subprocess
+  #execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
 
+  # OLD OPTION 2 (BMK-145) - main.sh as an entrypoint of docker run
   execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-
-  ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-
-  ###DOCSOCK="/var/run/docker.sock"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-
-  ###DOCSOCK="/var/run/docker.sock"
-  ###execute_command docker run --privileged ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-  ###execute_command docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $MAIN_HEPWLBUILDDIR:$MAIN_HEPWLBUILDDIR -v $DOCSOCK:$DOCSOCK -w $MAIN_HEPWLBUILDDIR -v /cvmfs:/cvmfs:shared $theimage $HEPWL_BMKOPTS -d || fail "[run_docker_wl] docker run $theimage"
-
-  ls -lRt ${MAIN_CVMFSTRACESDIR}
 
   echo -e "[run_docker_wl] Run WL in docker (to extract cvmfs) via execute_command - completed\n"
   for acvmfs in `ls ${MAIN_CVMFSTRACESDIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
@@ -213,9 +198,6 @@ function run_docker_wl(){
       fail "[run_docker_wl] cvmfs flush $acvmfs" # fix BMK-136 (flush was silently failing, leading to missing cvmfs files)
     fi
   done
-
-  ls -lRt ${MAIN_CVMFSTRACESDIR}
-
   echo "[run_docker_wl] finished at $(date)"
   return 0
 }
@@ -569,7 +551,7 @@ function execute_procedure(){
       run_shrinkwrap || fail "[execute_procedure] $1: run_shrinkwrap"
       copy_cvmfs || fail "[execute_procedure] $1: copy_cvmfs"
       build_standalone_image || fail "[execute_procedure] $1: build_standalone_image"
-      #clean_cvmfs_copy || fail "[execute_procedure] $1: clean_cvmfs_copy"
+      clean_cvmfs_copy || fail "[execute_procedure] $1: clean_cvmfs_copy"
       test_standalone_image_docker || fail "[execute_procedure] $1: test_standalone_image_docker"
       test_standalone_image_singularity || fail "[execute_procedure] $1: test_standalone_image_singularity"
       publish_standalone_image || fail "[execute_procedure] $1: publish_standalone_image"
@@ -604,10 +586,12 @@ echo "scriptDir is ${scriptDir}"
 export MAIN_CVMFSTRACESDIR=$CIENV_JOBDIR/cvmfs-traces/
 export MAIN_CVMFSEXPORTDIR=$CIENV_JOBDIR/cvmfs-export/
 
-# Previously these environment variables were reset to dafult values here
-# To make this cleaner (BMK-145), it is now expected that they must have been set in advance
 # [NB MAIN_NETWORKCONN, which was only used in main.sh, is now a local variable]
 MAIN_NETWORKCONN=0 # this can be overridden via the -n command line option
+
+# NEW OPTION 1 (BMK-145) - main.sh as a subprocess
+# Previously these environment variables were reset to default values here
+# To make this cleaner (BMK-145), it is now expected that they must have been set in advance
 #for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN; do
 #  # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
 #  if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
@@ -644,7 +628,7 @@ while getopts "hds:r:e:m" o; do
   esac
 done
 
-# BACK
+# OLD OPTION 2 (BMK-145) - main.sh as an entrypoint of docker run
 for var in CIENV_HEPWL_SPECFILE CIENV_BUILDEVENT CIENV_DOCKERREGISTRY CIENV_MOUNTCVMFS CIENV_JOBDIR MAIN_NETWORKCONN; do
   # Bash indirection ${!var}: see http://mywiki.wooledge.org/BashFAQ/006#Indirection
   if [ "${!var}" == "" ]; then echo "ERROR! $var is not set"; exit 1; fi
-- 
GitLab