From 3618654d4ce0905471144b9943c88f3a7110b8f3 Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sat, 6 Jul 2019 17:27:36 +0200 Subject: [PATCH 1/8] Rename NETWORK_CONN as MAIN_NETWORKCONN --- build-executor/main.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-executor/main.sh b/build-executor/main.sh index 35b5df22..32f96a4a 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -242,7 +242,7 @@ function test_standalone_image(){ [ -z "$HEPWL_BMKOPTS" ] && HEPWL_BMKOPTS="" strace="" net_conn="--network none" # by default run the test without network connectivity - [[ "$NETWORK_CONN" == "1" ]] && net_conn="" + [[ "$MAIN_NETWORKCONN" == "1" ]] && net_conn="" ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051) execute docker run ${strace} ${net_conn} --rm -v $JOB_DIR/results:/results $IMAGE $HEPWL_BMKOPTS || fail "[test_standalone_image] docker run $IMAGE" echo "[test_standalone_image] finished at $(date)" @@ -421,7 +421,7 @@ export CVMFS_EXPORT_DIR=$JOB_DIR/cvmfs-export/ [ -z $DOCKER_REGISTRY ] && DOCKER_REGISTRY="" [ -z $MOUNT_CVMFS ] && MOUNT_CVMFS=0 [ -z $JOB_DIR ] && JOB_DIR=/tmp -[ -z $NETWORK_CONN ] && NETWORK_CONN=0 +[ -z $MAIN_NETWORKCONN ] && MAIN_NETWORKCONN=0 # Parameters to be passed by command line # 1: SPEC file path @@ -441,7 +441,7 @@ while getopts "hds:r:e:m" o; do MOUNT_CVMFS=1 ;; n) - NETWORK_CONN=1 + MAIN_NETWORKCONN=1 ;; d) set -x # Enable debug printouts -- GitLab From bec17f2d0c64018980daacc61c14d08494739262 Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sat, 6 Jul 2019 17:31:45 +0200 Subject: [PATCH 2/8] Rename MOUNT_CVMFS as CIENV_MOUNTCVMFS --- .gitlab-ci.sh | 14 +++++++------- build-executor/main.sh | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index 360d8385..d5dceec8 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -60,12 +60,12 @@ if [ "$stage" == "build" ]; then if [ "$BUILD_EVENT" == "" ]; then BUILD_EVENT=all; fi echo "BUILD_EVENT=$BUILD_EVENT" - # Set MOUNT_CVMFS=1 in the gitlab CI as it is not yet set - # (conversely, MOUNT_CVMFS=0 should have been set in run_build.sh) - if [ "$MOUNT_CVMFS" == "" ]; then - MOUNT_CVMFS=1 # this was previously in runner's toml + # Set CIENV_MOUNTCVMFS=1 in the gitlab CI as it is not yet set + # (conversely, CIENV_MOUNTCVMFS=0 should have been set in run_build.sh) + if [ "$CIENV_MOUNTCVMFS" == "" ]; then + CIENV_MOUNTCVMFS=1 # this was previously in runner's toml fi - echo "MOUNT_CVMFS=$MOUNT_CVMFS" + echo "CIENV_MOUNTCVMFS=$CIENV_MOUNTCVMFS" # Set other environment variables which were previously in the runner's toml # Skip image publishing and use a different naming convention in noCI mode @@ -85,7 +85,7 @@ if [ "$stage" == "build" ]; then if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $JOB_DIR' failed"; exit 1; fi mkdir -p $CVMFS_VOLUME if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CVMFS_VOLUME' failed"; exit 1; fi - set | egrep -e "^(CI_|JOB_DIR|CVMFS_VOLUME|MOUNT_CVMFS|DOCKER_REGISTRY)" | sort > $JOB_DIR/.env.file # todo: only need CVMFS_VOLUME and JOB_DIR? + set | egrep -e "^(CI_|JOB_DIR|CVMFS_VOLUME|CIENV_MOUNTCVMFS|DOCKER_REGISTRY)" | sort > $JOB_DIR/.env.file # todo: only need CVMFS_VOLUME and JOB_DIR? DOCSOCK="/var/run/docker.sock" BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest" echo "running privileged docker with fixed entrypoint" @@ -112,7 +112,7 @@ if [ "$stage" == "build" ]; then # Clean up: remove directory CVMFS_VOLUME # Also unmount cvmfs and delete mount points below CVMFS_VOLUME if necessary (but main.sh # in docker run should have already done this and CVMFS_VOLUME should be empty by now) - if [ "$MOUNT_CVMFS" != "0" ]; then + if [ "$CIENV_MOUNTCVMFS" != "0" ]; then echo -e "\n\n--------------\n Clean up CVMFS Volumes\n--------------\n" if [ ! -d ${CVMFS_VOLUME} ]; then echo "WARNING! Directory not found: $CVMFS_VOLUME" # e.g. symlink in /mnt, but /mnt not shared in the toml diff --git a/build-executor/main.sh b/build-executor/main.sh index 32f96a4a..214b0120 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -419,7 +419,7 @@ export CVMFS_EXPORT_DIR=$JOB_DIR/cvmfs-export/ [ -z $HEP_WL_SPEC_FILE ] && HEP_WL_SPEC_FILE='' [ -z $BUILD_EVENT ] && BUILD_EVENT="all" [ -z $DOCKER_REGISTRY ] && DOCKER_REGISTRY="" -[ -z $MOUNT_CVMFS ] && MOUNT_CVMFS=0 +[ -z $CIENV_MOUNTCVMFS ] && CIENV_MOUNTCVMFS=0 [ -z $JOB_DIR ] && JOB_DIR=/tmp [ -z $MAIN_NETWORKCONN ] && MAIN_NETWORKCONN=0 @@ -438,7 +438,7 @@ while getopts "hds:r:e:m" o; do [ "$OPTARG" != "" ] && export DOCKER_REGISTRY="$OPTARG" ;; m) - MOUNT_CVMFS=1 + CIENV_MOUNTCVMFS=1 ;; n) MAIN_NETWORKCONN=1 @@ -457,7 +457,7 @@ echo -e "\n-------------\n[main] Input parameters" echo HEP_WL_SPEC_FILE $HEP_WL_SPEC_FILE echo BUILD_EVENT $BUILD_EVENT echo DOCKER_REGISTRY $DOCKER_REGISTRY -echo MOUNT_CVMFS $MOUNT_CVMFS +echo CIENV_MOUNTCVMFS $CIENV_MOUNTCVMFS echo JOB_DIR "$JOB_DIR" echo CVMFS_VOLUME "$CVMFS_VOLUME" echo "-------------" @@ -486,13 +486,13 @@ echo "-----------" # It may optionally set also HEPWL_ANNOUNCE and HEPWL_EXTEND_<repo>_SPEC source "$HEP_WL_SPEC_FILE" -if [[ "$MOUNT_CVMFS" -gt 0 ]]; then +if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then mount_cvmfs || fail '[main.sh] mount_cvmfs' fi execute_procedure "$BUILD_EVENT" -if [[ "$MOUNT_CVMFS" -gt 0 ]]; then +if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then unmount_cvmfs || fail '[main.sh] unmount_cvmfs' fi -- GitLab From 295c9092a6d9e29379440a5952b9ad3de01902fa Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sat, 6 Jul 2019 17:33:43 +0200 Subject: [PATCH 3/8] Rename JOB_DIR as CIENV_JOBDIR --- .gitlab-ci.sh | 22 +++++++++++----------- build-executor/main.sh | 38 +++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index d5dceec8..a29dcd1a 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -70,33 +70,33 @@ if [ "$stage" == "build" ]; then # Set other environment variables which were previously in the runner's toml # Skip image publishing and use a different naming convention in noCI mode if [ "$CI_JOB_NAME" != "noCI" ]; then - export JOB_DIR=/scratch/logs/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml + export CIENV_JOBDIR=/scratch/logs/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml export CVMFS_VOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml export DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} # previously in runner's toml else - export JOB_DIR=/tmp/${USER}/logs/noCI-${CI_JOB_ID} + export CIENV_JOBDIR=/tmp/${USER}/logs/noCI-${CI_JOB_ID} export CVMFS_VOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID} export DOCKER_REGISTRY="" fi # Prepare the environment for the docker-in-docker main.sh and execute it # NB: "docker run -v $CVMFS_VOLUME:/cvmfs:shared" would mkdir $CVMFS_VOLUME if missing, but with owner root! - mkdir -p $JOB_DIR - if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $JOB_DIR' failed"; exit 1; fi + mkdir -p $CIENV_JOBDIR + if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_JOBDIR' failed"; exit 1; fi mkdir -p $CVMFS_VOLUME if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CVMFS_VOLUME' failed"; exit 1; fi - set | egrep -e "^(CI_|JOB_DIR|CVMFS_VOLUME|CIENV_MOUNTCVMFS|DOCKER_REGISTRY)" | sort > $JOB_DIR/.env.file # todo: only need CVMFS_VOLUME and JOB_DIR? + set | egrep -e "^(CI_|CIENV_JOBDIR|CVMFS_VOLUME|CIENV_MOUNTCVMFS|DOCKER_REGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CVMFS_VOLUME and CIENV_JOBDIR? DOCSOCK="/var/run/docker.sock" BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest" echo "running privileged docker with fixed entrypoint" set -x # verbose on docker run --rm --privileged --entrypoint=/root/main.sh \ - --env-file $JOB_DIR/.env.file --name main_$CI_JOB_ID \ - -v $JOB_DIR:$JOB_DIR \ + --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \ + -v $CIENV_JOBDIR:$CIENV_JOBDIR \ -v $DOCSOCK:$DOCSOCK \ -v $CVMFS_VOLUME:/cvmfs:shared \ -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \ - $BUILDIMG -m -s $HEP_WL_SPEC_FILE -e $BUILD_EVENT | tee -a $JOB_DIR/main.log + $BUILDIMG -m -s $HEP_WL_SPEC_FILE -e $BUILD_EVENT | tee -a $CIENV_JOBDIR/main.log status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee... set +x # verbose off echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)" @@ -104,10 +104,10 @@ if [ "$stage" == "build" ]; then #Create tarball of JOBDIR results echo -e "\n\n--------------\nCreating tarball of json and log results for artifacts\n--------------\n" - if [ -e $JOB_DIR/results ];then - find $JOB_DIR/results -name "*.json" -or -name "*.log" | tar -cvzf $CI_PROJECT_DIR/results.tar.gz -T - + if [ -e $CIENV_JOBDIR/results ];then + find $CIENV_JOBDIR/results -name "*.json" -or -name "*.log" | tar -cvzf $CI_PROJECT_DIR/results.tar.gz -T - else - echo -e "No such directory $JOB_DIR/results . Artifacts will be empty\n" + echo -e "No such directory $CIENV_JOBDIR/results . Artifacts will be empty\n" fi # Clean up: remove directory CVMFS_VOLUME # Also unmount cvmfs and delete mount points below CVMFS_VOLUME if necessary (but main.sh diff --git a/build-executor/main.sh b/build-executor/main.sh index 214b0120..a8973f93 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -96,10 +96,10 @@ function build_docker_image(){ # Input environment variable ${HEPWL_BMKOPTS}: options for the HEP workload script # Input environment variable ${HEP_WL_SOURCE_DIR}: directory containing the Dockerfile.template # Input environment variable ${HEP_WL_SPEC_FILE}: spec file for the image to be built -# Input environment variable ${JOB_DIR}/results: host directory where results should be stored +# Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored # Input environment variable ${CVMFS_VOLUME}: host directory where cvmfs is mounted? # Input environment variable ${CVMFS_TRACES_DIR}: directory containing the cvmfs trace files -# Output: status code for the success of the test and results in ${JOB_DIR}/results +# Output: status code for the success of the test and results in ${CIENV_JOBDIR}/results # Output: cvmfs trace stored in ${CVMFS_TRACES_DIR} function run_docker_wl(){ echo "[run_docker_wl] starting at $(date)" @@ -112,7 +112,7 @@ function run_docker_wl(){ echo -ne "\n[run_docker_wl] Running: " strace="" ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051) - execute docker run ${strace} --rm -v $JOB_DIR/results:/results -v $HEP_WL_SOURCE_DIR:$HEP_WL_SOURCE_DIR -w $HEP_WL_SOURCE_DIR -v $CVMFS_VOLUME:/cvmfs:shared $IMAGE $HEPWL_BMKOPTS || fail "[run_docker_wl] docker run $IMAGE" + execute docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $HEP_WL_SOURCE_DIR:$HEP_WL_SOURCE_DIR -w $HEP_WL_SOURCE_DIR -v $CVMFS_VOLUME:/cvmfs:shared $IMAGE $HEPWL_BMKOPTS || fail "[run_docker_wl] docker run $IMAGE" date for acvmfs in `ls ${CVMFS_TRACES_DIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do echo "[run_docker_wl] Flush cvmfs trace for $acvmfs" @@ -126,7 +126,7 @@ function run_docker_wl(){ # Create a spec from the cvmfs trace file, then run shrinkwrap to create a cvmfs export # Input environment variable ${HEP_WL_SOURCE_DIR}: directory containing the Dockerfile.template # Input environment variable ${HEPWL_EXTEND_<repo>_SPEC}: optional extra cvmfs spec file -# Input environment variable ${JOB_DIR}: host directory where the cvmfs config should be stored +# Input environment variable ${CIENV_JOBDIR}: host directory where the cvmfs config should be stored # Input environment variable ${CVMFS_TRACES_DIR}: directory containing the cvmfs trace files # Input environment variable ${CVMFS_EXPORT_DIR}: directory where the cvmfs export should be stored # Output: cvmfs export in ${CVMFS_EXPORT_DIR}/cvmfs @@ -141,7 +141,7 @@ function run_shrinkwrap(){ rm -rf ${CVMFS_EXPORT_DIR} fi mkdir -p ${CVMFS_EXPORT_DIR}/cvmfs - CVMFS_SHRINK_CONF=$JOB_DIR/generic_config.cern.ch.config # no need to export this + CVMFS_SHRINK_CONF=$CIENV_JOBDIR/generic_config.cern.ch.config # no need to export this cat > ${CVMFS_SHRINK_CONF} <<EOF CVMFS_CACHE_BASE=/var/lib/cvmfs/shrinkwrap CVMFS_HTTP_PROXY=DIRECT @@ -209,9 +209,9 @@ function clean_cvmfs_copy(){ # Input environment variable ${HEPWL_DOCKERIMAGETAG}: image tag for the standalone HEP workload # Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed # Input environment variable ${HEPWL_BMKOPTS}: options for the HEP workload script -# Input environment variable ${JOB_DIR}/results: host directory where results should be stored +# Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored # Output: standalone WL image stored in the local Docker image registry -# Output: status code for the success of the test and results in ${JOB_DIR}/results +# Output: status code for the success of the test and results in ${CIENV_JOBDIR}/results # Output: standalone WL image stored in the remote Docker image registry function build_standalone_image(){ echo "[build_standalone_image] starting at $(date)" @@ -232,8 +232,8 @@ function build_standalone_image(){ # Input environment variable ${HEPWL_DOCKERIMAGETAG}: image tag for the standalone HEP workload # Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed # Input environment variable ${HEPWL_BMKOPTS}: options for the HEP workload script -# Input environment variable ${JOB_DIR}/results: host directory where results should be stored -# Output: status code for the success of the test and results in ${JOB_DIR}/results +# Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored +# Output: status code for the success of the test and results in ${CIENV_JOBDIR}/results function test_standalone_image(){ echo "[test_standalone_image] starting at $(date)" IMAGE="${HEPWL_DOCKERIMAGE}":"${HEPWL_DOCKERIMAGETAG}" @@ -244,7 +244,7 @@ function test_standalone_image(){ net_conn="--network none" # by default run the test without network connectivity [[ "$MAIN_NETWORKCONN" == "1" ]] && net_conn="" ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051) - execute docker run ${strace} ${net_conn} --rm -v $JOB_DIR/results:/results $IMAGE $HEPWL_BMKOPTS || fail "[test_standalone_image] docker run $IMAGE" + execute docker run ${strace} ${net_conn} --rm -v $CIENV_JOBDIR/results:/results $IMAGE $HEPWL_BMKOPTS || fail "[test_standalone_image] docker run $IMAGE" echo "[test_standalone_image] finished at $(date)" return 0 } @@ -402,7 +402,7 @@ echo -e "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n" echo "Current directory is $(pwd)" -echo "JOB_DIR is ${JOB_DIR}" # within docker this may come from .env.file +echo "CIENV_JOBDIR is ${CIENV_JOBDIR}" # within docker this may come from .env.file echo "CI_PROJECT_DIR is ${CI_PROJECT_DIR}" # within docker this may come from .env.file @@ -413,14 +413,14 @@ echo "SCRIPT_DIR is ${SCRIPT_DIR}" . $SCRIPT_DIR/generate_Dockerfile.sh # this simply loads function generate_Dockerfile() # These are fixed params, to be moved in an etc file -export CVMFS_TRACES_DIR=$JOB_DIR/cvmfs-traces/ -export CVMFS_EXPORT_DIR=$JOB_DIR/cvmfs-export/ +export CVMFS_TRACES_DIR=$CIENV_JOBDIR/cvmfs-traces/ +export CVMFS_EXPORT_DIR=$CIENV_JOBDIR/cvmfs-export/ [ -z $HEP_WL_SPEC_FILE ] && HEP_WL_SPEC_FILE='' [ -z $BUILD_EVENT ] && BUILD_EVENT="all" [ -z $DOCKER_REGISTRY ] && DOCKER_REGISTRY="" [ -z $CIENV_MOUNTCVMFS ] && CIENV_MOUNTCVMFS=0 -[ -z $JOB_DIR ] && JOB_DIR=/tmp +[ -z $CIENV_JOBDIR ] && CIENV_JOBDIR=/tmp [ -z $MAIN_NETWORKCONN ] && MAIN_NETWORKCONN=0 # Parameters to be passed by command line @@ -458,7 +458,7 @@ echo HEP_WL_SPEC_FILE $HEP_WL_SPEC_FILE echo BUILD_EVENT $BUILD_EVENT echo DOCKER_REGISTRY $DOCKER_REGISTRY echo CIENV_MOUNTCVMFS $CIENV_MOUNTCVMFS -echo JOB_DIR "$JOB_DIR" +echo CIENV_JOBDIR "$CIENV_JOBDIR" echo CVMFS_VOLUME "$CVMFS_VOLUME" echo "-------------" @@ -470,11 +470,11 @@ env | grep "CI_" | sort HEP_WL_SOURCE_DIR=$(dirname "$HEP_WL_SPEC_FILE") # Original HEP source dir -mkdir -p $JOB_DIR/build-wl -cp -r $HEP_WL_SOURCE_DIR/* $JOB_DIR/build-wl -export HEP_WL_SOURCE_DIR=$JOB_DIR/build-wl +mkdir -p $CIENV_JOBDIR/build-wl +cp -r $HEP_WL_SOURCE_DIR/* $CIENV_JOBDIR/build-wl +export HEP_WL_SOURCE_DIR=$CIENV_JOBDIR/build-wl -HEP_WL_SPEC_FILE=$JOB_DIR/build-wl/${HEP_WL_SPEC_FILE##*/} +HEP_WL_SPEC_FILE=$CIENV_JOBDIR/build-wl/${HEP_WL_SPEC_FILE##*/} echo "----------- $HEP_WL_SPEC_FILE" cat "$HEP_WL_SPEC_FILE" -- GitLab From b60614faa2bf74d8e79687039040f18f964f8acc Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sat, 6 Jul 2019 17:35:25 +0200 Subject: [PATCH 4/8] Rename CVMFS_VOLUME as CIENV_CVMFSVOLUME --- .gitlab-ci.sh | 44 +++++++++++++++++++++--------------------- build-executor/main.sh | 12 ++++++------ 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index a29dcd1a..20068710 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -71,21 +71,21 @@ if [ "$stage" == "build" ]; then # Skip image publishing and use a different naming convention in noCI mode if [ "$CI_JOB_NAME" != "noCI" ]; then export CIENV_JOBDIR=/scratch/logs/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml - export CVMFS_VOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml + export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml export DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} # previously in runner's toml else export CIENV_JOBDIR=/tmp/${USER}/logs/noCI-${CI_JOB_ID} - export CVMFS_VOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID} + export CIENV_CVMFSVOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID} export DOCKER_REGISTRY="" fi # Prepare the environment for the docker-in-docker main.sh and execute it - # NB: "docker run -v $CVMFS_VOLUME:/cvmfs:shared" would mkdir $CVMFS_VOLUME if missing, but with owner root! + # NB: "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared" would mkdir $CIENV_CVMFSVOLUME if missing, but with owner root! mkdir -p $CIENV_JOBDIR if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_JOBDIR' failed"; exit 1; fi - mkdir -p $CVMFS_VOLUME - if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CVMFS_VOLUME' failed"; exit 1; fi - set | egrep -e "^(CI_|CIENV_JOBDIR|CVMFS_VOLUME|CIENV_MOUNTCVMFS|DOCKER_REGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CVMFS_VOLUME and CIENV_JOBDIR? + mkdir -p $CIENV_CVMFSVOLUME + if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_CVMFSVOLUME' failed"; exit 1; fi + set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|DOCKER_REGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR? DOCSOCK="/var/run/docker.sock" BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest" echo "running privileged docker with fixed entrypoint" @@ -94,7 +94,7 @@ if [ "$stage" == "build" ]; then --env-file $CIENV_JOBDIR/.env.file --name main_$CI_JOB_ID \ -v $CIENV_JOBDIR:$CIENV_JOBDIR \ -v $DOCSOCK:$DOCSOCK \ - -v $CVMFS_VOLUME:/cvmfs:shared \ + -v $CIENV_CVMFSVOLUME:/cvmfs:shared \ -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \ $BUILDIMG -m -s $HEP_WL_SPEC_FILE -e $BUILD_EVENT | tee -a $CIENV_JOBDIR/main.log status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee... @@ -109,27 +109,27 @@ if [ "$stage" == "build" ]; then else echo -e "No such directory $CIENV_JOBDIR/results . Artifacts will be empty\n" fi - # Clean up: remove directory CVMFS_VOLUME - # Also unmount cvmfs and delete mount points below CVMFS_VOLUME if necessary (but main.sh - # in docker run should have already done this and CVMFS_VOLUME should be empty by now) + # Clean up: remove directory CIENV_CVMFSVOLUME + # Also unmount cvmfs and delete mount points below CIENV_CVMFSVOLUME if necessary (but main.sh + # in docker run should have already done this and CIENV_CVMFSVOLUME should be empty by now) if [ "$CIENV_MOUNTCVMFS" != "0" ]; then echo -e "\n\n--------------\n Clean up CVMFS Volumes\n--------------\n" - if [ ! -d ${CVMFS_VOLUME} ]; then - echo "WARNING! Directory not found: $CVMFS_VOLUME" # e.g. symlink in /mnt, but /mnt not shared in the toml + if [ ! -d ${CIENV_CVMFSVOLUME} ]; then + echo "WARNING! Directory not found: $CIENV_CVMFSVOLUME" # e.g. symlink in /mnt, but /mnt not shared in the toml else - echo "CVMFS_VOLUME is $CVMFS_VOLUME" - if [ `ls -A $CVMFS_VOLUME | wc -l` -gt 0 ]; then - echo "CVMFS_VOLUME directory contains:" - ls -Al $CVMFS_VOLUME - for mount in $(ls $CVMFS_VOLUME); do # this directory should be empty by now - echo "WARNING! $CVMFS_VOLUME/$mount exists: try to umount and remove it" - umount -fR $CVMFS_VOLUME/$mount || echo "WARNING! Could not umount $CVMFS_VOLUME/$mount" - rm -rf -fR $CVMFS_VOLUME/$mount || echo "WARNING! Could not remove $CVMFS_VOLUME/$mount" + echo "CIENV_CVMFSVOLUME is $CIENV_CVMFSVOLUME" + if [ `ls -A $CIENV_CVMFSVOLUME | wc -l` -gt 0 ]; then + echo "CIENV_CVMFSVOLUME directory contains:" + ls -Al $CIENV_CVMFSVOLUME + for mount in $(ls $CIENV_CVMFSVOLUME); do # this directory should be empty by now + echo "WARNING! $CIENV_CVMFSVOLUME/$mount exists: try to umount and remove it" + umount -fR $CIENV_CVMFSVOLUME/$mount || echo "WARNING! Could not umount $CIENV_CVMFSVOLUME/$mount" + rm -rf -fR $CIENV_CVMFSVOLUME/$mount || echo "WARNING! Could not remove $CIENV_CVMFSVOLUME/$mount" done else - echo "CVMFS_VOLUME directory is empty" + echo "CIENV_CVMFSVOLUME directory is empty" fi - rm -rf $CVMFS_VOLUME || echo "WARNING! Could not remove $CVMFS_VOLUME" # Remove /scratch/cvmfs_hep/CI-JOB-xxx/ top-directory + rm -rf $CIENV_CVMFSVOLUME || echo "WARNING! Could not remove $CIENV_CVMFSVOLUME" # Remove /scratch/cvmfs_hep/CI-JOB-xxx/ top-directory fi fi diff --git a/build-executor/main.sh b/build-executor/main.sh index a8973f93..12650d89 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -43,7 +43,7 @@ EOF echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local #[[ `ls /cvmfs | wc -l` -gt 0 ]] && fail "/cvmfs already contains files: remove them" cvmfs_config setup nostart || fail "[mount_cvmfs] problem with cvmfs_config setup nostart" - [ -d /cvmfs ] || fail "[mount_cvmfs] /cvmfs does not exist" # Assume that /cvmfs has been created by now (e.g. by docker run -v $CVMFS_VOLUME:/cvmfs:shared) + [ -d /cvmfs ] || fail "[mount_cvmfs] /cvmfs does not exist" # Assume that /cvmfs has been created by now (e.g. by docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared) for repo in `echo ${HEPWL_CVMFSREPOS}| sed -e 's@,@ @g'`; do umount /cvmfs/$repo rm -rf /cvmfs/$repo @@ -97,7 +97,7 @@ function build_docker_image(){ # Input environment variable ${HEP_WL_SOURCE_DIR}: directory containing the Dockerfile.template # Input environment variable ${HEP_WL_SPEC_FILE}: spec file for the image to be built # Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored -# Input environment variable ${CVMFS_VOLUME}: host directory where cvmfs is mounted? +# Input environment variable ${CIENV_CVMFSVOLUME}: host directory where cvmfs is mounted? # Input environment variable ${CVMFS_TRACES_DIR}: directory containing the cvmfs trace files # Output: status code for the success of the test and results in ${CIENV_JOBDIR}/results # Output: cvmfs trace stored in ${CVMFS_TRACES_DIR} @@ -112,7 +112,7 @@ function run_docker_wl(){ echo -ne "\n[run_docker_wl] Running: " strace="" ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051) - execute docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $HEP_WL_SOURCE_DIR:$HEP_WL_SOURCE_DIR -w $HEP_WL_SOURCE_DIR -v $CVMFS_VOLUME:/cvmfs:shared $IMAGE $HEPWL_BMKOPTS || fail "[run_docker_wl] docker run $IMAGE" + execute docker run ${strace} --rm -v $CIENV_JOBDIR/results:/results -v $HEP_WL_SOURCE_DIR:$HEP_WL_SOURCE_DIR -w $HEP_WL_SOURCE_DIR -v $CIENV_CVMFSVOLUME:/cvmfs:shared $IMAGE $HEPWL_BMKOPTS || fail "[run_docker_wl] docker run $IMAGE" date for acvmfs in `ls ${CVMFS_TRACES_DIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do echo "[run_docker_wl] Flush cvmfs trace for $acvmfs" @@ -364,9 +364,9 @@ function execute_procedure(){ ;; sleep) echo -e "\nSleep and stay alive just to expose cvmfs" - echo "cvmfs mount point $CVMFS_VOLUME" + echo "cvmfs mount point $CIENV_CVMFSVOLUME" echo "You can mount the available cvmfs as follow:" - echo "docker run -v $CVMFS_VOLUME:/cvmfs:shared ..." + echo "docker run -v $CIENV_CVMFSVOLUME:/cvmfs:shared ..." sleep infinity ;; build) @@ -459,7 +459,7 @@ echo BUILD_EVENT $BUILD_EVENT echo DOCKER_REGISTRY $DOCKER_REGISTRY echo CIENV_MOUNTCVMFS $CIENV_MOUNTCVMFS echo CIENV_JOBDIR "$CIENV_JOBDIR" -echo CVMFS_VOLUME "$CVMFS_VOLUME" +echo CIENV_CVMFSVOLUME "$CIENV_CVMFSVOLUME" echo "-------------" env | grep "CI_" | sort -- GitLab From cf4e4d07289434b9ff8175f3f0f6915d5e532732 Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sat, 6 Jul 2019 17:37:00 +0200 Subject: [PATCH 5/8] Rename DOCKER_REGISTRY as CIENV_DOCKERREGISTRY --- .gitlab-ci.sh | 6 +++--- build-executor/main.sh | 30 +++++++++++++++--------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index 20068710..38a89fef 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -72,11 +72,11 @@ if [ "$stage" == "build" ]; then if [ "$CI_JOB_NAME" != "noCI" ]; then export CIENV_JOBDIR=/scratch/logs/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}_${CI_JOB_NAME} # previously in runner's toml - export DOCKER_REGISTRY=${CI_REGISTRY_IMAGE} # previously in runner's toml + export CIENV_DOCKERREGISTRY=${CI_REGISTRY_IMAGE} # previously in runner's toml else export CIENV_JOBDIR=/tmp/${USER}/logs/noCI-${CI_JOB_ID} export CIENV_CVMFSVOLUME=/tmp/${USER}/cvmfs_hep/noCI-${CI_JOB_ID} - export DOCKER_REGISTRY="" + export CIENV_DOCKERREGISTRY="" fi # Prepare the environment for the docker-in-docker main.sh and execute it @@ -85,7 +85,7 @@ if [ "$stage" == "build" ]; then if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_JOBDIR' failed"; exit 1; fi mkdir -p $CIENV_CVMFSVOLUME if [ "$?" != "0" ]; then echo "ERROR! 'mkdir -p $CIENV_CVMFSVOLUME' failed"; exit 1; fi - set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|DOCKER_REGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR? + set | egrep -e "^(CI_|CIENV_JOBDIR|CIENV_CVMFSVOLUME|CIENV_MOUNTCVMFS|CIENV_DOCKERREGISTRY)" | sort > $CIENV_JOBDIR/.env.file # todo: only need CIENV_CVMFSVOLUME and CIENV_JOBDIR? DOCSOCK="/var/run/docker.sock" BUILDIMG="$CI_REGISTRY_IMAGE/hep-workload-builder:latest" echo "running privileged docker with fixed entrypoint" diff --git a/build-executor/main.sh b/build-executor/main.sh index 12650d89..c7352ccf 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -207,7 +207,7 @@ function clean_cvmfs_copy(){ # Build, test and publish the standalone HEP workload image with a local /cvmfs # Input environment variable ${HEPWL_DOCKERIMAGE}: image name for the HEP workload # Input environment variable ${HEPWL_DOCKERIMAGETAG}: image tag for the standalone HEP workload -# Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed +# Input environment variable ${CIENV_DOCKERREGISTRY}: registry where the image should be pushed # Input environment variable ${HEPWL_BMKOPTS}: options for the HEP workload script # Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored # Output: standalone WL image stored in the local Docker image registry @@ -216,7 +216,7 @@ function clean_cvmfs_copy(){ function build_standalone_image(){ echo "[build_standalone_image] starting at $(date)" IMAGE="${HEPWL_DOCKERIMAGE}":"${HEPWL_DOCKERIMAGETAG}" - [[ "${DOCKER_REGISTRY}" != "" ]] && IMAGE="${DOCKER_REGISTRY}/${IMAGE}" + [[ "${CIENV_DOCKERREGISTRY}" != "" ]] && IMAGE="${CIENV_DOCKERREGISTRY}/${IMAGE}" echo "[build_standalone_image] IMAGE: $IMAGE" build_docker_image "$IMAGE" || fail "[build_standalone_image] build_docker_image $IMAGE" clean_cvmfs_copy || fail "[build_standalone_image] clean_cvmfs_copy" @@ -230,14 +230,14 @@ function build_standalone_image(){ # Test the standalone HEP workload image (in the local registry) with a local /cvmfs and test it # Input environment variable ${HEPWL_DOCKERIMAGE}: image name for the HEP workload # Input environment variable ${HEPWL_DOCKERIMAGETAG}: image tag for the standalone HEP workload -# Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed +# Input environment variable ${CIENV_DOCKERREGISTRY}: registry where the image should be pushed # Input environment variable ${HEPWL_BMKOPTS}: options for the HEP workload script # Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored # Output: status code for the success of the test and results in ${CIENV_JOBDIR}/results function test_standalone_image(){ echo "[test_standalone_image] starting at $(date)" IMAGE="${HEPWL_DOCKERIMAGE}":"${HEPWL_DOCKERIMAGETAG}" - [[ "${DOCKER_REGISTRY}" != "" ]] && IMAGE="${DOCKER_REGISTRY}/${IMAGE}" + [[ "${CIENV_DOCKERREGISTRY}" != "" ]] && IMAGE="${CIENV_DOCKERREGISTRY}/${IMAGE}" echo "[test_standalone_image] IMAGE: $IMAGE" [ -z "$HEPWL_BMKOPTS" ] && HEPWL_BMKOPTS="" strace="" @@ -253,21 +253,21 @@ function test_standalone_image(){ # Publish (and announce) the standalone HEP workload image with a local /cvmfs # Input environment variable ${HEPWL_DOCKERIMAGE}: image name for the HEP workload # Input environment variable ${HEPWL_DOCKERIMAGETAG}: image tag for the standalone HEP workload -# Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed +# Input environment variable ${CIENV_DOCKERREGISTRY}: registry where the image should be pushed # Input environment variable ${CI_BUILD_TOKEN}: authentication token for the gitlab registry # Output: standalone WL image stored in the remote Docker image registry function publish_standalone_image(){ echo "[publish_standalone_image] starting at $(date)" - if [ "${DOCKER_REGISTRY}" == "" ]; then - echo "[publish_standalone_image] WARNING: empty DOCKER_REGISTRY, nothing to do" + if [ "${CIENV_DOCKERREGISTRY}" == "" ]; then + echo "[publish_standalone_image] WARNING: empty CIENV_DOCKERREGISTRY, nothing to do" else - IMAGE="${DOCKER_REGISTRY}/${HEPWL_DOCKERIMAGE}:${HEPWL_DOCKERIMAGETAG}" + IMAGE="${CIENV_DOCKERREGISTRY}/${HEPWL_DOCKERIMAGE}:${HEPWL_DOCKERIMAGETAG}" echo "[publish_standalone_image] IMAGE: $IMAGE" echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch || fail "[publish_standalone_image] docker login" echo "[publish_standalone_image] docker push ${IMAGE}" docker push "${IMAGE}" || fail "[publish_standalone_image] docker push ${IMAGE}" if [[ ${HEPWL_DOCKERIMAGETAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then # flag as latest only images that respect the format v[0-9]*\.[0-9]* - IMAGE_LATEST=${DOCKER_REGISTRY}/${HEPWL_DOCKERIMAGE}:latest + IMAGE_LATEST=${CIENV_DOCKERREGISTRY}/${HEPWL_DOCKERIMAGE}:latest docker tag "${IMAGE}" "${IMAGE_LATEST}" || fail "[publish_standalone_image] docker tag ${IMAGE} ${IMAGE_LATEST}" docker push "${IMAGE_LATEST}" || fail "[publish_standalone_image] docker push ${IMAGE_LATEST}" fi @@ -304,9 +304,9 @@ function announce(){ # tag HEPWL_DOCKERIMAGETAG is greater than the "latest" tag in the registry, otherwise fail # Input environment variable ${HEPWL_DOCKERIMAGE}: image name for the HEP workload # Input environment variable ${HEPWL_DOCKERIMAGETAG}: image tag for the standalone HEP workload -# Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed +# Input environment variable ${CIENV_DOCKERREGISTRY}: registry where the image should be pushed function check_tag_version(){ - IMAGE=${DOCKER_REGISTRY}/${HEPWL_DOCKERIMAGE} + IMAGE=${CIENV_DOCKERREGISTRY}/${HEPWL_DOCKERIMAGE} REFTAG=latest if [[ ! ${HEPWL_DOCKERIMAGETAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then echo "[check_tag_version] image tag $HEPWL_DOCKERIMAGETAG does NOT respect the format v[0-9]*\.[0-9]* " @@ -418,14 +418,14 @@ export CVMFS_EXPORT_DIR=$CIENV_JOBDIR/cvmfs-export/ [ -z $HEP_WL_SPEC_FILE ] && HEP_WL_SPEC_FILE='' [ -z $BUILD_EVENT ] && BUILD_EVENT="all" -[ -z $DOCKER_REGISTRY ] && DOCKER_REGISTRY="" +[ -z $CIENV_DOCKERREGISTRY ] && CIENV_DOCKERREGISTRY="" [ -z $CIENV_MOUNTCVMFS ] && CIENV_MOUNTCVMFS=0 [ -z $CIENV_JOBDIR ] && CIENV_JOBDIR=/tmp [ -z $MAIN_NETWORKCONN ] && MAIN_NETWORKCONN=0 # Parameters to be passed by command line # 1: SPEC file path -# 2: optional export DOCKER_REGISTRY gitlab-registry.cern.ch/giordano/hep-workloads +# 2: optional export CIENV_DOCKERREGISTRY gitlab-registry.cern.ch/giordano/hep-workloads while getopts "hds:r:e:m" o; do case ${o} in s) @@ -435,7 +435,7 @@ while getopts "hds:r:e:m" o; do [ "$OPTARG" != "" ] && export BUILD_EVENT="$OPTARG" ;; r) - [ "$OPTARG" != "" ] && export DOCKER_REGISTRY="$OPTARG" + [ "$OPTARG" != "" ] && export CIENV_DOCKERREGISTRY="$OPTARG" ;; m) CIENV_MOUNTCVMFS=1 @@ -456,7 +456,7 @@ done echo -e "\n-------------\n[main] Input parameters" echo HEP_WL_SPEC_FILE $HEP_WL_SPEC_FILE echo BUILD_EVENT $BUILD_EVENT -echo DOCKER_REGISTRY $DOCKER_REGISTRY +echo CIENV_DOCKERREGISTRY $CIENV_DOCKERREGISTRY echo CIENV_MOUNTCVMFS $CIENV_MOUNTCVMFS echo CIENV_JOBDIR "$CIENV_JOBDIR" echo CIENV_CVMFSVOLUME "$CIENV_CVMFSVOLUME" -- GitLab From 85ee1f1a69b323071547822313dce80a89b692a5 Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sat, 6 Jul 2019 17:39:55 +0200 Subject: [PATCH 6/8] Rename BUILD_EVENT as CIENV_BUILDEVENT --- .gitlab-ci.sh | 8 ++++---- build-executor/main.sh | 8 ++++---- build-executor/run_build.sh | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index 38a89fef..172cdf68 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -56,9 +56,9 @@ if [ "$stage" == "build" ]; then echo "$var=${!var}" done - # Set the default BUILD_EVENT if not yet set by the gitlab CI or in run_build.sh - if [ "$BUILD_EVENT" == "" ]; then BUILD_EVENT=all; fi - echo "BUILD_EVENT=$BUILD_EVENT" + # Set the default CIENV_BUILDEVENT if not yet set by the gitlab CI or in run_build.sh + if [ "$CIENV_BUILDEVENT" == "" ]; then CIENV_BUILDEVENT=all; fi + echo "CIENV_BUILDEVENT=$CIENV_BUILDEVENT" # Set CIENV_MOUNTCVMFS=1 in the gitlab CI as it is not yet set # (conversely, CIENV_MOUNTCVMFS=0 should have been set in run_build.sh) @@ -96,7 +96,7 @@ if [ "$stage" == "build" ]; then -v $DOCSOCK:$DOCSOCK \ -v $CIENV_CVMFSVOLUME:/cvmfs:shared \ -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \ - $BUILDIMG -m -s $HEP_WL_SPEC_FILE -e $BUILD_EVENT | tee -a $CIENV_JOBDIR/main.log + $BUILDIMG -m -s $HEP_WL_SPEC_FILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee... set +x # verbose off echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)" diff --git a/build-executor/main.sh b/build-executor/main.sh index c7352ccf..dfe1b6c6 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -417,7 +417,7 @@ export CVMFS_TRACES_DIR=$CIENV_JOBDIR/cvmfs-traces/ export CVMFS_EXPORT_DIR=$CIENV_JOBDIR/cvmfs-export/ [ -z $HEP_WL_SPEC_FILE ] && HEP_WL_SPEC_FILE='' -[ -z $BUILD_EVENT ] && BUILD_EVENT="all" +[ -z $CIENV_BUILDEVENT ] && CIENV_BUILDEVENT="all" [ -z $CIENV_DOCKERREGISTRY ] && CIENV_DOCKERREGISTRY="" [ -z $CIENV_MOUNTCVMFS ] && CIENV_MOUNTCVMFS=0 [ -z $CIENV_JOBDIR ] && CIENV_JOBDIR=/tmp @@ -432,7 +432,7 @@ while getopts "hds:r:e:m" o; do [ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG" ;; e) - [ "$OPTARG" != "" ] && export BUILD_EVENT="$OPTARG" + [ "$OPTARG" != "" ] && export CIENV_BUILDEVENT="$OPTARG" ;; r) [ "$OPTARG" != "" ] && export CIENV_DOCKERREGISTRY="$OPTARG" @@ -455,7 +455,7 @@ done echo -e "\n-------------\n[main] Input parameters" echo HEP_WL_SPEC_FILE $HEP_WL_SPEC_FILE -echo BUILD_EVENT $BUILD_EVENT +echo CIENV_BUILDEVENT $CIENV_BUILDEVENT echo CIENV_DOCKERREGISTRY $CIENV_DOCKERREGISTRY echo CIENV_MOUNTCVMFS $CIENV_MOUNTCVMFS echo CIENV_JOBDIR "$CIENV_JOBDIR" @@ -490,7 +490,7 @@ if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then mount_cvmfs || fail '[main.sh] mount_cvmfs' fi -execute_procedure "$BUILD_EVENT" +execute_procedure "$CIENV_BUILDEVENT" if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then unmount_cvmfs || fail '[main.sh] unmount_cvmfs' diff --git a/build-executor/run_build.sh b/build-executor/run_build.sh index beb5faf3..cf5fe24c 100755 --- a/build-executor/run_build.sh +++ b/build-executor/run_build.sh @@ -36,7 +36,7 @@ function usage(){ # Input parameters export HEP_WL_SPEC_FILE="" # no default (will fail) -export BUILD_EVENT="" # no default (will be set in .gitlab-ci.sh) +export CIENV_BUILDEVENT="" # no default (will be set in .gitlab-ci.sh) REBUILD=0 while getopts "hs:e:df" o; do case ${o} in @@ -44,7 +44,7 @@ while getopts "hs:e:df" o; do [ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG" ;; e) - [ "$OPTARG" != "" ] && export BUILD_EVENT="$OPTARG" + [ "$OPTARG" != "" ] && export CIENV_BUILDEVENT="$OPTARG" ;; d) set -x # Enable debug printouts -- GitLab From 71e21c9afa6f4cb7ee57cf0bb09fbb4210570482 Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sun, 7 Jul 2019 20:36:50 +0200 Subject: [PATCH 7/8] Rename HEP_WL_SPEC_FILE as CIENV_HEPWL_SPECFILE --- .gitlab-ci.sh | 12 ++++++------ .gitlab-ci.yml | 20 ++++++++++---------- build-executor/generate_Dockerfile.sh | 10 +++++----- build-executor/main.sh | 26 +++++++++++++------------- build-executor/run_build.sh | 8 ++++---- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.gitlab-ci.sh b/.gitlab-ci.sh index 172cdf68..a10d9f91 100755 --- a/.gitlab-ci.sh +++ b/.gitlab-ci.sh @@ -41,13 +41,13 @@ fi # if [ "$stage" == "build" ]; then - # Sanity check: the value of HEP_WL_SPEC_FILE must have been passed in .gitlab-ci.yml or run_build.sh - if [ "$HEP_WL_SPEC_FILE" == "" ]; then echo "ERROR! HEP_WL_SPEC_FILE is not set"; exit 1; fi - echo "Build image from spec file $HEP_WL_SPEC_FILE" + # Sanity check: the value of CIENV_HEPWL_SPECFILE must have been passed in .gitlab-ci.yml or run_build.sh + if [ "$CIENV_HEPWL_SPECFILE" == "" ]; then echo "ERROR! CIENV_HEPWL_SPECFILE is not set"; exit 1; fi + echo "Build image from spec file $CIENV_HEPWL_SPECFILE" # Check that the spec file exists and pass it to main.sh as a fully qualified path - if [ ! -e $HEP_WL_SPEC_FILE ]; then echo "ERROR! $HEP_WL_SPEC_FILE does not exist"; exit 1; fi - HEP_WL_SPEC_FILE=$(cd $(dirname $HEP_WL_SPEC_FILE); pwd)/$(basename $HEP_WL_SPEC_FILE) + if [ ! -e $CIENV_HEPWL_SPECFILE ]; then echo "ERROR! $CIENV_HEPWL_SPECFILE does not exist"; exit 1; fi + CIENV_HEPWL_SPECFILE=$(cd $(dirname $CIENV_HEPWL_SPECFILE); pwd)/$(basename $CIENV_HEPWL_SPECFILE) # Sanity check: the following CI_ variables must have been set by the gitlab CI or in run_build.sh for var in CI_JOB_ID CI_JOB_NAME CI_PROJECT_DIR CI_REGISTRY_IMAGE; do @@ -96,7 +96,7 @@ if [ "$stage" == "build" ]; then -v $DOCSOCK:$DOCSOCK \ -v $CIENV_CVMFSVOLUME:/cvmfs:shared \ -v $CI_PROJECT_DIR:$CI_PROJECT_DIR \ - $BUILDIMG -m -s $HEP_WL_SPEC_FILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log + $BUILDIMG -m -s $CIENV_HEPWL_SPECFILE -e $CIENV_BUILDEVENT | tee -a $CIENV_JOBDIR/main.log status=${PIPESTATUS[0]} # NB do not use $? if you pipe to tee... set +x # verbose off echo -e "\n[gitlab-ci.sh] $stage back to the CI at $(date)" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c69a1808..1a5a21c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,7 +51,7 @@ job_build-hepwlbuilder: job_test-hepwlbuilder: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/atlas/kv-bmk-v17.8.0.9/kv-bmk.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv-bmk-v17.8.0.9/kv-bmk.spec only: refs: - qa @@ -67,7 +67,7 @@ job_test-hepwlbuilder: job_build-alice-gen-sim: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/alice/gen-sim/alice-bmk.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/alice/gen-sim/alice-bmk.spec only: refs: - qa @@ -82,7 +82,7 @@ job_build-alice-gen-sim: job_build-kv: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/atlas/kv-bmk-v17.8.0.9/kv-bmk.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/kv-bmk-v17.8.0.9/kv-bmk.spec only: refs: - qa @@ -97,7 +97,7 @@ job_build-kv: job_build-atlas-gen: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/atlas/gen/atlas-gen.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/gen/atlas-gen.spec only: refs: - qa @@ -108,7 +108,7 @@ job_build-atlas-gen: job_build-atlas-sim: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/atlas/sim/atlas-sim.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/sim/atlas-sim.spec only: refs: - qa @@ -119,7 +119,7 @@ job_build-atlas-sim: job_build-atlas-reco: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/atlas/reco/atlas-reco.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/atlas/reco/atlas-reco.spec only: refs: - qa @@ -134,7 +134,7 @@ job_build-atlas-reco: job_build-cms-gen-sim: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/cms/gen-sim/cms-gen-sim.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/cms/gen-sim/cms-gen-sim.spec only: refs: - qa @@ -146,7 +146,7 @@ job_build-cms-gen-sim: job_build-cms-digi: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/cms/digi/cms-digi.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/cms/digi/cms-digi.spec only: refs: - qa @@ -157,7 +157,7 @@ job_build-cms-digi: job_build-cms-reco: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/cms/reco/cms-reco.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/cms/reco/cms-reco.spec only: refs: - qa @@ -173,7 +173,7 @@ job_build-cms-reco: job_build-lhcb-gen-sim: variables: - HEP_WL_SPEC_FILE: $CI_PROJECT_DIR/lhcb/gen-sim/lhcb-bmk.spec + CIENV_HEPWL_SPECFILE: $CI_PROJECT_DIR/lhcb/gen-sim/lhcb-bmk.spec only: refs: - qa diff --git a/build-executor/generate_Dockerfile.sh b/build-executor/generate_Dockerfile.sh index c86fffbc..fe5387b2 100644 --- a/build-executor/generate_Dockerfile.sh +++ b/build-executor/generate_Dockerfile.sh @@ -4,14 +4,14 @@ function generate_Dockerfile(){ - HEP_WL_SPEC_FILE="" + CIENV_HEPWL_SPECFILE="" DOCKERFILE_TEMPLATE="" local OPTIND while getopts "hs:d:" o; do case ${o} in s) - [ "$OPTARG" != "" ] && HEP_WL_SPEC_FILE="$OPTARG" + [ "$OPTARG" != "" ] && CIENV_HEPWL_SPECFILE="$OPTARG" ;; d) [ "$OPTARG" != "" ] && DOCKERFILE_TEMPLATE="$OPTARG" @@ -23,13 +23,13 @@ function generate_Dockerfile(){ esac done - echo "[generate_Dockerfile] HEP_WL_SPEC_FILE: $HEP_WL_SPEC_FILE" + echo "[generate_Dockerfile] CIENV_HEPWL_SPECFILE: $CIENV_HEPWL_SPECFILE" echo "[generate_Dockerfile] DOCKERFILE_TEMPLATE: $DOCKERFILE_TEMPLATE" - [[ "$HEP_WL_SPEC_FILE" == "" ]] && fail "SPEC file not specified" + [[ "$CIENV_HEPWL_SPECFILE" == "" ]] && fail "SPEC file not specified" [[ "$DOCKERFILE_TEMPLATE" == "" ]] && fail "No template Dockerfile specified" - source "$HEP_WL_SPEC_FILE" + source "$CIENV_HEPWL_SPECFILE" DOCKERFILE="`dirname $DOCKERFILE_TEMPLATE`/Dockerfile" cat $DOCKERFILE_TEMPLATE | sed -e "s@__insert_bmkdir__@$HEPWL_BMKDIR@g" \ -e "s@__insert_bmkdescription__@$HEPWL_BMKDESCRIPTION@g" \ diff --git a/build-executor/main.sh b/build-executor/main.sh index dfe1b6c6..88f6e9d1 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -74,7 +74,7 @@ function unmount_cvmfs(){ # [This is called twice: to build the image with cvmfs (cvmfs-shrink) and without cvmfs (standalone image)] # Input $1: docker image, without registry prefix (image_name:image_tag) # Input environment variable ${HEP_WL_SOURCE_DIR}: directory containing the Dockerfile.template -# Input environment variable ${HEP_WL_SPEC_FILE}: spec file for the image to be built +# Input environment variable ${CIENV_HEPWL_SPECFILE}: spec file for the image to be built # Output: image stored in the local Docker image registry function build_docker_image(){ echo "[build_docker_image] starting at $(date)" @@ -82,7 +82,7 @@ function build_docker_image(){ IMAGE=$1 echo "[build_docker_image] " $IMAGE cd ${HEP_WL_SOURCE_DIR} - generate_Dockerfile -s $HEP_WL_SPEC_FILE -d ./Dockerfile.template || fail "[build_docker_image] generate_Dockerfile" + generate_Dockerfile -s $CIENV_HEPWL_SPECFILE -d ./Dockerfile.template || fail "[build_docker_image] generate_Dockerfile" cat Dockerfile docker build -t "${IMAGE}" . || fail "[build_docker_image] docker build -t ${IMAGE} ." ###docker build --build-arg CACHEBUST=$(date +%s) -t "${IMAGE}" . || fail "[build_docker_image] docker build -t ${IMAGE} ." @@ -95,7 +95,7 @@ function build_docker_image(){ # Input environment variable ${HEPWL_DOCKERIMAGE}: image name for the HEP workload # Input environment variable ${HEPWL_BMKOPTS}: options for the HEP workload script # Input environment variable ${HEP_WL_SOURCE_DIR}: directory containing the Dockerfile.template -# Input environment variable ${HEP_WL_SPEC_FILE}: spec file for the image to be built +# Input environment variable ${CIENV_HEPWL_SPECFILE}: spec file for the image to be built # Input environment variable ${CIENV_JOBDIR}/results: host directory where results should be stored # Input environment variable ${CIENV_CVMFSVOLUME}: host directory where cvmfs is mounted? # Input environment variable ${CVMFS_TRACES_DIR}: directory containing the cvmfs trace files @@ -416,7 +416,7 @@ echo "SCRIPT_DIR is ${SCRIPT_DIR}" export CVMFS_TRACES_DIR=$CIENV_JOBDIR/cvmfs-traces/ export CVMFS_EXPORT_DIR=$CIENV_JOBDIR/cvmfs-export/ -[ -z $HEP_WL_SPEC_FILE ] && HEP_WL_SPEC_FILE='' +[ -z $CIENV_HEPWL_SPECFILE ] && CIENV_HEPWL_SPECFILE='' [ -z $CIENV_BUILDEVENT ] && CIENV_BUILDEVENT="all" [ -z $CIENV_DOCKERREGISTRY ] && CIENV_DOCKERREGISTRY="" [ -z $CIENV_MOUNTCVMFS ] && CIENV_MOUNTCVMFS=0 @@ -429,7 +429,7 @@ export CVMFS_EXPORT_DIR=$CIENV_JOBDIR/cvmfs-export/ while getopts "hds:r:e:m" o; do case ${o} in s) - [ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG" + [ "$OPTARG" != "" ] && export CIENV_HEPWL_SPECFILE="$OPTARG" ;; e) [ "$OPTARG" != "" ] && export CIENV_BUILDEVENT="$OPTARG" @@ -454,7 +454,7 @@ while getopts "hds:r:e:m" o; do done echo -e "\n-------------\n[main] Input parameters" -echo HEP_WL_SPEC_FILE $HEP_WL_SPEC_FILE +echo CIENV_HEPWL_SPECFILE $CIENV_HEPWL_SPECFILE echo CIENV_BUILDEVENT $CIENV_BUILDEVENT echo CIENV_DOCKERREGISTRY $CIENV_DOCKERREGISTRY echo CIENV_MOUNTCVMFS $CIENV_MOUNTCVMFS @@ -464,27 +464,27 @@ echo "-------------" env | grep "CI_" | sort -[ "$HEP_WL_SPEC_FILE" == "" ] && fail "SPEC file not specified" +[ "$CIENV_HEPWL_SPECFILE" == "" ] && fail "SPEC file not specified" -[ ! -f $HEP_WL_SPEC_FILE ] && fail "SPEC file $HEP_WL_SPEC_FILE not found" +[ ! -f $CIENV_HEPWL_SPECFILE ] && fail "SPEC file $CIENV_HEPWL_SPECFILE not found" -HEP_WL_SOURCE_DIR=$(dirname "$HEP_WL_SPEC_FILE") # Original HEP source dir +HEP_WL_SOURCE_DIR=$(dirname "$CIENV_HEPWL_SPECFILE") # Original HEP source dir mkdir -p $CIENV_JOBDIR/build-wl cp -r $HEP_WL_SOURCE_DIR/* $CIENV_JOBDIR/build-wl export HEP_WL_SOURCE_DIR=$CIENV_JOBDIR/build-wl -HEP_WL_SPEC_FILE=$CIENV_JOBDIR/build-wl/${HEP_WL_SPEC_FILE##*/} +CIENV_HEPWL_SPECFILE=$CIENV_JOBDIR/build-wl/${CIENV_HEPWL_SPECFILE##*/} -echo "----------- $HEP_WL_SPEC_FILE" -cat "$HEP_WL_SPEC_FILE" +echo "----------- $CIENV_HEPWL_SPECFILE" +cat "$CIENV_HEPWL_SPECFILE" echo "-----------" # This sets all environment variables starting with HEPWL_ used in this script (BMK-118) # It should always set HEPWL_BMKEXE, HEPWL_BMKOPTS, HEPWL_BMKDIR, HEPWL_BMKDESCRIPTION, # as well as HEPWL_DOCKERIMAGE, HEPWL_DOCKERIMAGETAG and HEPWL_CVMFSREPOS # It may optionally set also HEPWL_ANNOUNCE and HEPWL_EXTEND_<repo>_SPEC -source "$HEP_WL_SPEC_FILE" +source "$CIENV_HEPWL_SPECFILE" if [[ "$CIENV_MOUNTCVMFS" -gt 0 ]]; then mount_cvmfs || fail '[main.sh] mount_cvmfs' diff --git a/build-executor/run_build.sh b/build-executor/run_build.sh index cf5fe24c..1f69be39 100755 --- a/build-executor/run_build.sh +++ b/build-executor/run_build.sh @@ -35,13 +35,13 @@ function usage(){ } # Input parameters -export HEP_WL_SPEC_FILE="" # no default (will fail) +export CIENV_HEPWL_SPECFILE="" # no default (will fail) export CIENV_BUILDEVENT="" # no default (will be set in .gitlab-ci.sh) REBUILD=0 while getopts "hs:e:df" o; do case ${o} in s) - [ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG" + [ "$OPTARG" != "" ] && export CIENV_HEPWL_SPECFILE="$OPTARG" ;; e) [ "$OPTARG" != "" ] && export CIENV_BUILDEVENT="$OPTARG" @@ -57,7 +57,7 @@ while getopts "hs:e:df" o; do ;; esac done -if [ "$HEP_WL_SPEC_FILE" == "" ]; then usage; fi +if [ "$CIENV_HEPWL_SPECFILE" == "" ]; then usage; fi # First of all: build the image builder if [ "$REBUILD" == "1" ]; then @@ -67,5 +67,5 @@ if [ "$REBUILD" == "1" ]; then fi # Delegate the rest to the .gitlab-ci.sh script (BMK-50) -echo "*** Build from $HEP_WL_SPEC_FILE" +echo "*** Build from $CIENV_HEPWL_SPECFILE" ${CI_PROJECT_DIR}/.gitlab-ci.sh build -- GitLab From fd27759e28de4053361b5c6cd56fce26242f8b51 Mon Sep 17 00:00:00 2001 From: Andrea Valassi <andrea.valassi@cern.ch> Date: Sun, 7 Jul 2019 20:40:17 +0200 Subject: [PATCH 8/8] Revert for now: rename MAIN_NETWORKCONN as NETWORK_CONN --- build-executor/main.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-executor/main.sh b/build-executor/main.sh index 88f6e9d1..8361f429 100755 --- a/build-executor/main.sh +++ b/build-executor/main.sh @@ -242,7 +242,7 @@ function test_standalone_image(){ [ -z "$HEPWL_BMKOPTS" ] && HEPWL_BMKOPTS="" strace="" net_conn="--network none" # by default run the test without network connectivity - [[ "$MAIN_NETWORKCONN" == "1" ]] && net_conn="" + [[ "$NETWORK_CONN" == "1" ]] && net_conn="" ###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051) execute docker run ${strace} ${net_conn} --rm -v $CIENV_JOBDIR/results:/results $IMAGE $HEPWL_BMKOPTS || fail "[test_standalone_image] docker run $IMAGE" echo "[test_standalone_image] finished at $(date)" @@ -421,7 +421,7 @@ export CVMFS_EXPORT_DIR=$CIENV_JOBDIR/cvmfs-export/ [ -z $CIENV_DOCKERREGISTRY ] && CIENV_DOCKERREGISTRY="" [ -z $CIENV_MOUNTCVMFS ] && CIENV_MOUNTCVMFS=0 [ -z $CIENV_JOBDIR ] && CIENV_JOBDIR=/tmp -[ -z $MAIN_NETWORKCONN ] && MAIN_NETWORKCONN=0 +[ -z $NETWORK_CONN ] && NETWORK_CONN=0 # Parameters to be passed by command line # 1: SPEC file path @@ -441,7 +441,7 @@ while getopts "hds:r:e:m" o; do CIENV_MOUNTCVMFS=1 ;; n) - MAIN_NETWORKCONN=1 + NETWORK_CONN=1 ;; d) set -x # Enable debug printouts -- GitLab