Skip to content
Snippets Groups Projects

Cosmetic changes in main.sh

Merged Cosmetic changes in main.sh
AV-cosmetics into qa
Merged Domenico Giordano requested to merge AV-cosmetics into qa
Compare
and
1 file
+ 96
− 102
Compare changes
  • Side-by-side
  • Inline
build-executor/main.sh
+ 96
− 102
#!/bin/bash
function fail(){
echo "Failing " "$@" >&2
echo "Failing '$@'" >&2
exit -1
}
function execute(){
echo "Executing " "$@"
echo "Executing '$@'"
eval "$@"
}
function run_docker_wl(){
function run_docker_wl(){
[ -z "$bmkopts" ] && bmkopts=""
echo "[run_docker_wl] bmkopts options " $bmkopts
IMAGE="$DOCKER_IMAGE:cvmfs-shrink"
echo "[run_docker_wl] Building temporary docker image $IMAGE"
build_docker_image $IMAGE
build_docker_image $IMAGE
echo -ne "\n[run_docker_wl] Running: "
execute docker run --rm -v $JOB_DIR/results:/results -v $HEP_WL_SOURCE_DIR:$HEP_WL_SOURCE_DIR -w $HEP_WL_SOURCE_DIR -v $CVMFS_VOLUME:/cvmfs:shared $IMAGE $bmkopts
STATUS=$?
@@ -25,25 +26,24 @@ function run_docker_wl(){
return $STATUS
}
function flush_cache(){
for acvmfs in `ls ${CVMFS_TRACES_DIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
echo "[flush_cache] ... flushing " $acvmfs;
cvmfs_talk -i ${acvmfs}.cern.h tracebuffer flush;
function flush_cache(){
for acvmfs in `ls ${CVMFS_TRACES_DIR} | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1@'`; do
echo "[flush_cache] ... flushing " $acvmfs;
cvmfs_talk -i ${acvmfs}.cern.h tracebuffer flush;
done
}
function run_shrinkwrap(){
echo "CVMFS_EXPORT_DIR $CVMFS_EXPORT_DIR"
if [ -e ${CVMFS_EXPORT_DIR} ]; then
echo "[run_shrinkwrap] ${CVMFS_EXPORT_DIR} already exists. removing it" #? (y/[n]) "
#read myansw
#[ "$myansw" == "n" ] && return 1
rm -rf ${CVMFS_EXPORT_DIR}
echo "[run_shrinkwrap] ${CVMFS_EXPORT_DIR} already exists. removing it" #? (y/[n]) "
#read myansw
#[ "$myansw" == "n" ] && return 1
rm -rf ${CVMFS_EXPORT_DIR}
fi
mkdir -p ${CVMFS_EXPORT_DIR}/cvmfs
cat > ${CVMFS_SHRINK_CONF} <<EOF
CVMFS_CACHE_BASE=/var/lib/cvmfs/shrinkwrap
CVMFS_HTTP_PROXY=DIRECT
@@ -53,37 +53,36 @@ CVMFS_SERVER_URL='http://cvmfs-stratum-zero-hpc.cern.ch/cvmfs/@fqrn@' # from
CVMFS_SHARED_CACHE=no
export CMS_LOCAL_SITE=T0_CH_CERN
EOF
for acvmfs in `ls ${CVMFS_TRACES_DIR} | grep "\.log"`; do
echo "[run_shrinkwrap] ... shrinking " $acvmfs;
specname=`echo $acvmfs | sed -e 's@trace\.log@spec\.txt@'`
reponame=`echo $acvmfs | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1\.cern.ch@'`
echo "[run_shrinkwrap] specname $specname"
echo "[run_shrinkwrap] reponame $reponame"
echo "[run_shrinkwrap] python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact ${CVMFS_TRACES_DIR}/$acvmfs ${CVMFS_TRACES_DIR}/$specname"
python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact ${CVMFS_TRACES_DIR}/$acvmfs ${CVMFS_TRACES_DIR}/$specname
trimname=${reponame/.cern.ch}
SPEC_VAR=EXTEND_${trimname^^}_SPEC
[ ! -z ${!SPEC_VAR} ] && [ -e $HEP_WL_SOURCE_DIR/${!SPEC_VAR} ] && echo "[run_shrinkwrap] appending custom paths to ${CVMFS_TRACES_DIR}/$specname based on $HEP_WL_SOURCE_DIR/${!SPEC_VAR}" && echo " " >> ${CVMFS_TRACES_DIR}/$specname && cat $HEP_WL_SOURCE_DIR/${!SPEC_VAR} >> ${CVMFS_TRACES_DIR}/$specname
echo "[run_shrinkwrap] cvmfs_shrinkwrap --repo $reponame --src-config ${CVMFS_SHRINK_CONF} --spec-file ${CVMFS_TRACES_DIR}/$specname --dest-base ${CVMFS_EXPORT_DIR}/cvmfs/ -j 4"
cvmfs_shrinkwrap --repo $reponame --src-config ${CVMFS_SHRINK_CONF} --spec-file ${CVMFS_TRACES_DIR}/$specname --dest-base ${CVMFS_EXPORT_DIR}/cvmfs/ -j 4 || echo "[run_shrinkwrap] is the cvmfs_shrinkwrap failing??"
for acvmfs in `ls ${CVMFS_TRACES_DIR} | grep "\.log"`; do
echo "[run_shrinkwrap] ... shrinking " $acvmfs;
specname=`echo $acvmfs | sed -e 's@trace\.log@spec\.txt@'`
reponame=`echo $acvmfs | sed -e 's@cvmfs-\([^\.]*\)\.cern\.ch.*@\1\.cern.ch@'`
echo "[run_shrinkwrap] specname $specname"
echo "[run_shrinkwrap] reponame $reponame"
echo "[run_shrinkwrap] python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact ${CVMFS_TRACES_DIR}/$acvmfs ${CVMFS_TRACES_DIR}/$specname"
python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact ${CVMFS_TRACES_DIR}/$acvmfs ${CVMFS_TRACES_DIR}/$specname
trimname=${reponame/.cern.ch}
SPEC_VAR=EXTEND_${trimname^^}_SPEC
[ ! -z ${!SPEC_VAR} ] && [ -e $HEP_WL_SOURCE_DIR/${!SPEC_VAR} ] && echo "[run_shrinkwrap] appending custom paths to ${CVMFS_TRACES_DIR}/$specname based on $HEP_WL_SOURCE_DIR/${!SPEC_VAR}" && echo " " >> ${CVMFS_TRACES_DIR}/$specname && cat $HEP_WL_SOURCE_DIR/${!SPEC_VAR} >> ${CVMFS_TRACES_DIR}/$specname
echo "[run_shrinkwrap] cvmfs_shrinkwrap --repo $reponame --src-config ${CVMFS_SHRINK_CONF} --spec-file ${CVMFS_TRACES_DIR}/$specname --dest-base ${CVMFS_EXPORT_DIR}/cvmfs/ -j 4"
cvmfs_shrinkwrap --repo $reponame --src-config ${CVMFS_SHRINK_CONF} --spec-file ${CVMFS_TRACES_DIR}/$specname --dest-base ${CVMFS_EXPORT_DIR}/cvmfs/ -j 4 || echo "[run_shrinkwrap] is the cvmfs_shrinkwrap failing??"
done
}
function copy_cvmfs(){
rm -rf ${CVMFS_EXPORT_DIR}/cvmfs/.data
[[ -e $HEP_WL_SOURCE_DIR/cvmfs ]] && echo "[copy_cvmfs] removing cvmfs dir in $HEP_WL_SOURCE_DIR/" && rm -rf $HEP_WL_SOURCE_DIR/cvmfs
[[ -e $HEP_WL_SOURCE_DIR/cvmfs ]] && echo "[copy_cvmfs] removing cvmfs dir in $HEP_WL_SOURCE_DIR/" && rm -rf $HEP_WL_SOURCE_DIR/cvmfs
echo "[copy_cvmfs] mv ${CVMFS_EXPORT_DIR}/cvmfs $HEP_WL_SOURCE_DIR "
mv ${CVMFS_EXPORT_DIR}/cvmfs $HEP_WL_SOURCE_DIR || fail "cannot mv ${CVMFS_EXPORT_DIR}/cvmfs $HEP_WL_SOURCE_DIR"
mv ${CVMFS_EXPORT_DIR}/cvmfs $HEP_WL_SOURCE_DIR || fail "cannot mv ${CVMFS_EXPORT_DIR}/cvmfs $HEP_WL_SOURCE_DIR"
#FIXME: if cms repo, need to copy by hand the SITECONF/local, because it's a sym link
[[ -e /cvmfs/cms.cern.ch/SITECONF/local ]] && ( cp -r -H /cvmfs/cms.cern.ch/SITECONF/local $HEP_WL_SOURCE_DIR/cvmfs/cms.cern.ch/SITECONF/ || fail "cannot cp /cvmfs/cms.cern.ch/SITECONF/local")
ls $HEP_WL_SOURCE_DIR/cvmfs
}
function build_docker_image(){
[[ "$1" == "" ]] && fail "[build_docker_image] No image name provided. Failing "
[[ "$1" == "" ]] && fail "[build_docker_image] No image name provided. Failing "
IMAGE=$1
cd ${HEP_WL_SOURCE_DIR}
generate_Dockerfile -s $HEP_WL_SPEC_FILE -d ./Dockerfile.template
@@ -91,11 +90,12 @@ function build_docker_image(){
docker build -t "${IMAGE}" . || fail "docker build -t ${IMAGE}"
}
function clean_area(){
echo "removing copy of cvmfs dir " && rm -rf "$HEP_WL_SOURCE_DIR/cvmfs"
}
function test_docker_image(){
echo "[test_docker_image] " $1
IMAGE=$1
@@ -103,6 +103,7 @@ function test_docker_image(){
execute docker run --rm -v $JOB_DIR/results:/results $IMAGE $bmkopts
}
function publish_docker_image(){
[[ "${DOCKER_REGISTRY}" == "" ]] && return 0
echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch
@@ -110,6 +111,7 @@ function publish_docker_image(){
docker push "${IMAGE}" || fail "docker push ${IMAGE}"
}
function build_standalone_image(){
IMAGE="${DOCKER_IMAGE}":"${DOCKER_IMAGE_TAG}"
echo "[build_standalone_image] IMAGE: $IMAGE"
@@ -126,21 +128,18 @@ function set_dir(){
cd $2
eval $1=`pwd -P`
cd $current_dir
eval echo "setting var " $1 " to value " \$$1
eval echo "setting var " $1 " to value " \$$1
eval export $1
return 0
}
function mount_cvmfs(){
CVMFS_REPOSITORIES=$1
[[ "$CVMFS_REPOSITORIES" == "" ]] && fail "no repo to mount in cvmfs"
echo -e "\nmounting cvmfs repositories $CVMFS_REPOSITORIES"
echo -e "NB: it's fine if there is a single error message such as 'Failed to get D-Bus connection: Operation not permitted' "
[[ ! -e $CVMFS_TRACES_DIR ]] && ((mkdir -p $CVMFS_TRACES_DIR && chown cvmfs $CVMFS_TRACES_DIR) || fail "cannot create $CVMFS_TRACES_DIR")
cat > /etc/cvmfs/default.local <<EOF
CVMFS_REPOSITORIES=${CVMFS_REPOSITORIES}
CVMFS_QUOTA_LIMIT=6000
@@ -149,64 +148,59 @@ CVMFS_MOUNT_RW=yes
CVMFS_HTTP_PROXY="http://squid.cern.ch:8060|http://ca-proxy.cern.ch:3128;DIRECT"
CVMFS_TRACEFILE=${CVMFS_TRACES_DIR}/cvmfs-@fqrn@.trace.log
EOF
mkdir -p /etc/cvmfs/config.d
echo "export CMS_LOCAL_SITE=/cvmfs/cms.cern.ch/SITECONF/T0_CH_CERN" > /etc/cvmfs/config.d/cms.cern.ch.local
#[[ `ls /cvmfs | wc -l` -gt 0 ]] && fail "/cvmfs already contains files. remove them"
#[[ `ls /cvmfs | wc -l` -gt 0 ]] && fail "/cvmfs already contains files. remove them"
cvmfs_config setup nostart || fail "problem with cvmfs_config setup nostart"
for repo in `echo ${CVMFS_REPOSITORIES}| sed -e 's@,@ @g'`;
do
umount /cvmfs/$repo
rm -rf /cvmfs/$repo
mkdir -p /cvmfs/$repo
mount -t cvmfs $repo /cvmfs/$repo
umount /cvmfs/$repo
rm -rf /cvmfs/$repo
mkdir -p /cvmfs/$repo
mount -t cvmfs $repo /cvmfs/$repo
done
}
#Those are fixed params, to be moved in an etc file
export CVMFS_TRACES_DIR=$JOB_DIR/cvmfs-traces/
export CVMFS_EXPORT_DIR=$JOB_DIR/cvmfs-export/
export CVMFS_SHRINK_CONF=$JOB_DIR/generic_config.cern.ch.config
export DOCKER_BASE_IMAGE='gitlab-registry.cern.ch/cloud-infrastructure/cloud-benchmark-suite/hepworkloads-base:1.0'
### Main
#parameters to be passed by command line
#1: SPEC file path
#2: optional export DOCKER_REGISTRY gitlab-registry.cern.ch/giordano/hep-workloads
function execute_procedure(){
case $1 in
sleep)
echo "not running any step of the procedure, just keep alive to expose cvmfs"
sleep infinity
;;
build)
sleep)
echo "not running any step of the procedure, just keep alive to expose cvmfs"
sleep infinity
;;
build)
echo "building docker image. Skipping previous steps"
build_standalone_image || fail 'build_standalone_image'
;;
shrink)
build_standalone_image || fail 'build_standalone_image'
;;
shrink)
echo "Shrinking cvmfs and building docker image"
run_shrinkwrap || fail 'run_shrinkwrap'
copy_cvmfs || fail 'copy_cvmfs'
build_standalone_image || fail 'build_standalone_image'
;;
all | *)
echo "Running full sequence"
run_docker_wl || fail 'run_docker_wl'
#flush_cache || fail #this probably needs to share /val/lib/cvmfs inside the container
run_shrinkwrap || fail 'run_shrinkwrap'
copy_cvmfs || fail 'copy_cvmfs'
build_standalone_image || fail 'build_standalone_image'
run_shrinkwrap || fail 'run_shrinkwrap'
copy_cvmfs || fail 'copy_cvmfs'
build_standalone_image || fail 'build_standalone_image'
;;
all | *)
echo "Running full sequence"
run_docker_wl || fail 'run_docker_wl'
#flush_cache || fail #this probably needs to share /val/lib/cvmfs inside the container
run_shrinkwrap || fail 'run_shrinkwrap'
copy_cvmfs || fail 'copy_cvmfs'
build_standalone_image || fail 'build_standalone_image'
esac
}
### Main
#parameters to be passed by command line
#1: SPEC file path
#2: optional export DOCKER_REGISTRY gitlab-registry.cern.ch/giordano/hep-workloads
#Those are fixed params, to be moved in an etc file
export CVMFS_TRACES_DIR=$JOB_DIR/cvmfs-traces/
export CVMFS_EXPORT_DIR=$JOB_DIR/cvmfs-export/
export CVMFS_SHRINK_CONF=$JOB_DIR/generic_config.cern.ch.config
export DOCKER_BASE_IMAGE='gitlab-registry.cern.ch/cloud-infrastructure/cloud-benchmark-suite/hepworkloads-base:1.0'
SCRIPT_DIR=`dirname $0`
. $SCRIPT_DIR/generate_Dockerfile.sh
@@ -217,25 +211,25 @@ SCRIPT_DIR=`dirname $0`
[ -z $JOB_DIR ] && JOB_DIR=/tmp
while getopts "hs:r:e:m" o; do
case ${o} in
s)
[ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG"
;;
r)
[ "$OPTARG" != "" ] && export DOCKER_REGISTRY="$OPTARG"
;;
e)
[ "$OPTARG" != "" ] && export BUILD_EVENT="$OPTARG"
;;
m)
MOUNT_CVMFS=1
;;
h)
echo "usage: $0 -s <full path to HEP Workload Spec file> [-r <docker registry> -e <event from which to start (all|shrink|build|top|none)>]"
exit 0
;;
esac
case ${o} in
s)
[ "$OPTARG" != "" ] && export HEP_WL_SPEC_FILE="$OPTARG"
;;
r)
[ "$OPTARG" != "" ] && export DOCKER_REGISTRY="$OPTARG"
;;
e)
[ "$OPTARG" != "" ] && export BUILD_EVENT="$OPTARG"
;;
m)
MOUNT_CVMFS=1
;;
h)
echo "usage: $0 -s <full path to HEP Workload Spec file> [-r <docker registry> -e <event from which to start (all|shrink|build|top|none)>]"
exit 0
;;
esac
done
echo -e "\n-------------\n[main] Input parameters"
@@ -249,7 +243,7 @@ echo "-------------"
env | grep "CI_"
[[ ( "$HEP_WL_SPEC_FILE" == "" ) ]] && fail "SPEC file not specified"
[[ ( "$HEP_WL_SPEC_FILE" == "" ) ]] && fail "SPEC file not specified"
#Original HEP source dir
HEP_WL_SOURCE_DIR=`dirname "$HEP_WL_SPEC_FILE"`
@@ -260,7 +254,7 @@ export HEP_WL_SOURCE_DIR=$JOB_DIR/build-wl
HEP_WL_SPEC_FILE=$JOB_DIR/build-wl/${HEP_WL_SPEC_FILE##*/}
echo "----------- $HEP_WL_SPEC_FILE"
echo "----------- $HEP_WL_SPEC_FILE"
cat "$HEP_WL_SPEC_FILE"
echo "-----------"