Skip to content
Snippets Groups Projects

Resolve BMK-113

Merged Resolve BMK-113
valassi/hep-workloads:BMK-113 into qa
Merged Andrea Valassi requested to merge valassi/hep-workloads:BMK-113 into qa
Compare
and
2 files
+ 99
− 106
Compare changes
  • Side-by-side
  • Inline
Files
2
build-executor/main.sh
+ 64
− 72
@@ -7,7 +7,6 @@ function fail(){
exit 1
}
function execute(){
echo "Executing '$@'"
eval "$@"
@@ -67,7 +66,7 @@ function unmount_cvmfs(){
# Build a docker image
# [This is called twice: to build the image with cvmfs (cvmfs-shrink) and without cvmfs (standalone image)]
# Input $1: image name
# Input $1: docker image, without registry prefix (image_name:image_tag)
# Input environment variable ${HEP_WL_SOURCE_DIR}: directory containing the Dockerfile.template
# Input environment variable ${HEP_WL_SPEC_FILE}: spec file for the image to be built
# Output: image stored in the local Docker image registry
@@ -235,11 +234,8 @@ function test_standalone_image(){
echo "[test_standalone_image] IMAGE: $IMAGE"
[ -z "$bmkopts" ] && bmkopts=""
strace=""
#by default run the test without network connectivity
net_conn="--network none"
net_conn="--network none" # by default run the test without network connectivity
[[ "$NETWORK_CONN" == "1" ]] && net_conn=""
###strace="--cap-add SYS_PTRACE" # optionally add SYS_PTRACE capability to use strace (see https://github.com/moby/moby/issues/21051)
execute docker run ${strace} ${net_conn} --rm -v $JOB_DIR/results:/results $IMAGE $bmkopts || fail "[test_standalone_image] docker run $IMAGE"
echo "[test_standalone_image] finished at $(date)"
@@ -247,7 +243,7 @@ function test_standalone_image(){
}
# Publish the standalone HEP workload image with a local /cvmfs
# Publish (and announce) the standalone HEP workload image with a local /cvmfs
# Input environment variable ${DOCKER_IMAGE}: image name for the HEP workload
# Input environment variable ${DOCKER_IMAGE_TAG}: image tag for the standalone HEP workload
# Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed
@@ -262,86 +258,82 @@ function publish_standalone_image(){
echo "[publish_standalone_image] IMAGE: $IMAGE"
echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch || fail "[publish_standalone_image] docker login"
echo "[publish_standalone_image] docker push ${IMAGE}"
docker push "${IMAGE}" || fail "[publish_standalone_image] docker push ${IMAGE}"
#flag as latest only images that respect the format v[0-9]*\.[0-9]*
if [[ ${DOCKER_IMAGE_TAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then
IMAGE_LATEST=${DOCKER_REGISTRY}/${DOCKER_IMAGE}:latest
docker tag "${IMAGE}" "${IMAGE_LATEST}" || fail "[publish_standalone_image] docker tag ${IMAGE} ${IMAGE_LATEST}"
docker push "${IMAGE_LATEST}" || fail "[publish_standalone_image] docker push ${IMAGE_LATEST}"
docker push "${IMAGE}" || fail "[publish_standalone_image] docker push ${IMAGE}"
if [[ ${DOCKER_IMAGE_TAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then # flag as latest only images that respect the format v[0-9]*\.[0-9]*
IMAGE_LATEST=${DOCKER_REGISTRY}/${DOCKER_IMAGE}:latest
docker tag "${IMAGE}" "${IMAGE_LATEST}" || fail "[publish_standalone_image] docker tag ${IMAGE} ${IMAGE_LATEST}"
docker push "${IMAGE_LATEST}" || fail "[publish_standalone_image] docker push ${IMAGE_LATEST}"
fi
[[ ( -z $bmkannounce ) || ( "${bmkannounce,,}" != "false" ) ]] && announce "${IMAGE}"
fi
echo "[publish_standalone_image] finished at $(date)"
return 0
}
# Announce the standalone HEP workload image with a local /cvmfs
# Input full docker image path (registry/image_name:image_tag)
# Input $CI_MAIL_FROM list
# Output: email to the $CI_MAIL_FROM list
# Announce the standalone HEP workload image with a local /cvmfs (BMK-80)
# (send an email from ${CI_MAIL_FROM} to ${CI_ANNOUNCE_TO} announcing a new image created in ${CI_JOB_URL})
# Input $1: docker image, including registry prefix (registry/image_name:image_tag)
# Input environment variables ${CI_MAIL_FROM}, ${CI_ANNOUNCE_TO} and ${CI_JOB_URL}
function announce(){
echo "[announce] published image started at $(date)"
IMAGE=$1
postfix start
ANNOUNCEMENT="announce.txt"
echo -e "Dear HEP Benchmark developers, \n" > $ANNOUNCEMENT
echo -e "we are pleased to inform that a new version has been released for the container image \n\n${IMAGE}" >> $ANNOUNCEMENT
echo -e "\nPlease DO NOT REPLY\nReport automatically generated from GitLab CI in job ${CI_JOB_URL}\n[$(date)]" >> $ANNOUNCEMENT
echo -e "\nYours sincerely,\nHEPiX Benchmarking Working Group\n\n" >> $ANNOUNCEMENT
cat $ANNOUNCEMENT
echo "[announce] sending email to $CI_ANNOUNCE_TO"
cat $ANNOUNCEMENT | mail -r $CI_MAIL_FROM -s "New Docker container available $IMAGE" $CI_ANNOUNCE_TO
sleep 100s
echo "[announce] finished at $(date)"
return 0
echo "[announce] published image started at $(date)"
IMAGE=$1
postfix start
ANNOUNCEMENT="announce.txt"
echo -e "Dear HEP Benchmark developers, \n" > $ANNOUNCEMENT
echo -e "we are pleased to inform that a new version has been released for the container image \n\n${IMAGE}" >> $ANNOUNCEMENT
echo -e "\nPlease DO NOT REPLY\nReport automatically generated from GitLab CI in job ${CI_JOB_URL}\n[$(date)]" >> $ANNOUNCEMENT
echo -e "\nYours sincerely,\nHEPiX Benchmarking Working Group\n\n" >> $ANNOUNCEMENT
cat $ANNOUNCEMENT
echo "[announce] sending email to $CI_ANNOUNCE_TO"
cat $ANNOUNCEMENT | mail -r $CI_MAIL_FROM -s "New Docker container available $IMAGE" $CI_ANNOUNCE_TO
sleep 100s # keep the container alive, otherwise no email is sent (emails are sent only once per minute, see BMK-80)
echo "[announce] finished at $(date)"
return 0
}
# check that the version of the docker image is greater than the latest in the registry, otherwise fail
# INPUT 1 REGISTRY, such as gitlab-registry.cern.ch/hep-benchmarks/hep-workloads
# INPUT 2 IMAGE NAME, such as atlas-sim-bmk
# INPUT 3 IMAGE TAG
function check_tag_version(){
IMAGE=docker://${DOCKER_REGISTRY}/${DOCKER_IMAGE}
REFTAG=latest
if [[ ! ${DOCKER_IMAGE_TAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then
echo "[check_tag_version] image tag $DOCKER_IMAGE_TAG does NOT respect the format v[0-9]*\.[0-9]* "
return 0
# Before building a new image $DOCKER_IMAGE:$DOCKER_IMAGE_TAG, check that the required
# tag DOCKER_IMAGE_TAG is greater than the "latest" tag in the registry, otherwise fail
# Input environment variable ${DOCKER_IMAGE}: image name for the HEP workload
# Input environment variable ${DOCKER_IMAGE_TAG}: image tag for the standalone HEP workload
# Input environment variable ${DOCKER_REGISTRY}: registry where the image should be pushed
function check_tag_version(){
IMAGE=${DOCKER_REGISTRY}/${DOCKER_IMAGE}
REFTAG=latest
if [[ ! ${DOCKER_IMAGE_TAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then
echo "[check_tag_version] image tag $DOCKER_IMAGE_TAG does NOT respect the format v[0-9]*\.[0-9]* "
return 0
fi
echo "[check_tag_version] testing image $IMAGE:$REFTAG"
res=`skopeo inspect docker://$IMAGE:$REFTAG `
echo $res
thesha=`echo $res | jq '.["Digest"]' | sed -e 's@"@@g'`
tags=`echo $res | jq '.["RepoTags"][] | select (.!="latest")' | sed -e 's@"@@g'`
echo "[check_tag_version] signature $thesha"
echo "[check_tag_version] tags $tags " | xargs
thetag=''
for tag in $tags
do
testsha=`skopeo inspect docker://$IMAGE:$tag | jq '.["Digest"]' | sed -e 's@"@@g'`
if [ "$testsha" == "$thesha" ] && [[ ${tag} =~ ^v[0-9]*\.[0-9]*$ ]]; then
thetag=$tag
echo "[check_tag_version] identified tag is $thetag"
break;
fi
echo "[check_tag_version] testing image $IMAGE:$REFTAG"
res=`skopeo inspect $IMAGE:$REFTAG `
echo $res
thesha=`echo $res | jq '.["Digest"]' | sed -e 's@"@@g'`
tags=`echo $res | jq '.["RepoTags"][] | select (.!="latest")' | sed -e 's@"@@g'`
echo "[check_tag_version] signature $thesha"
echo "[check_tag_version] tags $tags " | xargs
thetag=''
for tag in $tags
do
testsha=`skopeo inspect $IMAGE:$tag | jq '.["Digest"]' | sed -e 's@"@@g'`
if [ "$testsha" == "$thesha" ] && [[ ${tag} =~ ^v[0-9]*\.[0-9]*$ ]]; then
thetag=$tag
echo "[check_tag_version] identified tag is $thetag"
break;
fi
done
if [ "$thetag" == "" ]; then
echo "[check_tag_version] tag $REFTAG has not other tag associated"
else
if (( $(echo "${thetag/v/} >= ${DOCKER_IMAGE_TAG/v/}" | bc -l) )); then
echo "[check_tag_version] the new image tag ${DOCKER_IMAGE_TAG/v/} MUST be greater than tag $thetag associated to $IMAGE:$REFTAG"
return 1
fi
done
if [ "$thetag" == "" ]; then
echo "[check_tag_version] tag $REFTAG has not other tag associated"
else
if (( $(echo "${thetag/v/} >= ${DOCKER_IMAGE_TAG/v/}" | bc -l) )); then
echo "[check_tag_version] new image tag ${DOCKER_IMAGE_TAG/v/} MUST be greater than tag $thetag associated to $IMAGE:$REFTAG"
return 1
fi
return 0
fi
return 0
}
# function set_dir(){
# echo "[set_dir] starting at $(date)"
# [[ $2 == "" ]] && echo "HEP Workload directory undefined. Exit" && return 1