diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8703a6ccc4479252f07b7b5e6b28656b4852b544..dd0005d1bd13d986a29908c7fd9feb8c536ccc42 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,175 +1,70 @@
----
stages:
- - build_0
- - build_1
- - build_2
- - snapshot
- - build_standalone
- - test
-#- publish
-#- announce
+- triggers
+- test
-##########################
-## Templates #############
+#####################################################
+### ATLAS KV (a test of cvmfs functionality)
+#####################################################
-# .definition_build_image_kaniko: &template_build_image_kaniko
-# tags:
-# - hep-workload-gpu-docker-builder
-# image: # NB enable shared runners and do not specify a CI tag
-# name: gitlab-registry.cern.ch/ci-tools/docker-image-builder # CERN version of the Kaniko image
-# entrypoint: [""]
-# script:
-# - echo "current commit is ${CI_COMMIT_SHA:0:8}"
-# - echo "current branch is ${CI_COMMIT_BRANCH}"
-# - echo "current tag is ${CI_COMMIT_TAG}"
-# - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi
-# - if [[ -z $CONTEXT ]]; then echo "ERROR variable CONTEXT is not defined "; exit 1; fi
-# - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi
-# - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi
-# - export DESTINATIONS="--destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}"
-# - echo "DESTINATIONS $DESTINATIONS"
-# # Prepare Kaniko configuration file
-# - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
-# # Build and push the image from the Dockerfile at the root of the project.
-# # To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
-# # See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
-# - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE $DESTINATIONS
+job_test_kv:
+ stage: test
+ image: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
+ tags:
+ - hep-workload-gpu-docker-builder
+ before_script:
+ - export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
+ - export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+ - docker pull ${CVMFS_IMAGE}
+ - docker run --name cvmfs_${CI_JOB_ID} -d --privileged -v ${CIENV_CVMFSVOLUME}:/cvmfs:shared ${CVMFS_IMAGE} -r atlas.cern.ch -t /tmp/traces
+ script:
+ - sleep 1m # to give time to cvmfs to start
+ - export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
+ - docker exec cvmfs_${CI_JOB_ID} cvmfs_config probe
+ - docker run --rm -v ${CIENV_CVMFSVOLUME}:/cvmfs gitlab-registry.cern.ch/hep-benchmarks/hep-workloads/atlas-kv-bmk:ci1.2 -c 2 -t 1 -e 4
+ - docker exec cvmfs_${CI_JOB_ID} cvmfs_talk -i atlas.cern.ch tracebuffer flush
+ - docker exec cvmfs_${CI_JOB_ID} python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact /tmp/traces/cvmfs-atlas.cern.ch.trace.log /tmp/traces/cvmfs-atlas.cern.ch.spec
+ - docker cp cvmfs_${CI_JOB_ID}:/tmp/traces ${CI_PROJECT_DIR}/traces
+ after_script:
+ - docker rm -f cvmfs_${CI_JOB_ID}
+ only:
+ variables:
+ - $CI_COMMIT_BRANCH =~ /^qa.*$/
+ - $CI_COMMIT_TAG =~ /^v.*$/
+ artifacts:
+ paths:
+ - ${CI_PROJECT_DIR}/traces
+ expire_in: 1 week
+ when: always
-.definition_build_image: &template_build_image
- tags:
- - hep-workload-gpu-docker-builder
- image:
- name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa # Use instead of kaniko. FIXME use a prod tag
- entrypoint: [""]
- script:
- - echo "current commit is ${CI_COMMIT_SHA:0:8}"
- - echo "current branch is ${CI_COMMIT_BRANCH}"
- - echo "current tag is ${CI_COMMIT_TAG}"
- - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi
- - if [[ -z $CONTEXT ]]; then echo "ERROR variable CONTEXT is not defined "; exit 1; fi
- - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi
- - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi
- - docker rmi -f $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG || echo "image $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG does not exist"
- - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch
- - docker build --no-cache -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} -f $DOCKERFILE $CONTEXT
- - dockert tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
- - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
- - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
- - docker rmi $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
+#####################################################
+### CMS PATATRACK
+#####################################################
-###########################################################
-# docker in docker image: to trigger other docker runs
-###########################################################
+patatrack:
+ stage: triggers
+ trigger:
+ include:
+ - local: cms/cms-patatrack-ci.yml
+ strategy: depend
+ only:
+ variables:
+ - $CI_COMMIT_BRANCH =~ /^qa.*$/
+ - $CI_COMMIT_TAG =~ /^v.*$/
-job_build_image_step0:
- stage: build_0
- before_script:
- - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
- - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
- - export IMAGE_NAME=cms/cms-patatrack-nvidia-0
- - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
- <<: *template_build_image
- only:
- changes:
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+#####################################################
+### LHC Simple Track
+#####################################################
-
-job_build_image_step1:
- stage: build_1
- before_script:
- - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
- - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
- - export IMAGE_NAME=cms/cms-patatrack-nvidia-1
- - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
- <<: *template_build_image
- only:
- changes:
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.1
-
-job_build_image_step2:
- stage: build_2
- before_script:
- - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
- - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
- - export IMAGE_NAME=cms/cms-patatrack-nvidia-2
- - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
- <<: *template_build_image
- only:
- changes:
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
- - cms/patatrack/cms-patatrack/*
- - cms/patatrack/cms-patatrack/utility_scripts/*
-
-job_snapshot_cvmfs:
- stage: snapshot
- tags:
- - hep-workload-gpu-docker-builder
- image:
- name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
- before_script:
- - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
- - _before_script
- script:
- - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
- - _script
- after_script:
- - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
- - _after_script
- only:
- variables:
- - $CI_COMMIT_BRANCH =~ /^qa.*$/
- - $CI_COMMIT_TAG =~ /^v.*$/
- changes:
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
- - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
- - cms/patatrack/cms-patatrack/*
- - cms/patatrack/cms-patatrack/utility_scripts/*
- artifacts:
- paths:
- - ${CI_PROJECT_DIR}/traces
- - ${CI_PROJECT_DIR}/cvmfs_export_dir_content
- - ${CI_PROJECT_DIR}/cvmfs_export_py2-scipy_content
- - ${CI_PROJECT_DIR}/cms/patatrack/cvmfs
- expire_in: 1 week
- when: always
-
-job_build_standalone_image:
- stage: build_standalone
- before_script:
- - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
- - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
- - export IMAGE_NAME=cms/cms-patatrack-nvidia-bmk
- - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
- <<: *template_build_image
- only:
- changes:
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
- - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
- - cms/patatrack/cms-patatrack/*
- - cms/patatrack/cms-patatrack/utility_scripts/*
-
-job_test_standalone_image:
- stage: test
- tags:
- - hep-workload-gpu-docker-builder
- image:
- name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
- script:
- - source cms/patatrack/ci-scripts/test_standalone_image.sh
- - _script
- only:
- variables:
- - $CI_COMMIT_BRANCH =~ /^qa.*$/
- - $CI_COMMIT_TAG =~ /^v.*$/
- changes:
- - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
- - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
- - cms/patatrack/ci-scripts/test_standalone_image.sh
- - cms/patatrack/cms-patatrack/*
- - cms/patatrack/cms-patatrack/utility_scripts/*
- artifacts:
- paths:
- - ${CI_PROJECT_DIR}/${RESULTS_DIR}
- expire_in: 1 week
- when: always
\ No newline at end of file
+simpletrack:
+ stage: triggers
+ trigger:
+ include: lhc/simpletrack/.simpletrack-ci.yml
+ strategy: depend
+ only:
+ variables:
+ - $CI_COMMIT_BRANCH =~ /^qa.*$/
+ - $CI_COMMIT_TAG =~ /^v.*$/
+ changes:
+ - lhc/simpletrack/Dockerfile.*
+ - lhc/simpletrack/lhc-simpletrack.*
+ - lhc/simpletrack/.simpletrack-ci.yml
diff --git a/cms/cms-patatrack-ci.yml b/cms/cms-patatrack-ci.yml
index 8703a6ccc4479252f07b7b5e6b28656b4852b544..ab394abe9287933378a2e3967e616557b2c41b26 100644
--- a/cms/cms-patatrack-ci.yml
+++ b/cms/cms-patatrack-ci.yml
@@ -52,7 +52,7 @@ stages:
- docker rmi -f $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG || echo "image $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG does not exist"
- echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch
- docker build --no-cache -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} -f $DOCKERFILE $CONTEXT
- - dockert tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+ - docker tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
- docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
- docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
- docker rmi $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
diff --git a/cms/patatrack/ci-scripts/build_2.sh b/cms/patatrack/ci-scripts/build_2.sh
index 8d3d64f84fc56df5869246e0316c623b72ad8db7..8a78e56332ccd274d0e5635f11b68433ebc6cd38 100755
--- a/cms/patatrack/ci-scripts/build_2.sh
+++ b/cms/patatrack/ci-scripts/build_2.sh
@@ -4,24 +4,29 @@ set -x
set -e
# First move all folders in the right place
+date
mv /stage/cvmfs /cvmfs
+
+date
mv /stage/cms-patatrack /bmk/./cms-patatrack
# Make only readable
+date
chmod -R 555 /cvmfs
# FIXME This checksum takes a lot of time.
# Commenting it. Can be substituted by a checksum using cvmfs utilities
#tar -cf /tmp/cvmfs_checksum.tar /cvmfs && md5sum /tmp/cvmfs_checksum.tar | cut -f1 -d" " > /tmp/cvmfs_checksum && rm /tmp/cvmfs_checksum.tar
-touch /tmp/cvmfs_checksum
# Checksum code in orchestrator dir.
# This MUST happen before linking the data dir
# otherwise will take a lot of time to tar
+date
tar -cf /tmp/bmk_checksum.tar /bmk && md5sum /tmp/bmk_checksum.tar | cut -f1 -d" " >/tmp/bmk_checksum && rm /tmp/bmk_checksum.tar #FIXME
# The data dir has already a checksum in /tmp/bmkdata_checksum
# generated in nvidia.Dockerfile.1
+date
if [ ! -d /bmk/./cms-patatrack/data ]; then
mkdir /bmk/./cms-patatrack/data
fi
@@ -29,6 +34,7 @@ for file in $(ls /bmk/data); do
ln -sf /bmk/data/$file /bmk/./cms-patatrack/data/$file
done
+date
cvmfs_checksum=$(cat /tmp/cvmfs_checksum || echo "NotAvailable")
bmkdata_checksum=$(cat /tmp/bmkdata_checksum || echo "NotAvailable")
bmk_checksum=$(cat /tmp/bmk_checksum || echo "NotAvailable")
diff --git a/cms/patatrack/ci-scripts/snapshot_cvmfs.sh b/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
index 0f9caf4e5f1f0bb1a02b45ac284a7fe37d627f86..4cf653363c44ddf57f4284270f001e2b6a97cb42 100644
--- a/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+++ b/cms/patatrack/ci-scripts/snapshot_cvmfs.sh
@@ -11,7 +11,6 @@ function _before_script() {
function _script() {
sleep 1m # to give time to cvmfs to start
-
echo "CVMFS_EXPORT_DIR is $CVMFS_EXPORT_DIR"
# check cvmfs is running
docker exec cvmfs_${CI_JOB_ID} cvmfs_config probe