diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dd0005d1bd13d986a29908c7fd9feb8c536ccc42..8703a6ccc4479252f07b7b5e6b28656b4852b544 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,70 +1,175 @@
+---
stages:
-- triggers
-- test
+ - build_0
+ - build_1
+ - build_2
+ - snapshot
+ - build_standalone
+ - test
+#- publish
+#- announce
-#####################################################
-### ATLAS KV (a test of cvmfs functionality)
-#####################################################
+##########################
+## Templates #############
-job_test_kv:
- stage: test
- image: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
- tags:
- - hep-workload-gpu-docker-builder
- before_script:
- - export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
- - export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
- - docker pull ${CVMFS_IMAGE}
- - docker run --name cvmfs_${CI_JOB_ID} -d --privileged -v ${CIENV_CVMFSVOLUME}:/cvmfs:shared ${CVMFS_IMAGE} -r atlas.cern.ch -t /tmp/traces
- script:
- - sleep 1m # to give time to cvmfs to start
- - export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
- - docker exec cvmfs_${CI_JOB_ID} cvmfs_config probe
- - docker run --rm -v ${CIENV_CVMFSVOLUME}:/cvmfs gitlab-registry.cern.ch/hep-benchmarks/hep-workloads/atlas-kv-bmk:ci1.2 -c 2 -t 1 -e 4
- - docker exec cvmfs_${CI_JOB_ID} cvmfs_talk -i atlas.cern.ch tracebuffer flush
- - docker exec cvmfs_${CI_JOB_ID} python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact /tmp/traces/cvmfs-atlas.cern.ch.trace.log /tmp/traces/cvmfs-atlas.cern.ch.spec
- - docker cp cvmfs_${CI_JOB_ID}:/tmp/traces ${CI_PROJECT_DIR}/traces
- after_script:
- - docker rm -f cvmfs_${CI_JOB_ID}
- only:
- variables:
- - $CI_COMMIT_BRANCH =~ /^qa.*$/
- - $CI_COMMIT_TAG =~ /^v.*$/
- artifacts:
- paths:
- - ${CI_PROJECT_DIR}/traces
- expire_in: 1 week
- when: always
+# .definition_build_image_kaniko: &template_build_image_kaniko
+# tags:
+# - hep-workload-gpu-docker-builder
+# image: # NB enable shared runners and do not specify a CI tag
+# name: gitlab-registry.cern.ch/ci-tools/docker-image-builder # CERN version of the Kaniko image
+# entrypoint: [""]
+# script:
+# - echo "current commit is ${CI_COMMIT_SHA:0:8}"
+# - echo "current branch is ${CI_COMMIT_BRANCH}"
+# - echo "current tag is ${CI_COMMIT_TAG}"
+# - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi
+# - if [[ -z $CONTEXT ]]; then echo "ERROR variable CONTEXT is not defined "; exit 1; fi
+# - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi
+# - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi
+# - export DESTINATIONS="--destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}"
+# - echo "DESTINATIONS $DESTINATIONS"
+# # Prepare Kaniko configuration file
+# - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+# # Build and push the image from the Dockerfile at the root of the project.
+# # To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
+# # See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
+# - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE $DESTINATIONS
-#####################################################
-### CMS PATATRACK
-#####################################################
+.definition_build_image: &template_build_image
+ tags:
+ - hep-workload-gpu-docker-builder
+ image:
+ name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa # Use instead of kaniko. FIXME use a prod tag
+ entrypoint: [""]
+ script:
+ - echo "current commit is ${CI_COMMIT_SHA:0:8}"
+ - echo "current branch is ${CI_COMMIT_BRANCH}"
+ - echo "current tag is ${CI_COMMIT_TAG}"
+ - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi
+ - if [[ -z $CONTEXT ]]; then echo "ERROR variable CONTEXT is not defined "; exit 1; fi
+ - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi
+ - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi
+ - docker rmi -f $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG || echo "image $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG does not exist"
+ - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch
+ - docker build --no-cache -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} -f $DOCKERFILE $CONTEXT
+ - dockert tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+ - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
+ - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+ - docker rmi $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
-patatrack:
- stage: triggers
- trigger:
- include:
- - local: cms/cms-patatrack-ci.yml
- strategy: depend
- only:
- variables:
- - $CI_COMMIT_BRANCH =~ /^qa.*$/
- - $CI_COMMIT_TAG =~ /^v.*$/
+###########################################################
+# docker in docker image: to trigger other docker runs
+###########################################################
-#####################################################
-### LHC Simple Track
-#####################################################
+job_build_image_step0:
+ stage: build_0
+ before_script:
+ - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+ - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+ - export IMAGE_NAME=cms/cms-patatrack-nvidia-0
+ - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+ <<: *template_build_image
+ only:
+ changes:
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
-simpletrack:
- stage: triggers
- trigger:
- include: lhc/simpletrack/.simpletrack-ci.yml
- strategy: depend
- only:
- variables:
- - $CI_COMMIT_BRANCH =~ /^qa.*$/
- - $CI_COMMIT_TAG =~ /^v.*$/
- changes:
- - lhc/simpletrack/Dockerfile.*
- - lhc/simpletrack/lhc-simpletrack.*
- - lhc/simpletrack/.simpletrack-ci.yml
+
+job_build_image_step1:
+ stage: build_1
+ before_script:
+ - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+ - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+ - export IMAGE_NAME=cms/cms-patatrack-nvidia-1
+ - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+ <<: *template_build_image
+ only:
+ changes:
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+
+job_build_image_step2:
+ stage: build_2
+ before_script:
+ - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+ - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+ - export IMAGE_NAME=cms/cms-patatrack-nvidia-2
+ - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+ <<: *template_build_image
+ only:
+ changes:
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+ - cms/patatrack/cms-patatrack/*
+ - cms/patatrack/cms-patatrack/utility_scripts/*
+
+job_snapshot_cvmfs:
+ stage: snapshot
+ tags:
+ - hep-workload-gpu-docker-builder
+ image:
+ name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
+ before_script:
+ - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+ - _before_script
+ script:
+ - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+ - _script
+ after_script:
+ - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+ - _after_script
+ only:
+ variables:
+ - $CI_COMMIT_BRANCH =~ /^qa.*$/
+ - $CI_COMMIT_TAG =~ /^v.*$/
+ changes:
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+ - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+ - cms/patatrack/cms-patatrack/*
+ - cms/patatrack/cms-patatrack/utility_scripts/*
+ artifacts:
+ paths:
+ - ${CI_PROJECT_DIR}/traces
+ - ${CI_PROJECT_DIR}/cvmfs_export_dir_content
+ - ${CI_PROJECT_DIR}/cvmfs_export_py2-scipy_content
+ - ${CI_PROJECT_DIR}/cms/patatrack/cvmfs
+ expire_in: 1 week
+ when: always
+
+job_build_standalone_image:
+ stage: build_standalone
+ before_script:
+ - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+ - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+ - export IMAGE_NAME=cms/cms-patatrack-nvidia-bmk
+ - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+ <<: *template_build_image
+ only:
+ changes:
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+ - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+ - cms/patatrack/cms-patatrack/*
+ - cms/patatrack/cms-patatrack/utility_scripts/*
+
+job_test_standalone_image:
+ stage: test
+ tags:
+ - hep-workload-gpu-docker-builder
+ image:
+ name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
+ script:
+ - source cms/patatrack/ci-scripts/test_standalone_image.sh
+ - _script
+ only:
+ variables:
+ - $CI_COMMIT_BRANCH =~ /^qa.*$/
+ - $CI_COMMIT_TAG =~ /^v.*$/
+ changes:
+ - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+ - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+ - cms/patatrack/ci-scripts/test_standalone_image.sh
+ - cms/patatrack/cms-patatrack/*
+ - cms/patatrack/cms-patatrack/utility_scripts/*
+ artifacts:
+ paths:
+ - ${CI_PROJECT_DIR}/${RESULTS_DIR}
+ expire_in: 1 week
+ when: always
\ No newline at end of file