debug

.gitlab-ci.yml

+---
 stages:
-- triggers
-- test
+    - build_0
+    - build_1
+    - build_2
+    - snapshot
+    - build_standalone
+    - test
+#- publish
+#- announce
 
-#####################################################
-### ATLAS KV (a test of cvmfs functionality)
-#####################################################
+##########################
+## Templates #############
 
-job_test_kv:
-  stage: test
-  image: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
-  tags:
-      - hep-workload-gpu-docker-builder
-  before_script:
-      - export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
-      - export CVMFS_IMAGE=gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/cvmfs-image:${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
-      - docker pull ${CVMFS_IMAGE}
-      - docker run --name cvmfs_${CI_JOB_ID} -d --privileged -v ${CIENV_CVMFSVOLUME}:/cvmfs:shared ${CVMFS_IMAGE} -r atlas.cern.ch -t /tmp/traces
-  script: 
-      - sleep 1m # to give time to cvmfs to start
-      - export CIENV_CVMFSVOLUME=/scratch/cvmfs_hep/CI-JOB-${CI_JOB_ID}
-      - docker exec cvmfs_${CI_JOB_ID} cvmfs_config probe
-      - docker run --rm -v ${CIENV_CVMFSVOLUME}:/cvmfs gitlab-registry.cern.ch/hep-benchmarks/hep-workloads/atlas-kv-bmk:ci1.2 -c 2 -t 1 -e 4
-      - docker exec cvmfs_${CI_JOB_ID} cvmfs_talk -i atlas.cern.ch tracebuffer flush
-      - docker exec cvmfs_${CI_JOB_ID} python /usr/libexec/cvmfs/shrinkwrap/spec_builder.py --policy=exact /tmp/traces/cvmfs-atlas.cern.ch.trace.log /tmp/traces/cvmfs-atlas.cern.ch.spec
-      - docker cp cvmfs_${CI_JOB_ID}:/tmp/traces ${CI_PROJECT_DIR}/traces  
-  after_script:
-      - docker rm -f cvmfs_${CI_JOB_ID}
-  only:
-      variables:
-      - $CI_COMMIT_BRANCH =~ /^qa.*$/
-      - $CI_COMMIT_TAG =~ /^v.*$/
-  artifacts:
-    paths:
-      - ${CI_PROJECT_DIR}/traces
-    expire_in: 1 week
-    when: always
+# .definition_build_image_kaniko: &template_build_image_kaniko
+#     tags:
+#         - hep-workload-gpu-docker-builder
+#     image: # NB enable shared runners and do not specify a CI tag
+#         name: gitlab-registry.cern.ch/ci-tools/docker-image-builder # CERN version of the Kaniko image
+#         entrypoint: [""]
+#     script:
+#         - echo "current commit is ${CI_COMMIT_SHA:0:8}"
+#         - echo "current branch is ${CI_COMMIT_BRANCH}"
+#         - echo "current tag is ${CI_COMMIT_TAG}"
+#         - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi 
+#         - if [[ -z $CONTEXT ]]; then echo "ERROR variable CONTEXT is not defined "; exit 1; fi 
+#         - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi 
+#         - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi 
+#         - export DESTINATIONS="--destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}"
+#         - echo "DESTINATIONS $DESTINATIONS"
+#         # Prepare Kaniko configuration file
+#         - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+#         # Build and push the image from the Dockerfile at the root of the project.
+#         # To push to a specific docker tag, amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
+#         # See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
+#         - /kaniko/executor --context $CONTEXT --dockerfile $DOCKERFILE $DESTINATIONS
 
-#####################################################
-### CMS PATATRACK 
-#####################################################
+.definition_build_image: &template_build_image
+    tags:
+        - hep-workload-gpu-docker-builder
+    image: 
+        name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa # Use instead of kaniko. FIXME use a prod tag
+        entrypoint: [""]
+    script:
+        - echo "current commit is ${CI_COMMIT_SHA:0:8}"
+        - echo "current branch is ${CI_COMMIT_BRANCH}"
+        - echo "current tag is ${CI_COMMIT_TAG}"
+        - if [[ -z $DOCKERFILE ]]; then echo "ERROR variable DOCKERFILE is not defined "; exit 1; fi 
+        - if [[ -z $CONTEXT ]]; then echo "ERROR variable CONTEXT is not defined "; exit 1; fi 
+        - if [[ -z $IMAGE_NAME ]]; then echo "ERROR variable IMAGE_NAME is not defined "; exit 1; fi 
+        - if [[ -z $IMAGE_TAG ]]; then echo "ERROR variable IMAGE_TAG is not defined "; exit 1; fi 
+        - docker rmi -f $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG || echo "image $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG does not exist"
+        - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin gitlab-registry.cern.ch
+        - docker build --no-cache -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} -f $DOCKERFILE $CONTEXT
+        - dockert tag $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8} $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+        - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
+        - docker push $CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG
+        - docker rmi $CI_REGISTRY_IMAGE/$IMAGE_NAME:ci-${CI_COMMIT_BRANCH}-${CI_COMMIT_SHA:0:8}
 
-patatrack:
-  stage: triggers
-  trigger:
-    include: 
-      - local: cms/cms-patatrack-ci.yml
-    strategy: depend
-  only:
-    variables:
-      - $CI_COMMIT_BRANCH =~ /^qa.*$/
-      - $CI_COMMIT_TAG =~ /^v.*$/
+###########################################################
+# docker in docker image: to trigger other docker runs
+###########################################################
 
-#####################################################
-### LHC Simple Track
-#####################################################
+job_build_image_step0: 
+    stage: build_0
+    before_script:
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-0
+        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+    <<: *template_build_image
+    only:
+        changes:
+          - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
 
-simpletrack:
-  stage: triggers
-  trigger:
-    include: lhc/simpletrack/.simpletrack-ci.yml
-    strategy: depend
-  only:
-    variables:
-    - $CI_COMMIT_BRANCH =~ /^qa.*$/
-    - $CI_COMMIT_TAG =~ /^v.*$/
-    changes:
-    - lhc/simpletrack/Dockerfile.*
-    - lhc/simpletrack/lhc-simpletrack.*
-    - lhc/simpletrack/.simpletrack-ci.yml
+
+job_build_image_step1: 
+    stage: build_1
+    before_script:
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-1
+        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+    <<: *template_build_image
+    only:
+        changes:
+            - cms/patatrack/ci-scripts/nvidia.Dockerfile.0
+            - cms/patatrack/ci-scripts/nvidia.Dockerfile.1
+
+job_build_image_step2: 
+    stage: build_2
+    before_script:
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-2
+        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+    <<: *template_build_image
+    only:
+        changes:
+            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/cms-patatrack/*
+            - cms/patatrack/cms-patatrack/utility_scripts/*
+
+job_snapshot_cvmfs: 
+    stage: snapshot
+    tags:
+        - hep-workload-gpu-docker-builder
+    image:
+        name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
+    before_script:
+        - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+        - _before_script
+    script: 
+        - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+        - _script
+    after_script:
+        - source cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+        - _after_script
+    only:
+        variables:
+            - $CI_COMMIT_BRANCH =~ /^qa.*$/
+            - $CI_COMMIT_TAG =~ /^v.*$/
+        changes:
+            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+            - cms/patatrack/cms-patatrack/*
+            - cms/patatrack/cms-patatrack/utility_scripts/*
+    artifacts:
+        paths:
+            - ${CI_PROJECT_DIR}/traces
+            - ${CI_PROJECT_DIR}/cvmfs_export_dir_content
+            - ${CI_PROJECT_DIR}/cvmfs_export_py2-scipy_content
+            - ${CI_PROJECT_DIR}/cms/patatrack/cvmfs
+        expire_in: 1 week
+        when: always
+
+job_build_standalone_image: 
+    stage: build_standalone
+    before_script:
+        - export DOCKERFILE=$CI_PROJECT_DIR/cms/patatrack/ci-scripts/nvidia.Dockerfile.2
+        - export CONTEXT=$CI_PROJECT_DIR/cms/patatrack
+        - export IMAGE_NAME=cms/cms-patatrack-nvidia-bmk
+        - export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
+    <<: *template_build_image
+    only:
+        changes:
+            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+            - cms/patatrack/cms-patatrack/*
+            - cms/patatrack/cms-patatrack/utility_scripts/*
+
+job_test_standalone_image: 
+    stage: test
+    tags:
+        - hep-workload-gpu-docker-builder
+    image:
+        name: gitlab-registry.cern.ch/hep-benchmarks/hep-workloads-builder/dind:qa
+    script: 
+        - source cms/patatrack/ci-scripts/test_standalone_image.sh
+        - _script        
+    only:
+        variables:
+            - $CI_COMMIT_BRANCH =~ /^qa.*$/
+            - $CI_COMMIT_TAG =~ /^v.*$/
+        changes:
+            - cms/patatrack/ci-scripts/nvidia.Dockerfile.*
+            - cms/patatrack/ci-scripts/snapshot_cvmfs.sh
+            - cms/patatrack/ci-scripts/test_standalone_image.sh
+            - cms/patatrack/cms-patatrack/*
+            - cms/patatrack/cms-patatrack/utility_scripts/*
+    artifacts:
+        paths:
+            - ${CI_PROJECT_DIR}/${RESULTS_DIR}
+        expire_in: 1 week
+        when: always
\ No newline at end of file