CI add comments and test buidling

.gitlab-ci.yml

@@ -2,17 +2,37 @@ stages:
   - check
   - build
 
-check_changed_files:
+# Triggered when the latest commit contains changes to cms/cvmfs/standalone
+# Triggers the pipeline for every folder containing changes
+check_changes_pipeline:
   stage: check
-  only:
-    variables:
-      - $CI_PIPELINE_SOURCE == 'push'
-    changes: [cc7-cms/*, cc7-cvmfs/*, slc6-cms/*, slc6-cvmfs/*, slc5-cms/*]
+  rules:
+    - changes: [cc7-cms/*, cc7-cvmfs/*, slc6-cms/*, slc6-cvmfs/*, slc5-cms/*, standalone/*]
+      if: $CI_PIPELINE_SOURCE == 'push' # Trigger if push (for test on branches) or merge event (for production on master)
   tags:
-    - docker
+      - docker # tags docker since those runners have git installed (needed by check.sh)
   script:
+    - echo "Pipeline source - $CI_PIPELINE_SOURCE"
     - ./check.sh
 
+# Test template for all images
+# Tagged $DATE-$CI_COMMIT_SHORT_SHA
+# Pushed to gitlab registry under name test
+.test_all_template: &test_all
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  script:
+    - export DATE=$(date +"%Y-%m-%d")
+    - if [[ $IMAGE_NAME == "standalone" ]]; then REPO_NAME=standalone; else REPO_NAME=$IMAGE_NAME; fi
+    - echo "Pipeline source - $CI_PIPELINE_SOURCE"
+    - echo "Building test image - $IMAGE_NAME"
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}, \"$DOCKER_REGISTRY\":{\"auth\":\"${DOCKER_AUTH}\"}}}" > /kaniko/.docker/config.json
+    - /kaniko/executor --context $CI_PROJECT_DIR/$REPO_NAME --dockerfile $CI_PROJECT_DIR/$REPO_NAME/Dockerfile --destination $CI_REGISTRY_IMAGE/test:$IMAGE_NAME-$DATE-$CI_COMMIT_SHORT_SHA --build-arg=BUILD_DATE="$DATE" --build-arg=VERSION="$DATE" --build-arg=VCS_URL="$CI_REPOSITORY_URL" --build-arg=VCS_REF="$CI_COMMIT_SHORT_SHA"
+
+# Build template for cms/cvmfs images
+# Tagged $DATE-$CI_COMMIT_SHORT_SHA and latest
+# Pushed to gitlab registry and Docker Hub (only latest) 
 .build_cms_cvmfs_template: &build_cms_cvmfs
   image:
     name: gcr.io/kaniko-project/executor:debug
@@ -21,23 +41,36 @@ check_changed_files:
     - export DATE=$(date +"%Y-%m-%d")
     - echo "Pipeline source - $CI_PIPELINE_SOURCE"
     - echo "Building image - $IMAGE_NAME"
-    - export DOCKER_AUTH="$(echo -n $DOCKER_USER:$DOCKER_PASS | base64)"
+    - export DOCKER_AUTH="$(echo -n $DOCKER_USER:$DOCKER_PASS | base64)" # https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}, \"$DOCKER_REGISTRY\":{\"auth\":\"${DOCKER_AUTH}\"}}}" > /kaniko/.docker/config.json
     - /kaniko/executor --context $CI_PROJECT_DIR/$IMAGE_NAME --dockerfile $CI_PROJECT_DIR/$IMAGE_NAME/Dockerfile --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$DATE-$CI_COMMIT_SHORT_SHA --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest --destination  $DOCKER_GROUP/$IMAGE_NAME:latest --build-arg=BUILD_DATE="$DATE" --build-arg=VERSION="$DATE" --build-arg=VCS_URL="$CI_REPOSITORY_URL" --build-arg=VCS_REF="$CI_COMMIT_SHORT_SHA"
 
+# Build template for standalone images
+# Tagged $DATE-$CI_COMMIT_SHORT_SHA
+# Pushed to gitlab registry
 .build_standalone_template: &build_standalone
   image:
     name: gcr.io/kaniko-project/executor:debug
     entrypoint: [""]
   script:
     - export DATE=$(date +"%Y-%m-%d")
-    - export RELEASE_CAP=$(echo $RELEASE | tr '[:lower:]' '[:upper:]')
+    - export RELEASE_CAP=$(echo $RELEASE | tr '[:lower:]' '[:upper:]') # Release is given in lower case to please docker image naming rules, however Dockerfile expects the release in upper case.
     - echo "Pipeline source - $CI_PIPELINE_SOURCE"
     - echo "Building image - $IMAGE_NAME"
     - printf "Release - $RELEASE_CAP\nScram Arch - $SCRAM_ARCH\nBase image - $BASE_IMAGE\n"
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
     - /kaniko/executor --context $CI_PROJECT_DIR/$REPO_NAME --dockerfile $CI_PROJECT_DIR/$REPO_NAME/Dockerfile --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$DATE-$CI_COMMIT_SHORT_SHA --build-arg=BUILD_DATE="$DATE" --build-arg=VERSION="$DATE" --build-arg=VCS_URL="$CI_REPOSITORY_URL" --build-arg=VCS_REF="$CI_COMMIT_SHORT_SHA" --build-arg BASEIMAGE="$BASE_IMAGE" --build-arg SCRAM_ARCH="$SCRAM_ARCH" --build-arg CMSSW_VERSION="$RELEASE_CAP"
 
+test_all_pipeline:
+  <<: *test_all
+  stage: build
+  only:
+    variables:
+      - $IMAGE_NAME
+      - $TEST
+  except:
+    refs:
+      - master
 
 build_cms_cvmfs_pipeline:
   <<: *build_cms_cvmfs
@@ -45,6 +78,8 @@ build_cms_cvmfs_pipeline:
   only:
     variables:
       - $IMAGE_NAME
+    refs:
+      - master
   variables:
     DOCKER_GROUP: cmscloud
     DOCKER_REGISTRY: https://index.docker.io/v1/
@@ -55,9 +90,13 @@ build_standalone_pipeline:
   only:
     variables:
       - $RELEASE && $SCRAM_ARCH && $OS
+    refs:
+      - master
   tags:
     - docker-privileged-xl
   variables:
     IMAGE_NAME: $RELEASE-$SCRAM_ARCH
     REPO_NAME: standalone
-    BASE_IMAGE: "clelange/$OS-cms:latest"
\ No newline at end of file
+    BASE_IMAGE: "$CI_REGISTRY_IMAGE/$OS-cms:latest"
+  # Necessary variables to be passed to Dockerfile for building a standalone image is CMSSW_VERSION=$RELEASE,
+  # SCRAM_ARCH=$SCRAM_ARCH and BASEIMAGE="gitlab-registry.cern.ch/cms-cloud/cmssw-docker/$OS-cms:latest"
\ No newline at end of file

check.sh

 #!/bin/bash
 
-files=$(git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA)
+files=$(git diff-tree --no-commit-id --name-only -r -m $CI_COMMIT_SHA) # Get path to files changed in commit
 for file in $files; do
-  dir=${file%/*}
+  dir=${file%/*} # Get directory from path
 
   echo "Changes in $dir directory"
 
-  if [[ "$dir" =~ ^(cc7-cms|cc7-cvmfs|slc6-cms|slc6-cvmfs|slc5-cms)$ ]]; then
+  if [[ "$dir" =~ ^(cc7-cms|cc7-cvmfs|slc6-cms|slc6-cvmfs|slc5-cms)$ ]] && [[ -z $CI_COMMIT_BRANCH ]]; then # $CI_COMMIT_BRANCH is the commit branch name. Present only when building branches.
+    # For production on master
 
+    echo "On master"
     echo "Triggered build of $dir"
 
     curl --request POST \
@@ -15,5 +17,19 @@ for file in $files; do
     --form ref=$CI_DEFAULT_BRANCH \
     --form "variables[IMAGE_NAME]=$dir" \
     https://gitlab.cern.ch/api/v4/projects/$CI_PROJECT_ID/trigger/pipeline
-    fi
+
+  elif [[ "$dir" =~ ^(cc7-cms|cc7-cvmfs|slc6-cms|slc6-cvmfs|slc5-cms|standalone)$ ]] && [[ ! -z $CI_COMMIT_BRANCH ]]; then
+    # For testing on branches
+
+    echo "On branch $CI_COMMIT_BRANCH"
+    echo "Triggered build of $dir"
+
+    curl --request POST \
+    --form token=$CI_JOB_TOKEN \
+    --form ref=$CI_COMMIT_BRANCH \
+    --form "variables[IMAGE_NAME]=$dir" \
+    --form "variables[TEST]=TEST" \
+    https://gitlab.cern.ch/api/v4/projects/$CI_PROJECT_ID/trigger/pipeline
+  
+  fi
 done