diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 14e8020d85b7d37480e53278a06088bacb0f0c64..ac3af51007c3efe05f08527788d9855e314174c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,17 +2,37 @@ stages: - check - build -check_changed_files: +# Triggered when the latest commit contains changes to cms/cvmfs/standalone +# Triggers the pipeline for every folder containing changes +check_changes_pipeline: stage: check - only: - variables: - - $CI_PIPELINE_SOURCE == 'push' - changes: [cc7-cms/*, cc7-cvmfs/*, slc6-cms/*, slc6-cvmfs/*, slc5-cms/*] + rules: + - changes: [cc7-cms/*, cc7-cvmfs/*, slc6-cms/*, slc6-cvmfs/*, slc5-cms/*, standalone/*] + if: $CI_PIPELINE_SOURCE == 'push' # Trigger if push (for test on branches) or merge event (for production on master) tags: - - docker + - docker # tags docker since those runners have git installed (needed by check.sh) script: + - echo "Pipeline source - $CI_PIPELINE_SOURCE" - ./check.sh +# Test template for all images +# Tagged $DATE-$CI_COMMIT_SHORT_SHA +# Pushed to gitlab registry under name test +.test_all_template: &test_all + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - export DATE=$(date +"%Y-%m-%d") + - if [[ $IMAGE_NAME == "standalone" ]]; then REPO_NAME=standalone; else REPO_NAME=$IMAGE_NAME; fi + - echo "Pipeline source - $CI_PIPELINE_SOURCE" + - echo "Building test image - $IMAGE_NAME" + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}, \"$DOCKER_REGISTRY\":{\"auth\":\"${DOCKER_AUTH}\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor --context $CI_PROJECT_DIR/$REPO_NAME --dockerfile $CI_PROJECT_DIR/$REPO_NAME/Dockerfile --destination $CI_REGISTRY_IMAGE/test:$IMAGE_NAME-$DATE-$CI_COMMIT_SHORT_SHA --build-arg=BUILD_DATE="$DATE" --build-arg=VERSION="$DATE" --build-arg=VCS_URL="$CI_REPOSITORY_URL" --build-arg=VCS_REF="$CI_COMMIT_SHORT_SHA" + +# Build template for cms/cvmfs images +# Tagged $DATE-$CI_COMMIT_SHORT_SHA and latest +# Pushed to gitlab registry and Docker Hub (only latest) .build_cms_cvmfs_template: &build_cms_cvmfs image: name: gcr.io/kaniko-project/executor:debug @@ -21,23 +41,36 @@ check_changed_files: - export DATE=$(date +"%Y-%m-%d") - echo "Pipeline source - $CI_PIPELINE_SOURCE" - echo "Building image - $IMAGE_NAME" - - export DOCKER_AUTH="$(echo -n $DOCKER_USER:$DOCKER_PASS | base64)" + - export DOCKER_AUTH="$(echo -n $DOCKER_USER:$DOCKER_PASS | base64)" # https://github.com/GoogleContainerTools/kaniko#pushing-to-docker-hub - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}, \"$DOCKER_REGISTRY\":{\"auth\":\"${DOCKER_AUTH}\"}}}" > /kaniko/.docker/config.json - /kaniko/executor --context $CI_PROJECT_DIR/$IMAGE_NAME --dockerfile $CI_PROJECT_DIR/$IMAGE_NAME/Dockerfile --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$DATE-$CI_COMMIT_SHORT_SHA --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest --destination $DOCKER_GROUP/$IMAGE_NAME:latest --build-arg=BUILD_DATE="$DATE" --build-arg=VERSION="$DATE" --build-arg=VCS_URL="$CI_REPOSITORY_URL" --build-arg=VCS_REF="$CI_COMMIT_SHORT_SHA" +# Build template for standalone images +# Tagged $DATE-$CI_COMMIT_SHORT_SHA +# Pushed to gitlab registry .build_standalone_template: &build_standalone image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] script: - export DATE=$(date +"%Y-%m-%d") - - export RELEASE_CAP=$(echo $RELEASE | tr '[:lower:]' '[:upper:]') + - export RELEASE_CAP=$(echo $RELEASE | tr '[:lower:]' '[:upper:]') # Release is given in lower case to please docker image naming rules, however Dockerfile expects the release in upper case. - echo "Pipeline source - $CI_PIPELINE_SOURCE" - echo "Building image - $IMAGE_NAME" - printf "Release - $RELEASE_CAP\nScram Arch - $SCRAM_ARCH\nBase image - $BASE_IMAGE\n" - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - /kaniko/executor --context $CI_PROJECT_DIR/$REPO_NAME --dockerfile $CI_PROJECT_DIR/$REPO_NAME/Dockerfile --destination $CI_REGISTRY_IMAGE/$IMAGE_NAME:$DATE-$CI_COMMIT_SHORT_SHA --build-arg=BUILD_DATE="$DATE" --build-arg=VERSION="$DATE" --build-arg=VCS_URL="$CI_REPOSITORY_URL" --build-arg=VCS_REF="$CI_COMMIT_SHORT_SHA" --build-arg BASEIMAGE="$BASE_IMAGE" --build-arg SCRAM_ARCH="$SCRAM_ARCH" --build-arg CMSSW_VERSION="$RELEASE_CAP" +test_all_pipeline: + <<: *test_all + stage: build + only: + variables: + - $IMAGE_NAME + - $TEST + except: + refs: + - master build_cms_cvmfs_pipeline: <<: *build_cms_cvmfs @@ -45,6 +78,8 @@ build_cms_cvmfs_pipeline: only: variables: - $IMAGE_NAME + refs: + - master variables: DOCKER_GROUP: cmscloud DOCKER_REGISTRY: https://index.docker.io/v1/ @@ -55,9 +90,13 @@ build_standalone_pipeline: only: variables: - $RELEASE && $SCRAM_ARCH && $OS + refs: + - master tags: - docker-privileged-xl variables: IMAGE_NAME: $RELEASE-$SCRAM_ARCH REPO_NAME: standalone - BASE_IMAGE: "clelange/$OS-cms:latest" \ No newline at end of file + BASE_IMAGE: "$CI_REGISTRY_IMAGE/$OS-cms:latest" + # Necessary variables to be passed to Dockerfile for building a standalone image is CMSSW_VERSION=$RELEASE, + # SCRAM_ARCH=$SCRAM_ARCH and BASEIMAGE="gitlab-registry.cern.ch/cms-cloud/cmssw-docker/$OS-cms:latest" \ No newline at end of file diff --git a/check.sh b/check.sh index 4fb6f2b8ef7b588da97c8496fe34ce978a9165aa..52964f76cde8d39d7c6a7a67851d290d9d02715f 100755 --- a/check.sh +++ b/check.sh @@ -1,13 +1,15 @@ #!/bin/bash -files=$(git diff-tree --no-commit-id --name-only -r $CI_COMMIT_SHA) +files=$(git diff-tree --no-commit-id --name-only -r -m $CI_COMMIT_SHA) # Get path to files changed in commit for file in $files; do - dir=${file%/*} + dir=${file%/*} # Get directory from path echo "Changes in $dir directory" - if [[ "$dir" =~ ^(cc7-cms|cc7-cvmfs|slc6-cms|slc6-cvmfs|slc5-cms)$ ]]; then + if [[ "$dir" =~ ^(cc7-cms|cc7-cvmfs|slc6-cms|slc6-cvmfs|slc5-cms)$ ]] && [[ -z $CI_COMMIT_BRANCH ]]; then # $CI_COMMIT_BRANCH is the commit branch name. Present only when building branches. + # For production on master + echo "On master" echo "Triggered build of $dir" curl --request POST \ @@ -15,5 +17,19 @@ for file in $files; do --form ref=$CI_DEFAULT_BRANCH \ --form "variables[IMAGE_NAME]=$dir" \ https://gitlab.cern.ch/api/v4/projects/$CI_PROJECT_ID/trigger/pipeline - fi + + elif [[ "$dir" =~ ^(cc7-cms|cc7-cvmfs|slc6-cms|slc6-cvmfs|slc5-cms|standalone)$ ]] && [[ ! -z $CI_COMMIT_BRANCH ]]; then + # For testing on branches + + echo "On branch $CI_COMMIT_BRANCH" + echo "Triggered build of $dir" + + curl --request POST \ + --form token=$CI_JOB_TOKEN \ + --form ref=$CI_COMMIT_BRANCH \ + --form "variables[IMAGE_NAME]=$dir" \ + --form "variables[TEST]=TEST" \ + https://gitlab.cern.ch/api/v4/projects/$CI_PROJECT_ID/trigger/pipeline + + fi done