From 911071fb4ff99be0720876dc0e38a552c81328a8 Mon Sep 17 00:00:00 2001 From: Matthew Joseph Basso <matthew.joseph.basso@cern.ch> Date: Fri, 10 Nov 2023 17:49:12 -0800 Subject: [PATCH 1/2] Update CI to make use of kaniko for building docker images --- .gitlab-ci.yml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 871165c92..56e203f62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,24 +73,31 @@ skopeo-helper: .cmake_template: stage: build - tags: - - docker-image-build - script: - - ignore variables: + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}:latest # TO: ${CI_REGISTRY_IMAGE}:sha-${CI_COMMIT_SHORT_SHA} BUILD_ARG_1: ANALYSISBASE_IMAGE=${ANALYSISBASE_IMAGE} BUILD_ARG_2: CODE_DIR=${CODE_DIR} - # BUILD_ARG_3: CMAKE_CXX_FLAGS="<put cxx flags here>" + BUILD_ARG_3: CMAKE_CXX_FLAGS= DOCKER_FILE: docker/Dockerfile GIT_STRATEGY: fetch + image: + name: gitlab-registry.cern.ch/ci-tools/docker-image-builder + entrypoint: [""] + before_script: + # Prepare Kaniko configuration file + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + script: + # Build and push the image from the Dockerfile at the root of the project. + - export IFS='' + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $DOCKER_FILE --destination $IMAGE_DESTINATION --build-arg $BUILD_ARG_1 --build-arg $BUILD_ARG_2 --build-arg $BUILD_ARG_3 # nominal build based on AnalysisBase image from cmake_template cmake: extends: - .cmake_template variables: - TO: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA} + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA} when: always # same as above, but for tagged commits @@ -98,7 +105,7 @@ cmake_tag: extends: - cmake variables: - TO: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG} + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG} only: - tags @@ -107,7 +114,7 @@ cmake_manual: extends: - cmake variables: - TO: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} when: manual # Same as above, but with most recent AnalysisBase version. Only run @@ -116,7 +123,7 @@ cmake_latest: extends: - .cmake_template variables: - TO: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA}-latest + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA}-latest BUILD_ARG_1: ANALYSISBASE_IMAGE=${ANALYSISBASE_IMAGE_LATEST} only: - master @@ -126,7 +133,7 @@ cmake_latest: .cmake_debug: extends: .cmake_template variables: - TO: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA}-debug + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA}-debug BUILD_ARG_3: CMAKE_CXX_FLAGS="-D_DEBUG_=1" # Same as above, but aborts compilation after the first warning. This job is executed in the test stage and only if the normal compilation succeeded. The image is deleted later. @@ -135,7 +142,7 @@ compiler_warnings: stage: test needs: ["cmake"] variables: - TO: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA}-warning + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/debug:sha-${CI_COMMIT_SHORT_SHA}-warning # BUILD_ARG_3: CMAKE_CXX_FLAGS="-Wfatal-errors" BUILD_ARG_3: CMAKE_CXX_FLAGS="-Werror -Wno-unused-function" @@ -144,7 +151,7 @@ cmake_standalone: extends: - .cmake_template variables: - TO: ${CI_REGISTRY_IMAGE}/debug-standalone:sha-${CI_COMMIT_SHORT_SHA} + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/debug-standalone:sha-${CI_COMMIT_SHORT_SHA} BUILD_ARG_1: STANDALONE_BASE=${CI_REGISTRY_IMAGE}/standalone-base:${ROOT_TAG} DOCKER_FILE: docker/Dockerfile_standalone @@ -153,7 +160,7 @@ cmake_standalone_tag: extends: - cmake_standalone variables: - TO: ${CI_REGISTRY_IMAGE}/standalone:${CI_COMMIT_TAG} + IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/standalone:${CI_COMMIT_TAG} only: - tags @@ -423,3 +430,4 @@ tag_standalone_master: when: always only: - master + \ No newline at end of file -- GitLab From 0e1af85857f3cadef66410421713fba6cca27355 Mon Sep 17 00:00:00 2001 From: Matthew Joseph Basso <matthew.joseph.basso@cern.ch> Date: Fri, 10 Nov 2023 17:51:45 -0800 Subject: [PATCH 2/2] Create full path (including any intermediate directories) --- QFramework/test/CAFTests.sh | 38 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/QFramework/test/CAFTests.sh b/QFramework/test/CAFTests.sh index b84348e98..fda6ca5fe 100644 --- a/QFramework/test/CAFTests.sh +++ b/QFramework/test/CAFTests.sh @@ -10,29 +10,29 @@ CAFCopyPlottingFiles(){ local toPath="$toPathRaw/plotting-artifacts" if [ -d "$toPath" ]; then - rm -r "$toPath" + rm -r "$toPath" fi - mkdir "$toPath" + mkdir -p "$toPath" if [ -f "$fromPath/messageForGitlab.txt" ]; then - # if messageForGitlab.txt exists, then plots don't need to be duplicated - mv "$fromPath/messageForGitlab.txt" "$toPath/README.txt" + # if messageForGitlab.txt exists, then plots don't need to be duplicated + mv "$fromPath/messageForGitlab.txt" "$toPath/README.txt" else - # if messageForGitlab.txt does not exist, upload all plots - if [ -d "$fromPath/dotCFiles" ]; then - mv "$fromPath/dotCFiles" "$toPath/referencePlots" - fi - if [ -d "$fromPath/pdfFiles" ]; then - mv "$fromPath/pdfFiles" "$toPath/referencePlotsPDF" - fi - if [ "$fromPath" != "$toPath" ]; then - if [ -f "$fromPath/listOfHashes.txt" ]; then - mv "$fromPath/listOfHashes.txt" "$toPath" - fi - if [ -f "$fromPath/fullDiff.txt" ]; then - mv "$fromPath/fullDiff.txt" "$toPath" - fi - fi + # if messageForGitlab.txt does not exist, upload all plots + if [ -d "$fromPath/dotCFiles" ]; then + mv "$fromPath/dotCFiles" "$toPath/referencePlots" + fi + if [ -d "$fromPath/pdfFiles" ]; then + mv "$fromPath/pdfFiles" "$toPath/referencePlotsPDF" + fi + if [ "$fromPath" != "$toPath" ]; then + if [ -f "$fromPath/listOfHashes.txt" ]; then + mv "$fromPath/listOfHashes.txt" "$toPath" + fi + if [ -f "$fromPath/fullDiff.txt" ]; then + mv "$fromPath/fullDiff.txt" "$toPath" + fi + fi fi } -- GitLab