From 28ca9d4ba197744a2f6100d2afffb0f5bb4ebed2 Mon Sep 17 00:00:00 2001 From: Tom McCauley Date: Thu, 8 Sep 2022 12:43:05 +0100 Subject: [PATCH 1/4] standalone build * rm standalone Makefile and add build rules to top-level Makefile * fix build error re: ln command in Dockerfile * closes #59 --- Makefile | 57 ++++++++++++++++++++++++++++++ README.md | 21 +++++------ standalone/Dockerfile | 6 ++-- standalone/Makefile | 81 ------------------------------------------- 4 files changed, 70 insertions(+), 95 deletions(-) delete mode 100644 standalone/Makefile diff --git a/Makefile b/Makefile index d0d9483..332dd65 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,40 @@ build: docker_build output # Build and push Docker image release: docker_build docker_push output +# Build a Docker standalone image +standalone: docker_build_standalone output + +# Build and push a Docker standalone image +standalone_release: docker_build_standalone docker_push_standalone output + +# If making a standalone image (and release) these need to be set (and re-set) +ifneq (,$(findstring standalone, $(MAKECMDGOALS))) +# variables need to be overwritten +SCRAM_ARCH=slc6_amd64_gcc481 +CMSSW_VERSION=CMSSW_7_1_25_patch5 +BASEIMAGE=gitlab-registry.cern.ch/cms-cloud/cmssw-docker/slc6-cms + +# function to return base version +get_version = $(strip $(subst CMSSW_, ,$1)) +BASE_VERSION = $(call get_version,$(CMSSW_VERSION)) + +DOCKER_IMAGE = cmscloud/cmssw +CERN_IMAGE = gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cmssw_$(BASE_VERSION)-$(SCRAM_ARCH) + +# If not, then proceed with another target release, specified by DIST +else + +# check for DIST +ifeq (,$(DIST)) +$(error echo set the DIST argument) +endif + # Image can be overidden with env var. DOCKER_IMAGE = cmscloud/$(DIST) CERN_IMAGE = gitlab-registry.cern.ch/cms-cloud/cmssw-docker/$(DIST) +endif + +#$(error echo $(DOCKER_IMAGE) $(CERN_IMAGE)) # Get the latest commit. GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) @@ -69,5 +100,31 @@ docker_push: docker push $(CERN_IMAGE):$(DOCKER_TAG) docker push $(CERN_IMAGE):latest +docker_build_standalone: + # Build Docker image + docker build \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg BASEIMAGE=$(BASEIMAGE) \ + --build-arg SCRAM_ARCH=$(SCRAM_ARCH) \ + --build-arg CMSSW_VERSION=$(CMSSW_VERSION) \ + --build-arg VERSION=$(CODE_VERSION) \ + --build-arg VCS_URL=`git config --get remote.origin.url` \ + --build-arg VCS_REF=$(GIT_COMMIT) \ + -t $(DOCKER_IMAGE):$(BASE_VERSION) \ + -f standalone/Dockerfile \ + --compress --squash . + +docker_push_standalone: + # Tag image also for CERN GitLab container registry + docker tag $(DOCKER_IMAGE):$(BASE_VERSION) $(CERN_IMAGE):$(DOCKER_TAG) + docker tag $(DOCKER_IMAGE):$(BASE_VERSION) $(CERN_IMAGE):latest + + # Push to DockerHub + docker push $(DOCKER_IMAGE):$(BASE_VERSION) + + # Push to CERN GitLab container registry + docker push $(CERN_IMAGE):$(DOCKER_TAG) + docker push $(CERN_IMAGE):latest + output: @echo Docker Image: $(DOCKER_IMAGE):$(DOCKER_TAG) diff --git a/README.md b/README.md index 3c58a1b..5173d64 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,13 @@ The images are based on different sets of CERN Linux distributions: ## Building containers -With the exception of the standalone images (see below), the images need to be built from the base directory of the repository, pointing to the respective `Dockerfile` in the subdirectory using `docker build -f `. A `Makefile` at the top level can be used to simplify this process. A separate `Makefile` is used for standalone images. +The images need to be built from the base directory of the repository, pointing to the respective `Dockerfile` in the subdirectory using `docker build -f `. A `Makefile` at the top level can be used to simplify this process. For properly tagging the containers and the `docker_push` to work you need to `docker login` to the container registry of your preference and pass additional parameters to the `make docker_push` command. The current default for the standalone images at the moment is: ```shell -DOCKER_IMAGE ?= cmscloud/cmssw -CERN_IMAGE ?= gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cmssw_$(BASE_VERSION)-$(SCRAM_ARCH) +DOCKER_IMAGE = cmscloud/cmssw +CERN_IMAGE = gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cmssw_$(BASE_VERSION)-$(SCRAM_ARCH) ``` where `$(BASE_VERSION)` is extracted automatically from the CMSSW release version, e.g. `5_3_32` for `CMSSW_5_3_32`. @@ -42,25 +42,22 @@ Examples are given for different `CMSSW_VERSION` and `SCRAM_ARCH`: Production releases: ```shell -cd standalone -make CMSSW_VERSION=CMSSW_9_2_1 SCRAM_ARCH=slc6_amd64_gcc530 -make docker_push CMSSW_VERSION=CMSSW_9_2_1 SCRAM_ARCH=slc6_amd64_gcc530 +make standalone CMSSW_VERSION=CMSSW_9_2_1 SCRAM_ARCH=slc6_amd64_gcc530 +make docker_push_standalone CMSSW_VERSION=CMSSW_9_2_1 SCRAM_ARCH=slc6_amd64_gcc530 ``` Patch releases work in the same way: ```shell -cd standalone -make CMSSW_VERSION=CMSSW_7_1_25_patch5 SCRAM_ARCH=slc6_amd64_gcc481 -make docker_push CMSSW_VERSION=CMSSW_7_1_25_patch5 SCRAM_ARCH=slc6_amd64_gcc481 +make standalone CMSSW_VERSION=CMSSW_7_1_25_patch5 SCRAM_ARCH=slc6_amd64_gcc481 +make docker_push_standalone CMSSW_VERSION=CMSSW_7_1_25_patch5 SCRAM_ARCH=slc6_amd64_gcc481 ``` Releases for other Linux distribution than SLC6 (here e.g. SLC5): ```shell -cd standalone -make CMSSW_VERSION=CMSSW_4_2_8 SCRAM_ARCH=slc5_amd64_gcc434 BASEIMAGE=gitlab-registry.cern.ch/cms-cloud/cmssw-docker/slc5-cms:latest -make docker_push CMSSW_VERSION=CMSSW_4_2_8 SCRAM_ARCH=slc5_amd64_gcc434 BASEIMAGE=gitlab-registry.cern.ch/cms-cloud/cmssw-docker/slc5-cms:latest +make standalone CMSSW_VERSION=CMSSW_4_2_8 SCRAM_ARCH=slc5_amd64_gcc434 BASEIMAGE=gitlab-registry.cern.ch/cms-cloud/cmssw-docker/slc5-cms:latest +make docker_push_standalone CMSSW_VERSION=CMSSW_4_2_8 SCRAM_ARCH=slc5_amd64_gcc434 BASEIMAGE=gitlab-registry.cern.ch/cms-cloud/cmssw-docker/slc5-cms:latest ``` ### Building CVMFS versions diff --git a/standalone/Dockerfile b/standalone/Dockerfile index 3f6f145..6c285fd 100644 --- a/standalone/Dockerfile +++ b/standalone/Dockerfile @@ -73,8 +73,10 @@ RUN /bin/cp -f ${CMS_INSTALL_DIR}/cmsset_default.sh /etc/profile.d/ USER cmsinst WORKDIR /cvmfs/cms.cern.ch/SITECONF ADD --chown=cmsinst:cmsinst siteconf/T3_CH_PublicCloud /cvmfs/cms.cern.ch/SITECONF/T3_CH_PublicCloud -RUN rm -rf /cvmfs/cms.cern.ch/SITECONF/local && \ - ln -s T3_CH_PublicCloud local +RUN rm -rf /cvmfs/cms.cern.ch/SITECONF/local +USER root +RUN ln -s T3_CH_PublicCloud local \ + & chown cmsinst:cmsinst local USER cmsusr WORKDIR /code diff --git a/standalone/Makefile b/standalone/Makefile deleted file mode 100644 index 740c587..0000000 --- a/standalone/Makefile +++ /dev/null @@ -1,81 +0,0 @@ -default: build - -# Build Docker image -build: docker_build output - -# Build and push Docker image -release: docker_build docker_push output - -# variables need to be overwritten -SCRAM_ARCH=slc6_amd64_gcc481 -CMSSW_VERSION=CMSSW_7_1_25_patch5 -BASEIMAGE=gitlab-registry.cern.ch/cms-cloud/cmssw-docker/slc6-cms - -# function to return base version -get_version = $(strip $(subst CMSSW_, ,$1)) -BASE_VERSION = $(call get_version,$(CMSSW_VERSION)) - -# Image can be overidden with env var. -DOCKER_IMAGE ?= cmscloud/cmssw -CERN_IMAGE ?= gitlab-registry.cern.ch/cms-cloud/cmssw-docker/cmssw_$(BASE_VERSION)-$(SCRAM_ARCH) - -# Get the latest commit. -GIT_COMMIT = $(strip $(shell git rev-parse --short HEAD)) - -# Get the date as version number -CODE_VERSION = $(strip $(shell date +"%Y-%m-%d")) - -# Find out if the working directory is clean -GIT_NOT_CLEAN_CHECK = $(shell git status --porcelain) -ifneq (x$(GIT_NOT_CLEAN_CHECK), x) -DOCKER_TAG_SUFFIX = "-dirty" -endif - -# If we're releasing to Docker Hub, and we're going to mark it with the latest tag, it should exactly match a version release -ifeq ($(MAKECMDGOALS),release) -# Use the version number as the release tag. -DOCKER_TAG = $(CODE_VERSION) - -# See what commit is tagged to match the version -VERSION_COMMIT = $(strip $(shell git rev-list $(CODE_VERSION) -n 1 | cut -c1-7)) -ifneq ($(VERSION_COMMIT), $(GIT_COMMIT)) -$(error echo You are trying to push a build based on commit $(GIT_COMMIT) but the tagged release version is $(VERSION_COMMIT)) -endif - -# Don't push to Docker Hub if this isn't a clean repo -ifneq (x$(GIT_NOT_CLEAN_CHECK), x) -$(error echo You are trying to release a build based on a dirty repo) -endif - -else -# Add the commit ref for development builds. Mark as dirty if the working directory isn't clean -DOCKER_TAG = $(CODE_VERSION)-$(GIT_COMMIT)$(DOCKER_TAG_SUFFIX) -endif - -docker_build: - # Build Docker image - docker build \ - --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ - --build-arg BASEIMAGE=$(BASEIMAGE) \ - --build-arg SCRAM_ARCH=$(SCRAM_ARCH) \ - --build-arg CMSSW_VERSION=$(CMSSW_VERSION) \ - --build-arg VERSION=$(CODE_VERSION) \ - --build-arg VCS_URL=`git config --get remote.origin.url` \ - --build-arg VCS_REF=$(GIT_COMMIT) \ - -t $(DOCKER_IMAGE):$(BASE_VERSION) \ - --compress --squash . - -docker_push: - # Tag image also for CERN GitLab container registry - docker tag $(DOCKER_IMAGE):$(BASE_VERSION) $(CERN_IMAGE):$(DOCKER_TAG) - docker tag $(DOCKER_IMAGE):$(BASE_VERSION) $(CERN_IMAGE):latest - - # Push to DockerHub - docker push $(DOCKER_IMAGE):$(BASE_VERSION) - - # Push to CERN GitLab container registry - docker push $(CERN_IMAGE):$(DOCKER_TAG) - docker push $(CERN_IMAGE):latest - -output: - @echo Docker Image: $(DOCKER_IMAGE):$(BASE_VERSION) -- GitLab From e8d54f1e437724ef67a46228099bac393bade303 Mon Sep 17 00:00:00 2001 From: Tom McCauley Date: Thu, 8 Sep 2022 14:35:48 +0100 Subject: [PATCH 2/4] combine RUN commands in Dockerfile --- standalone/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/standalone/Dockerfile b/standalone/Dockerfile index 6c285fd..75f1194 100644 --- a/standalone/Dockerfile +++ b/standalone/Dockerfile @@ -73,9 +73,9 @@ RUN /bin/cp -f ${CMS_INSTALL_DIR}/cmsset_default.sh /etc/profile.d/ USER cmsinst WORKDIR /cvmfs/cms.cern.ch/SITECONF ADD --chown=cmsinst:cmsinst siteconf/T3_CH_PublicCloud /cvmfs/cms.cern.ch/SITECONF/T3_CH_PublicCloud -RUN rm -rf /cvmfs/cms.cern.ch/SITECONF/local USER root -RUN ln -s T3_CH_PublicCloud local \ +RUN rm -rf /cvmfs/cms.cern.ch/SITECONF/local \ + & ln -s T3_CH_PublicCloud local \ & chown cmsinst:cmsinst local USER cmsusr -- GitLab From bb4fcc0069e797ff9c554aa4812889fb5c121942 Mon Sep 17 00:00:00 2001 From: Tom McCauley Date: Fri, 9 Sep 2022 09:20:56 +0100 Subject: [PATCH 3/4] small edit re: root as USER --- standalone/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/standalone/Dockerfile b/standalone/Dockerfile index 75f1194..c21d45b 100644 --- a/standalone/Dockerfile +++ b/standalone/Dockerfile @@ -73,7 +73,6 @@ RUN /bin/cp -f ${CMS_INSTALL_DIR}/cmsset_default.sh /etc/profile.d/ USER cmsinst WORKDIR /cvmfs/cms.cern.ch/SITECONF ADD --chown=cmsinst:cmsinst siteconf/T3_CH_PublicCloud /cvmfs/cms.cern.ch/SITECONF/T3_CH_PublicCloud -USER root RUN rm -rf /cvmfs/cms.cern.ch/SITECONF/local \ & ln -s T3_CH_PublicCloud local \ & chown cmsinst:cmsinst local -- GitLab From 1f30b4ef823ff31e52a07d02f00364ba3f97f7cc Mon Sep 17 00:00:00 2001 From: Thomas Mc Cauley Date: Fri, 9 Sep 2022 16:17:38 +0200 Subject: [PATCH 4/4] rm cmsinst as USER for sitconf stuff --- standalone/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/standalone/Dockerfile b/standalone/Dockerfile index c21d45b..1e76667 100644 --- a/standalone/Dockerfile +++ b/standalone/Dockerfile @@ -70,7 +70,6 @@ USER root RUN /bin/cp -f ${CMS_INSTALL_DIR}/cmsset_default.sh /etc/profile.d/ # Add siteconfig -USER cmsinst WORKDIR /cvmfs/cms.cern.ch/SITECONF ADD --chown=cmsinst:cmsinst siteconf/T3_CH_PublicCloud /cvmfs/cms.cern.ch/SITECONF/T3_CH_PublicCloud RUN rm -rf /cvmfs/cms.cern.ch/SITECONF/local \ -- GitLab