diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ded5b32dc76c622473bcb8cd0f4fdc3d3ba0c71e..cb9fea1ac745f7e7530616abd81258fe41237e1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,11 +35,10 @@ variables: # - COMMIT: default pipeline type, Runs static analysis, compiles the software and tests it. # - EOS_REGR_AGAINST_CTA_MAIN: test a new tagged EOS version against CTA main. # - EOS_REGR_AGAINST_CTA_TAG: test a new tagged EOS version against the latest tagged CTA version. - # - CTAGENERIC_IMAGE: generate a ctageneric image from a set of CTA,EOS,XRD RPMS avaiable in repos. + # - CTAGENERIC_IMAGE: generate a ctageneric image from a set of CTA RPMS avaiable in repos. PIPELINE_TYPE: "COMMIT" # Image building. Used by EOS Regression and Image build pipeline types - CUSTOM_XRD_TAG: "" CUSTOM_EOS_TAG: "" CUSTOM_CTA_TAG: "" diff --git a/.gitlab/ci/build-image.gitlab-ci.yml b/.gitlab/ci/build-image.gitlab-ci.yml index 353a75c61ef5ad251a400b69d6f5f9f31b5ca2a9..f6a312b06d8ec9b473aa114335a03586b5486aba 100644 --- a/.gitlab/ci/build-image.gitlab-ci.yml +++ b/.gitlab/ci/build-image.gitlab-ci.yml @@ -24,8 +24,6 @@ build-ctageneric-from-artifacts: - .build-ctageneric needs: # Needs RPM artifacts to build the image - job: build-cta-rpm - - job: modify-versionlock - optional: true rules: - if: $PIPELINE_TYPE == "EOS_REGR_AGAINST_CTA_TAG" || $PIPELINE_TYPE == "CTAGENERIC_IMAGE" when: never @@ -49,18 +47,77 @@ build-ctageneric-from-repos: needs: - job: pipeline-variable-check optional: true - - job: modify-versionlock - optional: true rules: - if: $PIPELINE_TYPE == "EOS_REGR_AGAINST_CTA_TAG" - if: $PIPELINE_TYPE == "CTAGENERIC_IMAGE" - when: never - !reference [.build-ctageneric, rules] script: - - echo "Building image with CTA ${CUSTOM_CTA_TAG} RPMs and EOS ${CUSTOM_EOS_TAG} RPMs" + - echo "Building image with CTA ${CUSTOM_CTA_TAG} RPMs" - echo "Image tag ${CI_REGISTRY}/cta/ctageneric:${CTA_BUILD_ID}" - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile continuousintegration/docker/alma9/tag.Dockerfile --destination ${CI_REGISTRY}/cta/ctageneric:${CTA_BUILD_ID} --build-arg "PUBLIC_REPO_VER=${CUSTOM_CTA_TAG:1}" - echo "Image pushed successfully to ${CI_REGISTRY}/cta/ctageneric:${CTA_BUILD_ID}" + +build-ctageneric-5-11-2: + extends: + - .build-ctageneric + needs: + - job: pipeline-variable-check + optional: true + rules: + - if: $PIPELINE_TYPE == "CTAGENERIC_IMAGE" + - when: never + - !reference [.build-ctageneric, rules] + variables: + IMAGE_TAG: 5.11.2.0-1-${CI_JOB_ID} + script: + - echo "Building image with CTA ${CUSTOM_CTA_TAG} RPMs" + - echo "Image tag ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG}" + - /kaniko/executor --context $CI_PROJECT_DIR + --dockerfile continuousintegration/docker/alma9/5.11.2.0-1.Dockerfile + --destination ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG} + --build-arg "PUBLIC_REPO_VER=5.11.2.0-1" + - echo "Image pushed successfully to ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG}" + +build-ctageneric-5-10-10: + extends: + - .build-ctageneric + needs: + - job: pipeline-variable-check + optional: true + rules: + - if: $PIPELINE_TYPE == "CTAGENERIC_IMAGE" + - when: never + - !reference [.build-ctageneric, rules] + variables: + IMAGE_TAG: 5.10.10.1-1-${CI_JOB_ID} + script: + - echo "Building image with CTA ${CUSTOM_CTA_TAG} RPMs" + - echo "Image tag ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG}" + - /kaniko/executor --context $CI_PROJECT_DIR + --dockerfile continuousintegration/docker/alma9/5.10.10.1-1.Dockerfile + --destination ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG} + - echo "Image pushed successfully to ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG}" + +build-ctageneric-5-11-0: + extends: + - .build-ctageneric + needs: + - job: pipeline-variable-check + optional: true + rules: + - if: $PIPELINE_TYPE == "CTAGENERIC_IMAGE" + - when: never + - !reference [.build-ctageneric, rules] + variables: + IMAGE_TAG: 5.11.0.1-1-${CI_JOB_ID} + script: + - echo "Building image with CTA ${CUSTOM_CTA_TAG} RPMs" + - echo "Image tag ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG}" + - /kaniko/executor --context $CI_PROJECT_DIR + --dockerfile continuousintegration/docker/alma9/5.11.0.1-1.Dockerfile + --destination ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG} + - echo "Image pushed successfully to ${CI_REGISTRY}/cta/ctageneric:${IMAGE_TAG}" diff --git a/.gitlab/ci/build.gitlab-ci.yml b/.gitlab/ci/build.gitlab-ci.yml index b399a8943906248988d5c19e7e0ed6c9f17f70e3..bd7e28120e69e817c79c9615062fcbb736499799 100644 --- a/.gitlab/ci/build.gitlab-ci.yml +++ b/.gitlab/ci/build.gitlab-ci.yml @@ -21,9 +21,6 @@ build-cta-srpm: stage: build extends: - .build-cta - needs: - - job: modify-versionlock - optional: true rules: - !reference [.build-cta, rules] - when: on_success @@ -46,9 +43,6 @@ build-cta-rpm: - .build-cta needs: - job: build-cta-srpm - - job: modify-versionlock - optional: true - artifacts: true rules: - !reference [.build-cta, rules] - when: on_success diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml deleted file mode 100644 index d277f485341b7bfb8942b23682191a4b6060c970..0000000000000000000000000000000000000000 --- a/.gitlab/ci/setup.gitlab-ci.yml +++ /dev/null @@ -1,60 +0,0 @@ -# Job to modify the version lock list file for external triggers -modify-versionlock: - stage: setup - variables: - GIT_DEPTH: 0 - rules: - - if: $PIPELINE_TYPE == "EOS_REGR_AGAINST_CTA_MAIN" - - if: $PIPELINE_TYPE == "EOS_REGR_AGAINST_CTA_TAG" - - if: $PIPELINE_TYPE == "CTAGENERIC_IMAGE" - script: - - versionlock_path_main="continuousintegration/docker/alma9/etc/yum/pluginconf.d/" - - versionlock_file="${versionlock_path_main}versionlock.list" - - | - if [ -n "${CUSTOM_CTA_TAG}" ]; then - versionlock_file=$(./continuousintegration/utils/find_versionlock_list.sh --tag "${CUSTOM_CTA_TAG}") - fi - - git checkout ${CUSTOM_CTA_TAG} -- "${versionlock_file}" - - echo "Extracting the currently version locked EOS version from the file..." - - current_eos_version=$(grep -o 'eos-server-[0-9]\+\.[0-9]\+\.[0-9]\+' "${versionlock_file}" | awk -F '-' '{print $3}' | head -1) - - echo "Current version found $current_eos_version" - - | - if [ -z "$current_eos_version" ]; then - echo "Version not found! Exiting."; - exit 1; - fi - - echo "Replacing $current_eos_version with $CUSTOM_EOS_TAG in the file..." - - sed -i "s/$current_eos_version/$CUSTOM_EOS_TAG/g" ${versionlock_file} - # Check if XROOTD_VERSION is provided and not just whitespace - - trimmed_xrootd_version=$(echo "$CUSTOM_XRD_TAG" | xargs) - # Check if CUSTOM_XRD_TAG is provided - - | - if [ -n "$trimmed_xrootd_version" ]; then - echo "Extracting the currently version locked XRootD version from the file..." - current_xrootd_version=$(grep -o 'xrootd-server-\([0-9]:\)*[0-9]\+\.[0-9]\+\.[0-9]\+' "${versionlock_file}" | awk -F '-' '{print $3}' | head -1) - echo "Current XRootD version found: $current_xrootd_version" - - if [ -n "$current_xrootd_version" ]; then - echo "Replacing $current_xrootd_version with $CUSTOM_XRD_TAG in the file..." - sed -i "s/$current_xrootd_version/$CUSTOM_XRD_TAG/g" "${versionlock_file}" - else - echo "XRootD version not found in the file." - fi - # Replace xrootd version in spec file - sed -i "s/^%define xrootdVersion $current_xrootd_version/%define xrootdVersion $CUSTOM_XRD_TAG/" cta.spec.in - else - echo "CUSTOM_XRD_TAG is not provided. Skipping XRootD version replacement." - fi - - echo "File content after replacement" - - cat "${versionlock_file}" - - | - if [ "${versionlock_path_main}versionlock.list" != "${versionlock_file}" ]; then - echo "Overwritting versionlock file as tagged version lock file resides in a different path." - cp "${versionlock_file}" "${versionlock_path_main}" - fi - artifacts: - paths: - - continuousintegration/docker/alma9/etc/yum/pluginconf.d/versionlock.list - - cta.spec.in - when: on_success - diff --git a/continuousintegration/docker/alma9/5.10.10.1-1.Dockerfile b/continuousintegration/docker/alma9/5.10.10.1-1.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..048e46c81ffbe4cd1c130629d12aa391d1a06b61 --- /dev/null +++ b/continuousintegration/docker/alma9/5.10.10.1-1.Dockerfile @@ -0,0 +1,60 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2024 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +# CTA generic image containing all CTA RPMs +# This image is similar to the one created by Dockerfile, except that this uses RPMs for a remote repo instead of local ones +# As a result, any installs on this image will also pull from public yum repos instead of private ones +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest + +ENV BASEDIR="continuousintegration/docker/alma9" \ + CTAREPODIR="/opt/repo" + +COPY ${BASEDIR}/etc/yum.repos.d/ /etc/yum.repos.d/ + +ENV PUBLIC_REPO_VER=5.10.10.1-1.el9 + +# Install necessary packages +RUN dnf install -y \ + python3-dnf-plugin-versionlock \ + yum-utils \ + epel-release \ + jq \ + bc \ + sqlite \ + wget \ + krb5-workstation && \ + # logrotate files must be 0644 or 0444 + chmod 0644 /etc/logrotate.d/* + +RUN wget https://download.oracle.com/otn_software/linux/instantclient/2112000/el9/oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm; \ + wget https://download.oracle.com/otn_software/linux/instantclient/2112000/el9/oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; \ + yum install -y oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm; \ + yum install -y oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; \ + rm oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; + +# We add the cta user so that we can consistently reference it in the Helm chart when changing keytab ownership +RUN useradd -m -u 1000 -g tape cta + +# Install cta-release and clean up +RUN dnf config-manager --enable epel --setopt="epel.priority=4" && \ + dnf config-manager --enable cta-public-testing && \ + dnf clean all --enablerepo=\* && \ + rm -rf /etc/rc.d/rc.local + + + + +ENV YUM_VERSIONLOCK_FILE=continuousintegration/docker/alma9/5.10.10.1-1.versionlock.list +COPY ${YUM_VERSIONLOCK_FILE} /etc/dnf/plugins/versionlock.list diff --git a/continuousintegration/docker/alma9/5.10.10.1-1.versionlock.list b/continuousintegration/docker/alma9/5.10.10.1-1.versionlock.list new file mode 100644 index 0000000000000000000000000000000000000000..0786e78db117b3e4973e31e77abbd64a97bbedaa --- /dev/null +++ b/continuousintegration/docker/alma9/5.10.10.1-1.versionlock.list @@ -0,0 +1,54 @@ +eos-archive-5.2.24-1.el9.x86_64 +eos-cleanup-5.2.24-1.el9.x86_64 +eos-client-5.2.24-1.el9.x86_64 +eos-fuse-5.2.24-1.el9.x86_64 +eos-fuse-core-5.2.24-1.el9.x86_64 +eos-fuse-sysv-5.2.24-1.el9.x86_64 +eos-fusex-5.2.24-1.el9.x86_64 +eos-fusex-core-5.2.24-1.el9.x86_64 +eos-fusex-selinux-5.2.24-1.el9.x86_64 +eos-ns-inspect-5.2.24-1.el9.x86_64 +eos-server-5.2.24-1.el9.x86_64 +eos-srm-5.2.24-1.el9.x86_64 +eos-test-5.2.24-1.el9.x86_64 +eos-testkeytab-5.2.24-1.el9.x86_64 +eos-quarkdb-5.2.24-1.el9.x86_64 +eos-folly-2019.11.11.00-1.el9.* +eos-folly-deps-2019.11.11.00-1.el9.* +python3-xrootd-1:5.6.1-1.el9.* +xrootd-1:5.6.1-1.el9.* +xrootd-client-1:5.6.1-1.el9.* +xrootd-client-devel-1:5.6.1-1.el9.* +xrootd-client-libs-1:5.6.1-1.el9.* +xrootd-debuginfo-1:5.6.1-1.el9.* +xrootd-devel-1:5.6.1-1.el9.* +xrootd-doc-1:5.6.1-1.el9.* +xrootd-fuse-1:5.6.1-1.el9.* +xrootd-libs-1:5.6.1-1.el9.* +xrootd-private-devel-1:5.6.1-1.el9.* +xrootd-selinux-1:5.6.1-1.el9.* +xrootd-server-1:5.6.1-1.el9.* +xrootd-server-devel-1:5.6.1-1.el9.* +xrootd-server-libs-1:5.6.1-1.el9.* +xrootd-tests-1:5.6.1-1.el9.* +cta-taped-0:5.10.10.1-1.el9.* +cta-frontend-0:5.10.10.1-1.el9.* +cta-frontend-grpc-0:5.10.10.1-1.el9.* +cta-cli-0:5.10.10.1-1.el9.* +cta-lib-common-0:5.10.10.1-1.el9.* +cta-lib-catalogue-occi-0:5.10.10.1-1.el9.* +cta-lib-catalogue-postgres-0:5.10.10.1-1.el9.* +cta-lib-catalogue-inmemory-0:5.10.10.1-1.el9.* +cta-lib-catalogue-0:5.10.10.1-1.el9.* +cta-lib-0:5.10.10.1-1.el9.* +cta-systemtests-0:5.10.10.1-1.el9.* +cta-immutable-file-test-0:5.10.10.1-1.el9.* +cta-objectstore-tools-0:5.10.10.1-1.el9.* +cta-catalogueutils-0:5.10.10.1-1.el9.* +cta-scheduler-utils-0:5.10.10.1-1.el9.* +cta-rmcd-0:5.10.10.1-1.el9.* +cta-smc-0:5.10.10.1-1.el9.* +cta-tape-label-0:5.10.10.1-1.el9.* +cta-readtp-0:5.10.10.1-1.el9.* +cta-common-0:5.10.10.1-1.el9.* +cta-fst-gcd-0:5.10.10.1-1.el9.* diff --git a/continuousintegration/docker/alma9/5.11.0.1-1.Dockerfile b/continuousintegration/docker/alma9/5.11.0.1-1.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e0acc4429731cd74e457d9f863bc3612bbabf4b2 --- /dev/null +++ b/continuousintegration/docker/alma9/5.11.0.1-1.Dockerfile @@ -0,0 +1,57 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2024 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +# CTA generic image containing all CTA RPMs +# This image is similar to the one created by Dockerfile, except that this uses RPMs for a remote repo instead of local ones +# As a result, any installs on this image will also pull from public yum repos instead of private ones +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest + +ENV BASEDIR="continuousintegration/docker/alma9" \ + CTAREPODIR="/opt/repo" + +COPY ${BASEDIR}/etc/yum.repos.d/cta-public-testing.repo /etc/yum.repos.d/cta-public-testing.repo + +ENV PUBLIC_REPO_VER=5.11.0.1-1 + +# Install necessary packages +RUN dnf install -y \ + python3-dnf-plugin-versionlock \ + yum-utils \ + epel-release \ + jq \ + bc \ + sqlite \ + wget \ + krb5-workstation && \ + # logrotate files must be 0644 or 0444 + chmod 0644 /etc/logrotate.d/* + +RUN wget https://download.oracle.com/otn_software/linux/instantclient/2112000/el9/oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm; \ + wget https://download.oracle.com/otn_software/linux/instantclient/2112000/el9/oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; \ + yum install -y oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm; \ + yum install -y oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; \ + rm oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; + + +# We add the cta user so that we can consistently reference it in the Helm chart when changing keytab ownership +RUN useradd -m -u 1000 -g tape cta + +# Install cta-release and clean up +RUN dnf config-manager --enable epel --setopt="epel.priority=4" && \ + dnf config-manager --enable cta-public-testing && \ + dnf install -y "cta-release-${PUBLIC_REPO_VER}.el9" && \ + cta-versionlock apply && \ + dnf clean all --enablerepo=\* && \ + rm -rf /etc/rc.d/rc.local diff --git a/continuousintegration/docker/alma9/5.11.2.0-1.Dockerfile b/continuousintegration/docker/alma9/5.11.2.0-1.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bb2e092d51390aa7e32079ff8305b9946de31b44 --- /dev/null +++ b/continuousintegration/docker/alma9/5.11.2.0-1.Dockerfile @@ -0,0 +1,57 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2024 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +# CTA generic image containing all CTA RPMs +# This image is similar to the one created by Dockerfile, except that this uses RPMs for a remote repo instead of local ones +# As a result, any installs on this image will also pull from public yum repos instead of private ones +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest + +ENV BASEDIR="continuousintegration/docker/alma9" \ + CTAREPODIR="/opt/repo" + +COPY ${BASEDIR}/etc/yum.repos.d/cta-public-testing.repo /etc/yum.repos.d/cta-public-testing.repo + +ENV PUBLIC_REPO_VER=5.11.2.0-1 + +# Install necessary packages +RUN dnf install -y \ + python3-dnf-plugin-versionlock \ + yum-utils \ + epel-release \ + jq \ + bc \ + sqlite \ + wget \ + krb5-workstation && \ + # logrotate files must be 0644 or 0444 + chmod 0644 /etc/logrotate.d/* + +RUN wget https://download.oracle.com/otn_software/linux/instantclient/2112000/el9/oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm; \ + wget https://download.oracle.com/otn_software/linux/instantclient/2112000/el9/oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; \ + yum install -y oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm; \ + yum install -y oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; \ + rm oracle-instantclient-basic-21.12.0.0.0-1.el9.x86_64.rpm oracle-instantclient-devel-21.12.0.0.0-1.el9.x86_64.rpm; + + +# We add the cta user so that we can consistently reference it in the Helm chart when changing keytab ownership +RUN useradd -m -u 1000 -g tape cta + +# Install cta-release and clean up +RUN dnf config-manager --enable epel --setopt="epel.priority=4" && \ + dnf config-manager --enable cta-public-testing && \ + dnf install -y "cta-release-${PUBLIC_REPO_VER}.el9" && \ + cta-versionlock apply && \ + dnf clean all --enablerepo=\* && \ + rm -rf /etc/rc.d/rc.local diff --git a/continuousintegration/docker/alma9/Dockerfile b/continuousintegration/docker/alma9/Dockerfile index 1b169cb441569bc7843a6a6d0bf2eb41ad55d279..91a28afce839510509efa507b783b38335b16f55 100644 --- a/continuousintegration/docker/alma9/Dockerfile +++ b/continuousintegration/docker/alma9/Dockerfile @@ -19,9 +19,6 @@ FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest ENV BASEDIR="continuousintegration/docker/alma9" \ CTAREPODIR="/opt/repo" -# Add orchestration run scripts locally -COPY ${BASEDIR}/../opt /opt - # Custom Yum repo setup ARG YUM_REPOS_DIR=continuousintegration/docker/alma9/etc/yum.repos.d/ ARG YUM_VERSIONLOCK_FILE=continuousintegration/docker/alma9/etc/yum/pluginconf.d/versionlock.list diff --git a/continuousintegration/docker/alma9/tag.Dockerfile b/continuousintegration/docker/alma9/tag.Dockerfile index 6d565bcfcefdf5e7425fc5fa0f7df2dd9530de09..c7b517f129f6a7a4b31b1f67e9e87c85d545847c 100644 --- a/continuousintegration/docker/alma9/tag.Dockerfile +++ b/continuousintegration/docker/alma9/tag.Dockerfile @@ -21,14 +21,11 @@ FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest ENV BASEDIR="continuousintegration/docker/alma9" \ CTAREPODIR="/opt/repo" -# Add orchestration run scripts locally -COPY ${BASEDIR}/../opt /opt -COPY ${BASEDIR}/etc/yum.repos.d/ /etc/yum.repos.d/ +COPY ${BASEDIR}/etc/yum.repos.d/cta-public-testing.repo /etc/yum.repos.d/cta-public-testing.repo # Variable to specify the tag to be used for CTA RPMs from the cta-ci-repo # Format: X.YY.ZZ.A-B ARG PUBLIC_REPO_VER -ARG YUM_VERSIONLOCK_FILE=continuousintegration/docker/alma9/etc/yum/pluginconf.d/versionlock.list # Install necessary packages RUN dnf install -y \ @@ -49,8 +46,6 @@ RUN useradd -m -u 1000 -g tape cta RUN dnf config-manager --enable epel --setopt="epel.priority=4" && \ dnf config-manager --enable cta-public-testing && \ dnf install -y "cta-release-${PUBLIC_REPO_VER}.el9" && \ - rm -f /etc/yum/pluginconf.d/versionlock.cta && \ + cta-versionlock apply && \ dnf clean all --enablerepo=\* && \ rm -rf /etc/rc.d/rc.local - -COPY ${YUM_VERSIONLOCK_FILE} /etc/dnf/plugins/versionlock.list diff --git a/continuousintegration/docker/opt/run/bin/init_pod.sh b/continuousintegration/docker/opt/run/bin/init_pod.sh deleted file mode 100755 index 290940b5d881e112df818e053c84a7ef3c12b429..0000000000000000000000000000000000000000 --- a/continuousintegration/docker/opt/run/bin/init_pod.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# This file must be sourced from another shell script -# . /opt/run/bin/init_pod.sh - -# @project The CERN Tape Archive (CTA) -# @copyright Copyright © 2022-2024 CERN -# @license This program is free software, distributed under the terms of the GNU General Public -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can -# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# In applying this licence, CERN does not waive the privileges and immunities -# granted to it by virtue of its status as an Intergovernmental Organization or -# submit itself to any jurisdiction. - -echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" - -LOGMOUNT=/mnt/logs - -PV_PATH="" - -if [ "-${MY_CONTAINER}-" != "--" ]; then - PV_PATH="${LOGMOUNT}/${MY_NAME}/${MY_CONTAINER}" -else - PV_PATH="${LOGMOUNT}/${MY_NAME}" -fi -mkdir -p ${PV_PATH} - -echo "Copying initial /var/log content to ${PV_PATH}" -cd /var/log -tar -c . | tar -C ${PV_PATH} -xv - -echo "Mounting logs volume ${PV_PATH} in /var/log" -mount --bind ${PV_PATH} /var/log - -# all core dumps will go there as all the pods AND kubelet are sharing the same kernel.core_pattern -mkdir -p /var/log/tmp -chmod 1777 /var/log/tmp -echo '/var/log/tmp/%h-%t-%e-%p-%s.core' > /proc/sys/kernel/core_pattern - -echo -n "Fixing reverse DNS for $(hostname) for xrootd: " -sed -i -c "s/^\($(hostname -i)\)\s\+.*$/\1 $(hostname -s).$(grep search /etc/resolv.conf | cut -d\ -f2) $(hostname -s)/" /etc/hosts -echo "DONE" - -echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Finished" diff --git a/continuousintegration/orchestration/helm/catalogue-updater/templates/pod.yaml b/continuousintegration/orchestration/helm/catalogue-updater/templates/pod.yaml index 9d900d60233ddbf78c64f3a0c67e7a32e3bf0af9..ffb7008f2e94fdde95975c6c4d8790904d9d7c22 100755 --- a/continuousintegration/orchestration/helm/catalogue-updater/templates/pod.yaml +++ b/continuousintegration/orchestration/helm/catalogue-updater/templates/pod.yaml @@ -56,15 +56,19 @@ spec: {{- include "common.extraEnv" . | nindent 4 }} {{- end }} volumeMounts: + {{- if .Values.customRepoFiles }} - name: yum-repos mountPath: /shared/etc_yum.repos.d + {{- end }} - name: catalogue-config mountPath: /shared/etc_cta/cta-catalogue.conf subPath: cta-catalogue.conf volumes: + {{- if .Values.customRepoFiles }} - name: yum-repos configMap: name: yum.repos.d-config + {{- end }} - name: catalogue-config configMap: name: cta-catalogue-conf diff --git a/continuousintegration/orchestration/helm/catalogue-updater/values.yaml b/continuousintegration/orchestration/helm/catalogue-updater/values.yaml index 503e935b2bc2902321203607c2b3f8cae63bbd9c..ed385b28687e1ff4e709489cf4318bf2a2860823 100644 --- a/continuousintegration/orchestration/helm/catalogue-updater/values.yaml +++ b/continuousintegration/orchestration/helm/catalogue-updater/values.yaml @@ -1,6 +1,6 @@ image: - repository: gitlab-registry.cern.ch/cta/eoscta-operations/registry/container_registry/cta-catalogue-updater - tag: "alma9-base.1" + repository: gitlab-registry.cern.ch/cta/public_registry/container_registry/cta-catalogue-updater + tag: "stable" pullPolicy: IfNotPresent pullSecrets: - ctaregsecret @@ -12,3 +12,6 @@ catalogueDestinationVersion: "" # Additional Extra environment variables extraEnv: + +# Set to true if you want to mount the configmap "yum.repos.d-config" +customRepoFiles: false diff --git a/continuousintegration/orchestration/helm/cli/scripts/ctacli.sh b/continuousintegration/orchestration/helm/cli/scripts/ctacli.sh index 9bf9c6587c5df685a3d235089b92ed7f89f98aa3..b670bddefff3b6b8a6567bd9e98761a1d675a9df 100755 --- a/continuousintegration/orchestration/helm/cli/scripts/ctacli.sh +++ b/continuousintegration/orchestration/helm/cli/scripts/ctacli.sh @@ -15,7 +15,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -. /opt/run/bin/init_pod.sh echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" # Install missing RPMs diff --git a/continuousintegration/orchestration/helm/client/scripts/client.sh b/continuousintegration/orchestration/helm/client/scripts/client.sh index 81623055cee93c2b21281087277d339aed774241..9202265ad6cf1e4c72a781f68878ed144ef94d24 100755 --- a/continuousintegration/orchestration/helm/client/scripts/client.sh +++ b/continuousintegration/orchestration/helm/client/scripts/client.sh @@ -15,7 +15,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -. /opt/run/bin/init_pod.sh echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" # Install missing RPMs diff --git a/continuousintegration/orchestration/helm/cta/values.yaml b/continuousintegration/orchestration/helm/cta/values.yaml index 3f3289b21e1dcfb88a70a36f5018fdba16fc2bd4..253374c5319b50256a11368f89cf008e5e47d4e2 100644 --- a/continuousintegration/orchestration/helm/cta/values.yaml +++ b/continuousintegration/orchestration/helm/cta/values.yaml @@ -22,14 +22,9 @@ global: - name: shared hostPath: path: /opt/cta - - name: logstorage - persistentVolumeClaim: - claimName: claimlogs volumeMounts: - name: shared mountPath: /shared - - name: logstorage - mountPath: /mnt/logs # Storage for the shared logs of the deployment logStorage: "8Gi" diff --git a/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend-grpc.sh b/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend-grpc.sh index e9bbea012f4930b56a7e31a1b2fe4ecf1883fc8b..4eadac229d94933ec61c2ad38a3349a847e96117 100755 --- a/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend-grpc.sh +++ b/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend-grpc.sh @@ -15,7 +15,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -. /opt/run/bin/init_pod.sh echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" # Install missing RPMs diff --git a/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend.sh b/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend.sh index 39376971e24446319765d76337a0e549dbc1f81c..4073249bb102d7b654d82168ffd30c612056abe6 100755 --- a/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend.sh +++ b/continuousintegration/orchestration/helm/frontend/scripts/ctafrontend.sh @@ -15,7 +15,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -. /opt/run/bin/init_pod.sh echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" # Install missing RPMs diff --git a/continuousintegration/orchestration/helm/frontend/templates/configmaps/cta-frontend-xrootd.yaml b/continuousintegration/orchestration/helm/frontend/templates/configmaps/cta-frontend-xrootd.yaml index 89552175837113077d1c304cae9613aed0bfc42f..9db48313f603ecd201dcb4cd0efedc3ecf0b2255 100644 --- a/continuousintegration/orchestration/helm/frontend/templates/configmaps/cta-frontend-xrootd.yaml +++ b/continuousintegration/orchestration/helm/frontend/templates/configmaps/cta-frontend-xrootd.yaml @@ -21,12 +21,13 @@ data: cta.schedulerdb.scheduler_backend_name {{ $schedulerConfig.backend }} cta.schedulerdb.numberofthreads {{ .Values.conf.schedulerdb.numberOfThreads }} cta.schedulerdb.threadstacksize_mb {{ .Values.conf.schedulerdb.threadStackSizeMb }} - cta.schedulerdb.disable_repack_requests {{ .Values.conf.schedulerdb.disableRepackRequests }} - cta.schedulerdb.disable_user_requests {{ .Values.conf.schedulerdb.disableUserRequests }} # CTA Scheduler DB options - Cache timeout options (decreased for tests) cta.schedulerdb.tape_cache_max_age_secs {{ .Values.conf.schedulerdb.tapeCacheMaxAgeSecs }} cta.schedulerdb.retrieve_queue_cache_max_age_secs {{ .Values.conf.schedulerdb.retrieveQueueCacheMaxAgeSecs }} + # This is for backward compatibility sake + cta.schedulerdb.enable_repack_requests on + cta.schedulerdb.enable_user_requests on # CTA Catalogue options cta.catalogue.numberofconnections {{ .Values.conf.catalogue.numberOfConnections }} diff --git a/continuousintegration/orchestration/helm/frontend/values.yaml b/continuousintegration/orchestration/helm/frontend/values.yaml index e320c0f09cbe36c0a286f3a8959f8afd668e23c3..341ee5b1a948712bf6e816ef473a72161eefd41e 100644 --- a/continuousintegration/orchestration/helm/frontend/values.yaml +++ b/continuousintegration/orchestration/helm/frontend/values.yaml @@ -17,8 +17,6 @@ conf: schedulerdb: numberOfThreads: 500 threadStackSizeMb: 1 - disableRepackRequests: false - disableUserRequests: false tapeCacheMaxAgeSecs: 1 retrieveQueueCacheMaxAgeSecs: 1 diff --git a/continuousintegration/orchestration/helm/tpsrv/scripts/rmcd.sh b/continuousintegration/orchestration/helm/tpsrv/scripts/rmcd.sh index 7b80b0492c93af726628bc8a40e6c8d872bfbbf1..c171811464734b78d69f851073735578d5aa2b4b 100755 --- a/continuousintegration/orchestration/helm/tpsrv/scripts/rmcd.sh +++ b/continuousintegration/orchestration/helm/tpsrv/scripts/rmcd.sh @@ -15,7 +15,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -. /opt/run/bin/init_pod.sh echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" # Note that this sets the symbolic link for ALL containers, as /dev/ is shared diff --git a/continuousintegration/orchestration/helm/tpsrv/scripts/taped.sh b/continuousintegration/orchestration/helm/tpsrv/scripts/taped.sh index 62bec88122a275fede5a3d6b7089cf55dcd9d290..2c8f8a0ea63900e703bdc8233f5a0b026ea0f4aa 100755 --- a/continuousintegration/orchestration/helm/tpsrv/scripts/taped.sh +++ b/continuousintegration/orchestration/helm/tpsrv/scripts/taped.sh @@ -15,7 +15,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -. /opt/run/bin/init_pod.sh echo "$(date '+%Y-%m-%d %H:%M:%S') [$(basename "${BASH_SOURCE[0]}")] Started" # Install missing RPMs diff --git a/continuousintegration/orchestration/tests/prepare_tests.sh b/continuousintegration/orchestration/tests/prepare_tests.sh index 5c8d96196f6d30723819325e69d598069f6efdb9..cdead1d00b66d366499a894e67a85cdd3e1f1a1c 100755 --- a/continuousintegration/orchestration/tests/prepare_tests.sh +++ b/continuousintegration/orchestration/tests/prepare_tests.sh @@ -176,6 +176,7 @@ kubectl --namespace ${NAMESPACE} exec ${CTA_CLI_POD} -c cta-cli -- cta-admin tap --name ctasystest \ --vo vo \ --partialtapesnumber 5 \ + --encrypted false \ --comment "ctasystest" kubectl --namespace ${NAMESPACE} exec ${CTA_CLI_POD} -c cta-cli -- cta-admin storageclass add \ @@ -194,16 +195,19 @@ kubectl --namespace ${NAMESPACE} exec ${CTA_CLI_POD} -c cta-cli -- cta-admin tap --name ctasystest_A \ --vo vo \ --partialtapesnumber 5 \ + --encrypted false \ --comment "ctasystest_A" kubectl --namespace ${NAMESPACE} exec ${CTA_CLI_POD} -c cta-cli -- cta-admin tapepool add \ --name ctasystest_B \ --vo vo \ --partialtapesnumber 5 \ + --encrypted false \ --comment "ctasystest_B" kubectl --namespace ${NAMESPACE} exec ${CTA_CLI_POD} -c cta-cli -- cta-admin tapepool add \ --name ctasystest_C \ --vo vo \ --partialtapesnumber 5 \ + --encrypted false \ --comment "ctasystest_C" kubectl --namespace ${NAMESPACE} exec ${CTA_CLI_POD} -c cta-cli -- cta-admin storageclass add \ diff --git a/continuousintegration/orchestration/tests/test_liquibase_update.sh b/continuousintegration/orchestration/tests/test_liquibase_update.sh index f5b800db0003faa0f93aa20e1385651a71d8c434..6dec7cee7ea029f862bc22ede1dd91ab56d9aace 100755 --- a/continuousintegration/orchestration/tests/test_liquibase_update.sh +++ b/continuousintegration/orchestration/tests/test_liquibase_update.sh @@ -82,6 +82,7 @@ kubectl -n ${NAMESPACE} create configmap yum.repos.d-config --from-file=${yum_re helm install catalogue-updater ../helm/catalogue-updater --namespace ${NAMESPACE} \ --set catalogueSourceVersion=$prev_catalogue_schema_version \ --set catalogueDestinationVersion=$catalogue_schema_version \ + --set customRepoFiles=true \ --wait --timeout 2m kubectl -n ${NAMESPACE} exec -it liquibase-update -- /bin/bash -c "/launch_liquibase.sh \"tag --tag=test_update\"" diff --git a/continuousintegration/validation/ci_input_validate.py b/continuousintegration/validation/ci_input_validate.py index e6aa253f886305cfc11806a0f9cb2071a931394d..02682327daf2cc47c2ed62b3cd64e1e9892f67e7 100644 --- a/continuousintegration/validation/ci_input_validate.py +++ b/continuousintegration/validation/ci_input_validate.py @@ -30,8 +30,6 @@ SUPPORTED = { "EOS_REGR_AGAINST_CTA_MAIN", "EOS_REGR_AGAINST_CTA_TAG", "CTAGENERIC_IMAGE"], - "CUSTOM_XRD_TAG_REGEX": [ re.compile("^\d+:\d+\.\d+\.\d+(-\d+)*$") ], - "CUSTOM_EOS_TAG_REGEX": [ re.compile("^\d+\.\d+\.\d+$") ], "CUSTOM_CTA_TAG_REGEX": [ re.compile("^v\d+\.\d+\.\d+\.\d+-\d+$"), re.compile("^v\d+\.\d+\.\d+\.\d+-\d+\.el9$") # EL9 Transition format ] @@ -48,7 +46,7 @@ DEFINED_INPUT_VARS = [ "CUSTOM_EOS_TAG", "CUSTOM_CTA_TAG", "CTA_BUILD_ID", "CTA_PIPELINE_NAME", "GIT_STRATEGY", "GIT_SUBMODULE_STRATEGY", "GIT_DEPTH", "GIT_SUBMODULE_DEPTH", - "IMAGE_DOCKER_IMAGE_BUILDER", "IMAGE_CPPCHECK", + "IMAGE_DOCKER_IMAGE_BUILDER", "IMAGE_GITLAB_RELEASE_CLI", "IMAGE_BUILD", "IMAGE_TEST", "IMAGE_RELEASE", "IMAGE_ANALYSIS" ] @@ -126,65 +124,15 @@ def validate_default(ci_input_vars): pass -def validate_eos_regr_against_cta_main(ci_input_vars): - """ - Validation for the pipeline type `EOS_REGR_AGAINST_CTA_MAIN`. Checks the XRootD and EOS - RPM packages. - - XRootD is fetched from the stable repo. - - EOS version should be available in the testing repo as it comes from - their latest tag. - - :param ci_input_vars: CI input variables dictionary received by the program. - """ - print("Validating XRootD...") - _check_remote_rpm(ci_input_vars, - "CUSTOM_XRD_TAG", - "CUSTOM_XRD_TAG_REGEX", - ["xrd", "https://xrootd.web.cern.ch/repo/stable/el9/x86_64"], - "xrootd-server", - ci_input_vars["CUSTOM_XRD_TAG"]) - print("XRootD validation passed") - - print("Validating EOS...") - _check_remote_rpm(ci_input_vars, - "CUSTOM_EOS_TAG", - "CUSTOM_EOS_TAG_REGEX", - ["eos", "https://storage-ci.web.cern.ch/storage-ci/eos/diopside/tag/testing/el-9/x86_64/"], - "eos-server", - ci_input_vars["CUSTOM_EOS_TAG"]) - print("EOS validation passed") - - def validate_eos_regr_against_cta_tag(ci_input_vars): """ - Validation for the pipeline type `EOS_REGR_AGAINST_CTA_TAG`. Checks the XRootD, EOS - and CTA RPM packages. - - XRootD is fetched from the stable repo. - - EOS version should be available in the testing repo as it comes from - their latest tag. + Validation for the pipeline type `EOS_REGR_AGAINST_CTA_TAG`. Checks that the desired + CTA RPM packages (associated with the given tag) are available. - CTA should be available in the testing public repository as the latest available version in there should match the latest version in production. :param ci_input_vars: CI input variables dictionary received by the program. """ - print("Validating XRootD...") - _check_remote_rpm(ci_input_vars, - "CUSTOM_XRD_TAG", - "CUSTOM_XRD_TAG_REGEX", - ["xrd", "https://xrootd.web.cern.ch/repo/stable/el9/x86_64"], - "xrootd-server", - ci_input_vars["CUSTOM_XRD_TAG"]) - print("XRootD validation passed") - - print("Validating EOS...") - _check_remote_rpm(ci_input_vars, - "CUSTOM_EOS_TAG", - "CUSTOM_EOS_TAG_REGEX", - ["eos", "https://storage-ci.web.cern.ch/storage-ci/eos/diopside/tag/testing/el-9/x86_64/"], - "eos-server", - ci_input_vars["CUSTOM_EOS_TAG"]) - print("EOS validation passed") - print("Validating CTA...") _check_remote_rpm(ci_input_vars, "CUSTOM_CTA_TAG", @@ -197,26 +145,8 @@ def validate_eos_regr_against_cta_tag(ci_input_vars): def validate_ctageneric_image(ci_input_vars): """ Validation for the pipeline type `CTAGENERIC_IMAGE`. Checks that the desired - XRootD, EOS and CTA RPM packages are available. + CTA RPM packages are available. """ - print("Validating XRootD...") - _check_remote_rpm(ci_input_vars, - "CUSTOM_XRD_TAG", - "CUSTOM_XRD_TAG_REGEX", - ["xrd", "https://xrootd.web.cern.ch/repo/stable/el9/x86_64"], - "xrootd-server", - ci_input_vars["CUSTOM_XRD_TAG"]) - print("XRootD validation passed") - - print("Validating EOS...") - _check_remote_rpm(ci_input_vars, - "CUSTOM_EOS_TAG", - "CUSTOM_EOS_TAG_REGEX", - ["eos", "https://storage-ci.web.cern.ch/storage-ci/eos/diopside/tag/testing/el-9/x86_64/"], - "eos-server", - ci_input_vars["CUSTOM_EOS_TAG"]) - print("EOS validation passed") - print("Validating CTA...") _check_remote_rpm(ci_input_vars, "CUSTOM_CTA_TAG",