diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 66a87a857349335ac319dda28dad5b4b0b2546d2..0b161e76970233b2b65c00f9e7b8c927ce916fa2 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -18,6 +18,7 @@ - cta/CTA#647 - Upgrade eos-5 to eos-5.2.21-1 - cta/CTA#658 - Revert eviction test to original behaviour and sleeps for Mgm syncer to catch up - cta/CTA#659 - Allow running only branch's systemtests in CI using ctageneric image from main +- cta/CTA#663 - Setup dev env containers for Alma9 ### Code Quality - cta/CTA#575 - Remove rados metrics logging diff --git a/continuousintegration/ci_runner/development/cta/alma9/Dockerfile b/continuousintegration/ci_runner/development/cta/alma9/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..3e158dfaf9d540305406a853e8020de5444434f3 --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/Dockerfile @@ -0,0 +1,59 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2023 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base + +ARG CTA_VERSION=main +RUN set -ex; \ + echo "CTA version is ${CTA_VERSION}"; + +COPY ./repos/ /etc/yum.repos.d/ + +RUN mkdir -p shared; + +RUN set -ex; \ + dnf -y update; \ + dnf -y install epel-release almalinux-release-devel; \ + dnf -y install git vim rsync wget cmake3 dnf-utils ccache openssh-server; \ + dnf -y group install "Development Tools"; + +RUN set -ex; \ + git clone https://gitlab.cern.ch/cta/CTA.git cta; \ + cd cta; \ + git checkout ${CTA_VERSION}; \ + git submodule sync --recursive && git submodule update --init --recursive; + +RUN set -ex; \ + cd cta; \ + ./continuousintegration/docker/ctafrontend/alma9/installOracle21.sh; + +RUN set -ex; \ + mkdir build_srpm; \ + cd build_srpm; \ + cmake3 -DPackageOnly:Bool=true ../cta; \ + make cta_srpm; + +RUN set -ex; \ + yum-builddep -y --nogpgcheck build_srpm/RPM/SRPMS/*; + +RUN set -ex; \ + rm -rf build_srpm; + +# Setup SSH server +RUN /usr/bin/ssh-keygen -A; \ + rm /run/nologin; + +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/continuousintegration/ci_runner/development/cta/alma9/Readme.md b/continuousintegration/ci_runner/development/cta/alma9/Readme.md new file mode 100644 index 0000000000000000000000000000000000000000..88900ec9a222e9d42582dadb7f3935c6786d05c8 --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/Readme.md @@ -0,0 +1,85 @@ +### How to use + +#### 1. Build the docker image + +This script will build the docker image that can be used for CTA development: + +```bash +./prepare_ctadev_image.sh [<cta_version>] +``` +The `<cta_version>` is the version of CTA to get the dependencies from. It can be a commit, branch or a tag. \ +If not defined, `<cta_version>` will default to `main`. + +#### 2. Run the docker image + +To list the latest built images, run: +```bash +podman images +``` + +To launch one of them (detached), proceed with: +```bash +./start_ctadev.sh [-p <port>] [-v <mount>] <image> +``` + +The `<port>` parameter can be used to forward port `<port>` on the localhost to port `22` on the container. This can be used by ssh clients to access the container through a network. \ +The `<mount>` parameter can be used to mount a local directory to the container directory `/shared`. +The `<image>` is the image of the container that we want to run. + +##### 2.1 Setup `ssh` server + +By default, the image comes with `openssh-server` installed and running. \ +In order to accept SSH connections, we need to upload a public key to the container. + +Example: +```bash +# Do not upload private key! +podman cp <id_rsa_or_other.pub> <container_id>:/root/.ssh/authorized_keys +``` + +And also open the port `<port>` on the host: +```bash +sudo firewall-cmd --permanent --add-port=<port>/tcp +sudo firewall-cmd --reload +sudo firewall-cmd --list-all +``` + +With this completed, accessing the container is simple: +```bash +# Access the container on host <hostname> and port <port> +ssh -p <port> root@<hostname> +``` + +#### 3. Build CTA + +Once inside the container, building CTA is simple! +```bash +git clone https://gitlab.cern.ch/cta/CTA.git +cd CTA/ +git submodule update --init --recursive +mkdir ../CTA_build +cd ../CTA_build +cmake3 ../CTA +make -j 4 +``` + +To validate that CTA was build correctly, run the unit tests inside `CTA_build`: +```bash +cd tests/ +./cta-unitTests +``` + +##### 3.1. Build CTA RPMs + +Building the RPMs follows a similar procedure: +```bash +mkdir CTA_rpm +cd CTA_rpm +cmake3 ../CTA +make cta_rpm +``` + +In order to make the RPMs available to the host dev machine, copy them to the `shared` directory (mounted volume): +```bash +cp CTA_rpm/RPM/RPMS/x86_64/* /root/shared/ +``` \ No newline at end of file diff --git a/continuousintegration/ci_runner/development/cta/alma9/prepare_ctadev_image.sh b/continuousintegration/ci_runner/development/cta/alma9/prepare_ctadev_image.sh new file mode 100755 index 0000000000000000000000000000000000000000..37a44d8b4d17ed48aecd692e4fdc650f6d5fcb82 --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/prepare_ctadev_image.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2023 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +image_tag="dev" +if [ -n "$1" ]; then + version_arg="--build-arg CTA_VERSION=$1" + image_tag="${image_tag}-$1" +fi + +# Pass to docker the version of CTA to build +podman build --platform linux/amd64 -f Dockerfile -t ctadev:${image_tag} ${version_arg} . diff --git a/continuousintegration/ci_runner/development/cta/alma9/repos/ceph.repo b/continuousintegration/ci_runner/development/cta/alma9/repos/ceph.repo new file mode 100644 index 0000000000000000000000000000000000000000..3ae1aa3b4069423515f8f2cfe985c7e6fffaff8d --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/repos/ceph.repo @@ -0,0 +1,6 @@ +[ceph] +name=Ceph packages for $basearch +baseurl=https://download.ceph.com/rpm-17.2.7/el9/x86_64/ +enabled=1 +priority=2 +gpgcheck=0 diff --git a/continuousintegration/ci_runner/development/cta/alma9/repos/database.repo b/continuousintegration/ci_runner/development/cta/alma9/repos/database.repo new file mode 100644 index 0000000000000000000000000000000000000000..4f03b0f0f2fbaec38b07936c496ca7abe720550a --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/repos/database.repo @@ -0,0 +1,7 @@ +[dbclients9el-stable] +name=IT-DB database client tools [stable] +baseurl=http://linuxsoft.cern.ch/internal/repos/dbclients9el-stable/x86_64/os +enabled=1 +gpgcheck=True +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kojiv2 +priority=5 \ No newline at end of file diff --git a/continuousintegration/ci_runner/development/cta/alma9/repos/eos-citrine-depend.repo b/continuousintegration/ci_runner/development/cta/alma9/repos/eos-citrine-depend.repo new file mode 100644 index 0000000000000000000000000000000000000000..0070dc32e004ca0a39a2bc64a21db4f887fc1818 --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/repos/eos-citrine-depend.repo @@ -0,0 +1,20 @@ +[cta-eos-diopside] +name=EOS diopside releases from EOS project +baseurl=https://storage-ci.web.cern.ch/storage-ci/eos/diopside/tag/testing/el-$releasever/$basearch/ +enabled=1 +gpgcheck=0 +priority=4 + +[cta-eos-diopside-depend] +name=dependencies for EOS diopside releases from EOS project +baseurl=http://storage-ci.web.cern.ch/storage-ci/eos/diopside-depend/el-$releasever/$basearch/ +enabled=1 +gpgcheck=0 +priority=4 + +[cta-eos-quarkdb] +name=EOS quarkdb releases from EOS project +baseurl=http://storage-ci.web.cern.ch/storage-ci/quarkdb/tag/el8/x86_64/ +enabled=1 +gpgcheck=0 +priority=10 diff --git a/continuousintegration/ci_runner/development/cta/alma9/repos/xrootd-stable.repo b/continuousintegration/ci_runner/development/cta/alma9/repos/xrootd-stable.repo new file mode 100644 index 0000000000000000000000000000000000000000..2233f709391714990b6fd78290f88950eeeb9a75 --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/repos/xrootd-stable.repo @@ -0,0 +1,9 @@ +[xrootd-stable] +name=XRootD Stable Releases for Enterprise Linux $releasever +baseurl=https://xrootd.web.cern.ch/repo/stable/el$releasever/x86_64/ +gpgkey=https://xrootd.web.cern.ch/sw/releases/RPM-GPG-KEY.txt +metadata_expire=7d +repo_gpgcheck=0 +priority=10 +gpgcheck=0 +enabled=1 diff --git a/continuousintegration/ci_runner/development/cta/alma9/start_ctadev.sh b/continuousintegration/ci_runner/development/cta/alma9/start_ctadev.sh new file mode 100755 index 0000000000000000000000000000000000000000..e74e077a57b0c8d904cfde11f1db475c177797f8 --- /dev/null +++ b/continuousintegration/ci_runner/development/cta/alma9/start_ctadev.sh @@ -0,0 +1,41 @@ +#!/bin/bash -e + +usage() { cat <<EOF 1>&2 +Usage: $0 [-p <port>] [-v <volumemount>] <image_tag> +Options: + <image_tag> Image to run + -p SSH port (need to be opened on firewall) + -v Volume mount +EOF +exit 1 +} + +port_arg="" +volume_mount_arg="" + +while getopts "p:v:" o; do + case "${o}" in + p) + port_arg="-p ${OPTARG}:22" + ;; + v) + volume_mount_arg="-v ${OPTARG}:/shared:Z" + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "$1" ]; then + usage +fi + +image_tag=$1 +echo "podman run -d -it ${port_arg} ${volume_mount_arg} ${image_tag}" +container_id=$(podman run -d -it ${port_arg} ${volume_mount_arg} ${image_tag}) +echo "Container ID: ${container_id:0:12}" +echo "" +echo "Container running in detached mode" +echo "To access, add your public key to ${container_id} and ssh to port ${port}" diff --git a/continuousintegration/ci_runner/development/eos/alma9/Dockerfile b/continuousintegration/ci_runner/development/eos/alma9/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d669992db169d14d7290988b0724d3d89f05e7f2 --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/Dockerfile @@ -0,0 +1,55 @@ +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2023 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base + +ARG EOS_VERSION=master +RUN set -ex; \ + echo "EOS version is ${EOS_VERSION}"; + +COPY ./repos/ /etc/yum.repos.d/ + +RUN mkdir -p shared; + +RUN set -ex; \ + dnf -y update; \ + dnf -y install epel-release almalinux-release-devel; \ + dnf -y install git vim rsync cmake3 dnf-utils ccache openssh-server; \ + dnf -y group install "Development Tools"; + +RUN set -ex; \ + git clone https://gitlab.cern.ch/dss/eos.git eos; \ + cd eos; \ + git checkout ${EOS_VERSION}; \ + git submodule sync --recursive && git submodule update --init --recursive; + +RUN set -ex; \ + mkdir build_srpm; \ + cd build_srpm; \ + cmake3 -DPACKAGEONLY:Bool=true ../eos; \ + make srpm; + +RUN set -ex; \ + yum-builddep -y --nogpgcheck build_srpm/SRPMS/*; + +RUN set -ex; \ + rm -rf build_srpm; + +# Setup SSH server for remote access +RUN /usr/bin/ssh-keygen -A; \ + rm /run/nologin; + +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/continuousintegration/ci_runner/development/eos/alma9/Readme.md b/continuousintegration/ci_runner/development/eos/alma9/Readme.md new file mode 100644 index 0000000000000000000000000000000000000000..b71d5d0ff623c82eb51cfd8cbd328cbd69b2a7ff --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/Readme.md @@ -0,0 +1,93 @@ +### How to use + +#### 1. Build the docker image + +This script will build the docker image that can be used for EOS development: + +```bash +./prepare_eosdev_image.sh [<eos_version>] +``` +The `<eos_version>` is the version of EOS to get the dependencies from. It can be a commit, branch or a tag. \ +If not defined, `<eos_version>` will default to `master`. However, for EOS development, it's recommended to use a tagged commit instead of `master` (it's guaranteed to be more stable). + +#### 2. Run the docker image + +To list the latest built images, run: +```bash +podman images +``` + +To launch one of them (detached), proceed with: +```bash +./start_ctadev.sh [-p <port>] [-v <mount>] <image> +``` + +The `<port>` parameter can be used to forward port `<port>` on the localhost to port `22` on the container. This can be used by ssh clients to access the container through a network. \ +The `<mount>` parameter can be used to mount a local directory to the container directory `/shared`. +The `<image>` is the image of the container that we want to run. + +##### 2.1 Setup `ssh` server + +By default, the image comes with `openssh-server` installed and running. \ +In order to accept SSH connections, we need to upload a public key to the container. + +Example: +```bash +# Do not upload private key! +podman cp <id_rsa_or_other.pub> <container_id>:/root/.ssh/authorized_keys +``` + +And also open the port `<port>` on the host: +```bash +sudo firewall-cmd --permanent --add-port=<port>/tcp +sudo firewall-cmd --reload +sudo firewall-cmd --list-all +``` + +With this completed, accessing the container is simple: +```bash +# Access the container on host <hostname> and port <port> +ssh -p <port> root@<hostname> +``` + +#### 3. Build EOS + +Once inside the container, building EOS is simple! +```bash +git clone https://gitlab.cern.ch/dss/eos.git +cd eos/ +git submodule update --init --recursive +mkdir ../eos_build +cd ../eos_build +cmake3 ../eos +make -j 4 +``` + +To validate that CTA was build correctly, run the unit tests inside `eos_build`: +```bash +# We need to set the EOS libraries path, unless we have installed them +cd eos_build +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/common:$(pwd)/mq +# Run the tests +cd unit_tests/ +./eos-unit-tests +``` + +##### 3.1. Build EOS RPMs + +Building the RPMs follows a similar procedure: +```bash +mkdir eos_rpm +cd eos_rpm +cmake3 ../eos +make rpm +``` + +In order to make the RPMs available to the host dev machine, copy them to the `shared` directory (mounted volume): +```bash +cp eos_rpm/RPMS/x86_64/* /root/shared/ +``` + +#### 4. Other notes + +For EOS development it's best to modify the files in the same development development environment where they are build (ie. do both changes and build inside the container). Developing on a local machine, and synching the files to be built on a remote host will probably lead to build issues. diff --git a/continuousintegration/ci_runner/development/eos/alma9/prepare_eosdev_image.sh b/continuousintegration/ci_runner/development/eos/alma9/prepare_eosdev_image.sh new file mode 100755 index 0000000000000000000000000000000000000000..9ec969e4fe898b09fdd7403b844bf5f463903b1d --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/prepare_eosdev_image.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2023 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +image_tag="dev" +if [ -n "$1" ]; then + version_arg="--build-arg EOS_VERSION=$1" + image_tag="${image_tag}-$1" +fi + +# Pass to docker the version of EOS to build +podman build --platform linux/amd64 -f Dockerfile -t eosdev:${image_tag} ${version_arg} . diff --git a/continuousintegration/ci_runner/development/eos/alma9/repos/ceph.repo b/continuousintegration/ci_runner/development/eos/alma9/repos/ceph.repo new file mode 100644 index 0000000000000000000000000000000000000000..3ae1aa3b4069423515f8f2cfe985c7e6fffaff8d --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/repos/ceph.repo @@ -0,0 +1,6 @@ +[ceph] +name=Ceph packages for $basearch +baseurl=https://download.ceph.com/rpm-17.2.7/el9/x86_64/ +enabled=1 +priority=2 +gpgcheck=0 diff --git a/continuousintegration/ci_runner/development/eos/alma9/repos/database.repo b/continuousintegration/ci_runner/development/eos/alma9/repos/database.repo new file mode 100644 index 0000000000000000000000000000000000000000..4f03b0f0f2fbaec38b07936c496ca7abe720550a --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/repos/database.repo @@ -0,0 +1,7 @@ +[dbclients9el-stable] +name=IT-DB database client tools [stable] +baseurl=http://linuxsoft.cern.ch/internal/repos/dbclients9el-stable/x86_64/os +enabled=1 +gpgcheck=True +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-kojiv2 +priority=5 \ No newline at end of file diff --git a/continuousintegration/ci_runner/development/eos/alma9/repos/eos-citrine-depend.repo b/continuousintegration/ci_runner/development/eos/alma9/repos/eos-citrine-depend.repo new file mode 100644 index 0000000000000000000000000000000000000000..0070dc32e004ca0a39a2bc64a21db4f887fc1818 --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/repos/eos-citrine-depend.repo @@ -0,0 +1,20 @@ +[cta-eos-diopside] +name=EOS diopside releases from EOS project +baseurl=https://storage-ci.web.cern.ch/storage-ci/eos/diopside/tag/testing/el-$releasever/$basearch/ +enabled=1 +gpgcheck=0 +priority=4 + +[cta-eos-diopside-depend] +name=dependencies for EOS diopside releases from EOS project +baseurl=http://storage-ci.web.cern.ch/storage-ci/eos/diopside-depend/el-$releasever/$basearch/ +enabled=1 +gpgcheck=0 +priority=4 + +[cta-eos-quarkdb] +name=EOS quarkdb releases from EOS project +baseurl=http://storage-ci.web.cern.ch/storage-ci/quarkdb/tag/el8/x86_64/ +enabled=1 +gpgcheck=0 +priority=10 diff --git a/continuousintegration/ci_runner/development/eos/alma9/repos/xrootd-stable.repo b/continuousintegration/ci_runner/development/eos/alma9/repos/xrootd-stable.repo new file mode 100644 index 0000000000000000000000000000000000000000..2233f709391714990b6fd78290f88950eeeb9a75 --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/repos/xrootd-stable.repo @@ -0,0 +1,9 @@ +[xrootd-stable] +name=XRootD Stable Releases for Enterprise Linux $releasever +baseurl=https://xrootd.web.cern.ch/repo/stable/el$releasever/x86_64/ +gpgkey=https://xrootd.web.cern.ch/sw/releases/RPM-GPG-KEY.txt +metadata_expire=7d +repo_gpgcheck=0 +priority=10 +gpgcheck=0 +enabled=1 diff --git a/continuousintegration/ci_runner/development/eos/alma9/start_eosdev.sh b/continuousintegration/ci_runner/development/eos/alma9/start_eosdev.sh new file mode 100755 index 0000000000000000000000000000000000000000..e74e077a57b0c8d904cfde11f1db475c177797f8 --- /dev/null +++ b/continuousintegration/ci_runner/development/eos/alma9/start_eosdev.sh @@ -0,0 +1,41 @@ +#!/bin/bash -e + +usage() { cat <<EOF 1>&2 +Usage: $0 [-p <port>] [-v <volumemount>] <image_tag> +Options: + <image_tag> Image to run + -p SSH port (need to be opened on firewall) + -v Volume mount +EOF +exit 1 +} + +port_arg="" +volume_mount_arg="" + +while getopts "p:v:" o; do + case "${o}" in + p) + port_arg="-p ${OPTARG}:22" + ;; + v) + volume_mount_arg="-v ${OPTARG}:/shared:Z" + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +if [ -z "$1" ]; then + usage +fi + +image_tag=$1 +echo "podman run -d -it ${port_arg} ${volume_mount_arg} ${image_tag}" +container_id=$(podman run -d -it ${port_arg} ${volume_mount_arg} ${image_tag}) +echo "Container ID: ${container_id:0:12}" +echo "" +echo "Container running in detached mode" +echo "To access, add your public key to ${container_id} and ssh to port ${port}"