From 99310d98ac0a11da7a890b033b477f783e6aea0b Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Fri, 28 Jun 2024 12:29:54 +0200 Subject: [PATCH 1/8] edited gitlab CI for etf-dune for running from every branch, hadolint for etf-dune Dockerfile --- .gitlab-ci.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77ff978..6dfd537 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,16 +1,24 @@ variables: DOCKER_VERSION: "17.05" -build:etf_dune_qa: +build:etf_dune: stage: build variables: - IMAGE_DESTINATION: gitlab-registry.cern.ch/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/etf-dune:qa + IMAGE_DESTINATION: "gitlab-registry.cern.ch/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/etf-dune:" image: # The kaniko debug image is recommended because it has a shell, and a shell is required for an image to be used with GitLab CI/CD. name: gcr.io/kaniko-project/executor:debug entrypoint: [""] - environment: master + environment: $CI_COMMIT_REF_NAME script: + # Determine the tag based on the branch name + - | + if [ "$CI_COMMIT_REF_NAME" == "master" ]; then + TAG="qa" + else + TAG="$CI_COMMIT_REF_NAME" + fi + IMAGE_DESTINATION="$IMAGE_DESTINATION$TAG" # Prepare Kaniko configuration file - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json # Build and push the image from the Dockerfile at the root of the project. @@ -18,11 +26,11 @@ build:etf_dune_qa: # Print the full registry path of the pushed image - echo "Image pushed successfully to ${IMAGE_DESTINATION}" only: - - master + - branches deploy:production: dependencies: - - build:etf_dune_qa + - build:etf_dune stage: deploy image: # Use the ':debug' image as it provides a shell, which is a requirement for GitLab CI @@ -33,3 +41,10 @@ deploy:production: - crane auth login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - crane validate --remote "gitlab-registry.cern.ch/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/etf-dune:qa" - crane tag "gitlab-registry.cern.ch/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/etf-dune:qa" prod + +lint:etf_dune: + stage: test + image: "hadolint/hadolint:latest-debian" + script: + - hadolint Dockerfile + allow_failure: True -- GitLab From aa08144c07b8f119b995e4dc17458aa76f5e924e Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Mon, 1 Jul 2024 15:45:37 +0200 Subject: [PATCH 2/8] upgrading etf-dune container to podman - systemd, restructuring following etf-alice format --- Dockerfile | 66 +++++++++++++---------------------------- config/add-keys.sh | 6 ++++ config/grid-env_ipv6.sh | 14 --------- docker-entrypoint.sh | 23 +++++++++++--- 4 files changed, 46 insertions(+), 63 deletions(-) create mode 100755 config/add-keys.sh delete mode 100644 config/grid-env_ipv6.sh diff --git a/Dockerfile b/Dockerfile index 33ffb20..6a00c20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,60 +1,36 @@ -FROM gitlab-registry.cern.ch/etf/docker/etf-exp:qa - -LABEL maintainer="Marian Babik <Marian.Babik@cern.ch>" -LABEL description="WLCG ETF DUNE" -LABEL version="1.0" +FROM gitlab-registry.cern.ch/etf/docker/etf-base:el9 ENV NSTREAM_ENABLED=0 -# OSG Middleware -RUN yum -y install yum-priorities -RUN yum -y clean all -RUN yum -y install https://repo.opensciencegrid.org/osg/3.6/osg-3.6-el7-release-latest.rpm -#RUN rpm -Uvh https://repo.opensciencegrid.org/osg/3.4/osg-3.4-el7-release-latest.rpm -RUN sed "7i priority=99" -i /etc/yum.repos.d/epel.repo - -# Core deps -RUN yum -y install voms voms-clients-java oidc-agent-cli - -# Condor client -RUN yum -y install condor condor-python - -# ETF base plugins -RUN yum -y install python-pip -RUN yum -y install pexpect -RUN yum -y install python-jess python-wnfm nagios-plugins-tokens nagios-plugins-globus nagios-plugins - -# ETF WN-qFM payload -RUN mkdir -p /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages -RUN mkdir -p /usr/libexec/grid-monitoring/wnfm/bin -RUN cp /usr/bin/etf_wnfm /usr/libexec/grid-monitoring/wnfm/bin/ -#RUN cp -r /usr/lib/python2.7/site-packages/pexpect /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages -#RUN cp -r /usr/lib/python2.7/site-packages/ptyprocess /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages -RUN cp -r /usr/lib/python2.7/site-packages/wnfm /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages -#RUN cp /usr/lib/python2.7/site-packages/argparse.py /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages/ - -# ETF Streaming -RUN mkdir -p /var/spool/nstream/outgoing && chmod 777 /var/spool/nstream/outgoing -RUN mkdir /etc/stompclt +# Streaming COPY ./config/ocsp_handler.cfg /etc/nstream/ -# MW env -COPY ./config/grid-env.sh /etc/profile.d/ -RUN echo "source /etc/profile.d/grid-env.sh" >> /opt/omd/sites/$CHECK_MK_SITE/.profile +################################ +# OSG Middleware +# RUN dnf config-manager --set-enabled epel \ +# && dnf -y install nagios-plugins \ + # ETF WN-qFM payload + # && mkdir -p /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ + # && mkdir -p /usr/libexec/grid-monitoring/wnfm/bin \ + # && cp /usr/bin/etf_wnfm /usr/libexec/grid-monitoring/wnfm/bin/ \ + # && cp -r /usr/lib/python3.9/site-packages/wnfm /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ + # ETF Streaming + # && dnf clean all + -# DUNE config +COPY ./src/probes /usr/libexec/grid-monitoring/probes/org.dune/wnjob/org.dune/probes/org.dune +#################################### + +# ETF local checks COPY config/dune_plugin.py /usr/lib/ncgx/x_plugins/ COPY ./config/wlcg_dune.cfg /etc/ncgx/metrics.d/ -# DUNE payload RUN mkdir -p /usr/libexec/grid-monitoring/probes/org.dune/wnjob -COPY ./src/probes /usr/libexec/grid-monitoring/probes/org.dune/wnjob/org.dune/probes/org.dune -# DUNE storage testing -#COPY ./src/srmvometrics.py /usr/libexec/grid-monitoring/probes/org.dune/ # ETF config COPY ./config/dune_checks.cfg /etc/ncgx/conf.d/ COPY ./config/ncgx.cfg /etc/ncgx/ +COPY ./config/add-keys.sh /opt/omd/sites/etf/.oidc-agent/ + EXPOSE 80 443 6557 -COPY ./docker-entrypoint.sh / -ENTRYPOINT /docker-entrypoint.sh +ENTRYPOINT ["/usr/sbin/init"] diff --git a/config/add-keys.sh b/config/add-keys.sh new file mode 100755 index 0000000..fe53990 --- /dev/null +++ b/config/add-keys.sh @@ -0,0 +1,6 @@ +#!/bin/bash +source /opt/omd/sites/etf/.oidc-agent/oidc-env.sh +cp -f /etc/grid-security/tokens/* /opt/omd/sites/etf/.oidc-agent/ +/usr/bin/oidc-add --pw-file=/opt/omd/sites/etf/.oidc-agent/etf_dune_ce.key etf_dune_ce +/usr/lib64/nagios/plugins/refresh_token -t 7200 --token-config etf-dune --token-time 345600 --aud /var/lib/gridprobes/dune/scondor/ + diff --git a/config/grid-env_ipv6.sh b/config/grid-env_ipv6.sh deleted file mode 100644 index a7fb6c6..0000000 --- a/config/grid-env_ipv6.sh +++ /dev/null @@ -1,14 +0,0 @@ -export GLITE_LOCATION="/usr" -export GLOBUS_TCP_PORT_RANGE="20000,25000" -export GLITE_LOCATION_VAR="/var" -export GLOBUS_HOSTNAME="`hostname`" -export MYPROXY_SERVER="myproxy.cern.ch" -export LCG_LOCATION="/usr" -export GRID_ENV_LOCATION="/usr/libexec" -export LCG_GFAL_INFOSYS="top-bdii.cern.ch:2170" -export GLOBUS_FTP_CLIENT_IPV6="true" -export GLOBUS_IO_IPV6="true" - -if [[ -f /opt/omd/sites/etf/.oidc-agent/oidc-env.sh ]]; then - source /opt/omd/sites/etf/.oidc-agent/oidc-env.sh -fi \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index ab6f2c8..ad2224b 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,8 +1,23 @@ #!/bin/bash -#!/bin/bash +set -e +# Source the initialization script source /usr/bin/etf-init.sh +function start_systemd() { + echo "Starting systemd..." + # Ensure cgroups directories exist and are properly mounted + mkdir -p /sys/fs/cgroup/systemd + mountpoint -q /sys/fs/cgroup/systemd || mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd || { + echo "Failed to mount cgroup: permission denied. Trying to continue..." + } + + # Start systemd + exec /usr/sbin/init & + # Wait for systemd to fully start + sleep 5 +} + cat << "EOF" _____ _____ _____ ____ _ _ _ _ _____ | ____|_ _| ___| | _ \| | | | \ | | ____| @@ -16,7 +31,7 @@ print_header etf_update -start_xinetd +start_systemd etf_init @@ -41,8 +56,8 @@ etf_start start_oidc_agent echo "Fetching DUNE credentials ..." -su etf -c "/usr/lib/nagios/plugins/globus/refresh_proxy --vo-fqan /dune/Role=Production --myproxyuser nagios -H myproxy.cern.ch -t 120 --key /opt/omd/sites/etf/etc/nagios/globus/etf_srv_key.pem --vo dune --lifetime 24 --name NagiosRetrieve-ETF-dune -x /opt/omd/sites/etf/etc/nagios/globus/userproxy.pem--dune-Role_production --cert /opt/omd/sites/etf/etc/nagios/globus/etf_srv_cert.pem" -su etf -c "/usr/lib/nagios/plugins/globus/refresh_proxy --vo-fqan /dune/Role=ETF --myproxyuser nagios -H myproxy.cern.ch -t 120 --key /opt/omd/sites/etf/etc/nagios/globus/etf_srv_key.pem --vo dune --lifetime 24 --name NagiosRetrieve-ETF-dune -x /opt/omd/sites/etf/etc/nagios/globus/userproxy.pem--dune-Role_etf --cert /opt/omd/sites/etf/etc/nagios/globus/etf_srv_cert.pem" +su etf -c "/usr/lib/nagios/plugins/globus/refresh_proxy --vo-fqan /dune/Role=Production --myproxyuser nagios -H myproxy.cern.ch -t 120 --key /opt/omd/sites/etf/etc/nagios/globus/etf_srv_key.pem --vo dune --lifetime 24 --name NagiosRetrieve-ETF-dune -x /opt/omd/sites/etf/etc/nagios/globus/userproxy.pem --dune-Role_production --cert /opt/omd/sites/etf/etc/nagios/globus/etf_srv_cert.pem" +su etf -c "/usr/lib/nagios/plugins/globus/refresh_proxy --vo-fqan /dune/Role=ETF --myproxyuser nagios -H myproxy.cern.ch -t 120 --key /opt/omd/sites/etf/etc/nagios/globus/etf_srv_key.pem --vo dune --lifetime 24 --name NagiosRetrieve-ETF-dune -x /opt/omd/sites/etf/etc/nagios/globus/userproxy.pem --dune-Role_etf --cert /opt/omd/sites/etf/etc/nagios/globus/etf_srv_cert.pem" echo "Initialising tokens ..." #su - etf -c "oidc-add --pw-file=/opt/omd/sites/etf/.oidc-agent/etf_dune_ce.key etf_dune_ce" -- GitLab From 2242c141c473fa2dd6d6fe46147de4ba9a939983 Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Fri, 18 Oct 2024 15:13:23 +0200 Subject: [PATCH 3/8] adding the grid env --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6a00c20..696b0e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ COPY ./config/ocsp_handler.cfg /etc/nstream/ COPY ./src/probes /usr/libexec/grid-monitoring/probes/org.dune/wnjob/org.dune/probes/org.dune #################################### +# MW env +COPY ./config/grid-env.sh /etc/profile.d/ +RUN echo "source /etc/profile.d/grid-env.sh" >> /opt/omd/sites/$CHECK_MK_SITE/.profile + # ETF local checks COPY config/dune_plugin.py /usr/lib/ncgx/x_plugins/ COPY ./config/wlcg_dune.cfg /etc/ncgx/metrics.d/ -- GitLab From 8ea1939f6752c93b7ddd42e145700d42cac48caf Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Tue, 22 Oct 2024 17:00:27 +0200 Subject: [PATCH 4/8] fix add-keys --- config/add-keys.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/add-keys.sh b/config/add-keys.sh index fe53990..abf4171 100755 --- a/config/add-keys.sh +++ b/config/add-keys.sh @@ -2,5 +2,5 @@ source /opt/omd/sites/etf/.oidc-agent/oidc-env.sh cp -f /etc/grid-security/tokens/* /opt/omd/sites/etf/.oidc-agent/ /usr/bin/oidc-add --pw-file=/opt/omd/sites/etf/.oidc-agent/etf_dune_ce.key etf_dune_ce -/usr/lib64/nagios/plugins/refresh_token -t 7200 --token-config etf-dune --token-time 345600 --aud /var/lib/gridprobes/dune/scondor/ +/usr/lib64/nagios/plugins/refresh_token -t 890 --token-config dune-ce --token-time 21600 -x /opt/omd/sites/etf/etc/nagios/globus/dune.token -- GitLab From a07cabd7acae998daeb0d3e3f6a66889049dd958 Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Wed, 23 Oct 2024 11:13:07 +0200 Subject: [PATCH 5/8] WN-qFM payloads and comment out ARC --- Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 696b0e5..7c0fb31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,17 +5,21 @@ ENV NSTREAM_ENABLED=0 # Streaming COPY ./config/ocsp_handler.cfg /etc/nstream/ +# # ARC config +# RUN mkdir /opt/omd/sites/$CHECK_MK_SITE/.arc +# COPY ./config/client.conf /opt/omd/sites/$CHECK_MK_SITE/.arc/ +# RUN chown -R $CHECK_MK_SITE /opt/omd/sites/$CHECK_MK_SITE/.arc/ + ################################ # OSG Middleware # RUN dnf config-manager --set-enabled epel \ # && dnf -y install nagios-plugins \ # ETF WN-qFM payload - # && mkdir -p /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ - # && mkdir -p /usr/libexec/grid-monitoring/wnfm/bin \ - # && cp /usr/bin/etf_wnfm /usr/libexec/grid-monitoring/wnfm/bin/ \ - # && cp -r /usr/lib/python3.9/site-packages/wnfm /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ - # ETF Streaming - # && dnf clean all +RUN mkdir -p /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ + && mkdir -p /usr/libexec/grid-monitoring/wnfm/bin \ + && cp /usr/bin/etf_wnfm /usr/libexec/grid-monitoring/wnfm/bin/ \ + && cp -r /usr/lib/python3.9/site-packages/wnfm /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ + && dnf clean all COPY ./src/probes /usr/libexec/grid-monitoring/probes/org.dune/wnjob/org.dune/probes/org.dune -- GitLab From 69e2662660f5a50478bb85c784f675f5087fc5c7 Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Thu, 31 Oct 2024 07:23:11 +0100 Subject: [PATCH 6/8] some dune changes --- Dockerfile | 31 +++++++++++++++++-------------- config/add-keys.sh | 2 +- config/grid-env.sh | 1 - 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7c0fb31..2d60917 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,30 +5,33 @@ ENV NSTREAM_ENABLED=0 # Streaming COPY ./config/ocsp_handler.cfg /etc/nstream/ -# # ARC config -# RUN mkdir /opt/omd/sites/$CHECK_MK_SITE/.arc -# COPY ./config/client.conf /opt/omd/sites/$CHECK_MK_SITE/.arc/ -# RUN chown -R $CHECK_MK_SITE /opt/omd/sites/$CHECK_MK_SITE/.arc/ +# # # RUN yum -y install python3-gfal2 python3-gfal2-util gfal2-plugin-srm gfal2-plugin-gridftp gfal2-plugin-http gfal2-plugin-sftp gfal2-plugin-xrootd xrootd-client + +# ARC config +RUN mkdir /opt/omd/sites/$CHECK_MK_SITE/.arc +COPY ./config/client.conf /opt/omd/sites/$CHECK_MK_SITE/.arc/ +RUN chown -R $CHECK_MK_SITE /opt/omd/sites/$CHECK_MK_SITE/.arc/ + +# MW env +COPY ./config/grid-env.sh /etc/profile.d/ +RUN echo "source /etc/profile.d/grid-env.sh" >> /opt/omd/sites/$CHECK_MK_SITE/.profile ################################ # OSG Middleware # RUN dnf config-manager --set-enabled epel \ # && dnf -y install nagios-plugins \ - # ETF WN-qFM payload -RUN mkdir -p /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ - && mkdir -p /usr/libexec/grid-monitoring/wnfm/bin \ - && cp /usr/bin/etf_wnfm /usr/libexec/grid-monitoring/wnfm/bin/ \ - && cp -r /usr/lib/python3.9/site-packages/wnfm /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages \ - && dnf clean all +# ETF WN-qFM payload +RUN mkdir -p /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages +RUN mkdir -p /usr/libexec/grid-monitoring/wnfm/bin +RUN cp /usr/bin/etf_wnfm /usr/libexec/grid-monitoring/wnfm/bin/ +RUN cp -r /usr/lib/python3.9/site-packages/pexpect /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages +RUN cp -r /usr/lib/python3.9/site-packages/ptyprocess /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages +RUN cp -r /usr/lib/python3.9/site-packages/wnfm /usr/libexec/grid-monitoring/wnfm/lib/python/site-packages COPY ./src/probes /usr/libexec/grid-monitoring/probes/org.dune/wnjob/org.dune/probes/org.dune #################################### -# MW env -COPY ./config/grid-env.sh /etc/profile.d/ -RUN echo "source /etc/profile.d/grid-env.sh" >> /opt/omd/sites/$CHECK_MK_SITE/.profile - # ETF local checks COPY config/dune_plugin.py /usr/lib/ncgx/x_plugins/ COPY ./config/wlcg_dune.cfg /etc/ncgx/metrics.d/ diff --git a/config/add-keys.sh b/config/add-keys.sh index abf4171..f3f58cf 100755 --- a/config/add-keys.sh +++ b/config/add-keys.sh @@ -1,6 +1,6 @@ #!/bin/bash source /opt/omd/sites/etf/.oidc-agent/oidc-env.sh cp -f /etc/grid-security/tokens/* /opt/omd/sites/etf/.oidc-agent/ +cp /opt/omd/sites/etf/.oidc-agent/etf_dune_ce{.K8s,} /usr/bin/oidc-add --pw-file=/opt/omd/sites/etf/.oidc-agent/etf_dune_ce.key etf_dune_ce /usr/lib64/nagios/plugins/refresh_token -t 890 --token-config dune-ce --token-time 21600 -x /opt/omd/sites/etf/etc/nagios/globus/dune.token - diff --git a/config/grid-env.sh b/config/grid-env.sh index 302c07a..adaf382 100644 --- a/config/grid-env.sh +++ b/config/grid-env.sh @@ -6,7 +6,6 @@ export MYPROXY_SERVER="myproxy.cern.ch" export LCG_LOCATION="/usr" export GRID_ENV_LOCATION="/usr/libexec" export LCG_GFAL_INFOSYS="top-bdii.cern.ch:2170" -export ETF_NAGIOS_HOST="ETF_NAGIOS_HOSTXX" if [[ -f /opt/omd/sites/etf/.oidc-agent/oidc-env.sh ]]; then source /opt/omd/sites/etf/.oidc-agent/oidc-env.sh -- GitLab From e1cdb9533c55bdb5053a9b016eba7529ceb4b1e5 Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Thu, 31 Oct 2024 07:27:36 +0100 Subject: [PATCH 7/8] client.conf --- config/client.conf | 1 + 1 file changed, 1 insertion(+) create mode 100644 config/client.conf diff --git a/config/client.conf b/config/client.conf new file mode 100644 index 0000000..edfaaca --- /dev/null +++ b/config/client.conf @@ -0,0 +1 @@ +joblisttype=XML \ No newline at end of file -- GitLab From 74d80278714032df3676d1763a2941ae181c641b Mon Sep 17 00:00:00 2001 From: Lorenzo Valentini <lorenzo.valentini@cern.ch> Date: Mon, 11 Nov 2024 14:26:05 +0100 Subject: [PATCH 8/8] exiting immediately add-keys script --- config/add-keys.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/config/add-keys.sh b/config/add-keys.sh index f3f58cf..fca40f8 100755 --- a/config/add-keys.sh +++ b/config/add-keys.sh @@ -1,4 +1,5 @@ #!/bin/bash +exit source /opt/omd/sites/etf/.oidc-agent/oidc-env.sh cp -f /etc/grid-security/tokens/* /opt/omd/sites/etf/.oidc-agent/ cp /opt/omd/sites/etf/.oidc-agent/etf_dune_ce{.K8s,} -- GitLab