diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 25db00fc1acaa8c1055b3fd9156eb2baeea0d60e..c5227df87b6b2bd1b228ff05ff7a82dc74d1afd7 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -53,6 +53,7 @@ - cta/CTA#847 - Introduced version-lock for Oracle instant client - cta/CTA#821 - Improved pipeline logic for faster performance and correct cancel propagation - cta/CTA#708 - Added a basic test for the archive metadata to the CI +- cta/CTA#835 - Refactor scripts that made use of /etc/gitlab/gitlabregistry.txt to use kubernetes secret instead ### Catalogue Schema - cta/CTA#801 - Update CTA catalogue schema to version 15.0 diff --git a/continuousintegration/ci_helpers/get_registry_credentials.sh b/continuousintegration/ci_helpers/get_registry_credentials.sh new file mode 100755 index 0000000000000000000000000000000000000000..d26cbb3a5454ec3706c9117c83933d2a6b731657 --- /dev/null +++ b/continuousintegration/ci_helpers/get_registry_credentials.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# @project The CERN Tape Archive (CTA) +# @copyright Copyright © 2024 CERN +# @license This program is free software, distributed under the terms of the GNU General Public +# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can +# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# In applying this licence, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +secret_is_dockerconfigjson() { + test $(kubectl get secret $1 -o jsonpath='{.type}') == "kubernetes.io/dockerconfigjson" +} + +get_credentials() { + # The Kubernetes secret stores a base64 encoded .dockerconfigjson. This json has the following format: + # { + # "auths": { + # "gitlab-registry.cern.ch": { + # "auth": "base64 encoded string of 'username:password'" + # } + # } + # } + + local check_mode=false + while [[ "$#" -gt 0 ]]; do + case $1 in + --check) check_mode=true ;; + *) echo "Unknown option: $1" ;; + esac + shift + done + + local secret_name="ctaregsecret" + local registry_name="cta/ctageneric" + local gitlab_server="gitlab.cern.ch" + + # These variable are capatalised to match the variable in gitlabregistry.txt + local DOCKER_REGISTRY="" + local DOCKER_LOGIN_USERNAME="" + local DOCKER_LOGIN_PASSWORD="" + if secret_is_dockerconfigjson $secret_name ; then + local auth_json=$(kubectl get secret $secret_name -o jsonpath='{.data.\.dockerconfigjson}' | base64 --decode | jq -r '.auths') + + DOCKER_REGISTRY=$(echo $auth_json | jq -r 'keys[0]') + DOCKER_LOGIN_USERNAME=$(echo $auth_json | jq -r '.[].auth' | base64 --decode | cut -d: -f1) + DOCKER_LOGIN_PASSWORD=$(echo $auth_json | jq -r '.[].auth' | base64 --decode | cut -d: -f2) + else + if [[ $check_only == true ]]; then + echo "No secret with name $secret_name of type \"kubernetes.io/dockerconfigjson\" was found. Falling back to /etc/gitlab/gitlabregistry.txt..." + fi + source /etc/gitlab/gitlabregistry.txt + fi + + if [[ -z "$DOCKER_REGISTRY" ]]; then + echo "Error: Missing required variable: DOCKER_REGISTRY" + return 1 + fi + if [[ -z "$DOCKER_LOGIN_USERNAME" ]]; then + echo "Error: Missing required variable: DOCKER_LOGIN_USERNAME" + return 1 + fi + if [[ -z "$DOCKER_LOGIN_PASSWORD" ]]; then + echo "Error: Missing required variable: DOCKER_LOGIN_PASSWORD" + return 1 + fi + + # Retrieve JWT pull token from GitLab + local jwt_token=$(curl -s -u "${DOCKER_LOGIN_USERNAME}:${DOCKER_LOGIN_PASSWORD}" \ + "https://${gitlab_server}/jwt/auth?service=container_registry&scope=repository:${registry_name}:pull,push" | jq -r '.token') + + if [[ -z "$jwt_token" ]]; then + echo "Error: Failed to retrieve JWT pull token." + echo "\tRegistry: $DOCKER_REGISTRY" + echo "\tUsername: $DOCKER_LOGIN_USERNAME" + return 1 + fi + + if [[ $check_only == true ]]; then + echo "Credentials verified" + else + echo $jwt_token + fi + return 0 +} + +get_credentials "$@" \ No newline at end of file diff --git a/continuousintegration/ci_helpers/list_images.sh b/continuousintegration/ci_helpers/list_images.sh index 06fe5ccb72a4aed5423289278f209073c3deeea1..0d86c6645ebfd2029af55565b160e8ff6f18364e 100755 --- a/continuousintegration/ci_helpers/list_images.sh +++ b/continuousintegration/ci_helpers/list_images.sh @@ -1,7 +1,7 @@ #!/bin/bash # @project The CERN Tape Archive (CTA) -# @copyright Copyright © 2022 CERN +# @copyright Copyright © 2024 CERN # @license This program is free software, distributed under the terms of the GNU General Public # Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can # redistribute it and/or modify it under the terms of the GPL Version 3, or (at your @@ -15,22 +15,39 @@ # granted to it by virtue of its status as an Intergovernmental Organization or # submit itself to any jurisdiction. -# env variables used: -# DOCKER_LOGIN_USERNAME -# DOCKER_LOGIN_PASSWORD -# -# set in /etc/gitlab/gitlabregistry.txt managed by Puppet -. /etc/gitlab/gitlabregistry.txt +list_images() { + # The Kubernetes secret stores a base64 encoded .dockerconfigjson. This json has the following format: + # { + # "auths": { + # "gitlab-registry.cern.ch": { + # "auth": "base64 encoded string of 'username:password'" + # } + # } + # } + + local registry_name="cta/ctageneric" + local docker_registry="gitlab-registry.cern.ch" + + + local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local jwt_token=$(bash ${script_dir}/get_registry_credentials.sh) -TO=gitlab-registry.cern.ch/cta/ctageneric + if [[ -z "$jwt_token" ]]; then + echo "Error: Failed to retrieve JWT token." + return 1 + fi -CI_REGISTRY=$(echo ${TO} | sed -e 's%/.*%%') -REPOSITORY=$(echo ${TO} | sed -e 's%[^/]\+/%%') + # List the tags in the Docker registry repository + local list_response=$(curl -s "https://${docker_registry}/v2/${registry_name}/tags/list" -H "Authorization: Bearer ${jwt_token}") + local tags=$(echo "$list_response" | jq -c ".tags[]" | sed -e 's/^"//;s/"$//') -GITLAB_HOST=gitlab.cern.ch + if [[ -z "$tags" ]]; then + echo "Error: Failed to retrieve tags from repository:" + echo "$list_response" + return 1 + fi -JWT_PULL_PUSH_TOKEN=$(curl -q -u ${DOCKER_LOGIN_USERNAME}:${DOCKER_LOGIN_PASSWORD} \ - "https://${GITLAB_HOST}/jwt/auth?service=container_registry&scope=repository:${REPOSITORY}:pull,push" | cut -d\" -f4 ) + echo "$tags" +} -# echo "List of tags in registry" -curl "https://${CI_REGISTRY}/v2/${REPOSITORY}/tags/list" -H "Authorization: Bearer ${JWT_PULL_PUSH_TOKEN}" | jq -c ".tags[]" | sed -e 's/^"//;s/"$//' +list_images diff --git a/continuousintegration/ci_helpers/rename_tag.sh b/continuousintegration/ci_helpers/rename_tag.sh deleted file mode 100755 index c014ae0a0dc6ca2c806d6dfb28d87851b419225f..0000000000000000000000000000000000000000 --- a/continuousintegration/ci_helpers/rename_tag.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# @project The CERN Tape Archive (CTA) -# @copyright Copyright © 2022 CERN -# @license This program is free software, distributed under the terms of the GNU General Public -# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can -# redistribute it and/or modify it under the terms of the GPL Version 3, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -# PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# In applying this licence, CERN does not waive the privileges and immunities -# granted to it by virtue of its status as an Intergovernmental Organization or -# submit itself to any jurisdiction. - -# env variables used: -# DOCKER_LOGIN_USERNAME -# DOCKER_LOGIN_PASSWORD -# OLDTAG -# NEWTAG - -# TO=gitlab-registry.cern.ch/cta/cta-orchestration - -CI_REGISTRY=$(echo ${TO} | sed -e 's%/.*%%') -REPOSITORY=$(echo ${TO} | sed -e 's%[^/]\+/%%') - -GITLAB_HOST=gitlab.cern.ch - -if [[ "-${OLDTAG}-" == "-${NEWTAG}-" ]]; then - echo "The 2 tags are identical: ${OLDTAG}/${NEWTAG} no need to rename" - exit 0 -fi - -JWT_PULL_PUSH_TOKEN=$(curl -q -u ${DOCKER_LOGIN_USERNAME}:${DOCKER_LOGIN_PASSWORD} \ - "https://${GITLAB_HOST}/jwt/auth?service=container_registry&scope=repository:${REPOSITORY}:pull,push" | cut -d\" -f4 ) - -echo "List of tags in registry" -curl "https://${CI_REGISTRY}/v2/${REPOSITORY}/tags/list" -H "Authorization: Bearer ${JWT_PULL_PUSH_TOKEN}" - - -echo "Pulling the manifest of tag:${OLDTAG}" -curl "https://${CI_REGISTRY}/v2/${REPOSITORY}/manifests/${OLDTAG}" -H "Authorization: Bearer ${JWT_PULL_PUSH_TOKEN}" -H 'accept: application/vnd.docker.distribution.manifest.v2+json' > manifest.json - -echo "Pushing new tag: ${NEWTAG}" -curl -XPUT "https://${CI_REGISTRY}/v2/${REPOSITORY}/manifests/${NEWTAG}" -H "Authorization: Bearer ${JWT_PULL_PUSH_TOKEN}" -H 'content-type: application/vnd.docker.distribution.manifest.v2+json' -d '@manifest.json' -v - -echo "List of tags in registry" -curl "https://${CI_REGISTRY}/v2/${REPOSITORY}/tags/list" -H "Authorization: Bearer ${JWT_PULL_PUSH_TOKEN}" diff --git a/continuousintegration/orchestration/create_instance.sh b/continuousintegration/orchestration/create_instance.sh index e6e4e6f7c98fa43a4b365490e80ad6c35f753378..6c8222d18e0f89c92acdad49c781287791bc694b 100755 --- a/continuousintegration/orchestration/create_instance.sh +++ b/continuousintegration/orchestration/create_instance.sh @@ -166,6 +166,7 @@ if [ "$updatedatabasetest" == "1" ] ; then fi # We are going to run with repository based images (they have rpms embedded) +../ci_helpers/get_registry_credentials.sh --check || { echo "Error: Credential check failed"; exit 1; } if [[ ${systest_only} -eq 1 ]]; then COMMITID=$(curl --url "https://gitlab.cern.ch/api/v4/projects/139306/repository/commits" | jq -cr '.[0] | .short_id' | sed -e 's/\(........\).*/\1/') else