From 2c9e8c608b7596721fe72d83e9cb7822638529a3 Mon Sep 17 00:00:00 2001 From: David Southwick <david.southwick@cern.ch> Date: Tue, 6 Dec 2022 17:44:28 +0100 Subject: [PATCH 1/4] BMK-1061: remove oras binary --- build-executor/image/install.sh | 14 -------------- build-executor/utility.sh | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/build-executor/image/install.sh b/build-executor/image/install.sh index df869e5e..4b44cae4 100644 --- a/build-executor/image/install.sh +++ b/build-executor/image/install.sh @@ -5,16 +5,12 @@ set -ex function install_x86(){ # Instructions to install packages only for x86 echo -e "\n[install_x86]\n" - - export ORAS_URL=https://github.com/oras-project/oras/releases/download/v0.14.0/oras_0.14.0_linux_amd64.tar.gz install_common } function install_aarch(){ # Instructions to install packages only for x86 echo -e "\n[install_aarch]\n" - - export ORAS_URL=https://github.com/oras-project/oras/releases/download/v0.14.0/oras_0.14.0_linux_arm64.tar.gz install_common } @@ -73,16 +69,6 @@ function install_common(){ python3 -m pip install --upgrade pip; #pip2 install dictdiffer pyyaml numpy; python3 -m pip install dictdiffer pyyaml numpy - - - echo "install ORAS" - curl -LO ${ORAS_URL} - mkdir -p oras-install/ - tar -zxf oras_*.tar.gz -C oras-install/ - mv oras-install/oras /usr/local/bin/ - rm -rf oras_*.tar.gz oras-install/ - # Temporary ORAS binary to push singularity images to registry.cern.ch - # to be removed when singularity 3.8+ is available } INSTALL_SCRIPT=$(readlink -f $0) diff --git a/build-executor/utility.sh b/build-executor/utility.sh index b9b73c6c..797afb7d 100644 --- a/build-executor/utility.sh +++ b/build-executor/utility.sh @@ -492,10 +492,10 @@ function docker_to_sif(){ # BMK-1075 : moving sif registry back to gitlab-registry if [[ "${CIENV_SINGULARITYREGISTRY}" =~ (^gitlab-.*$) ]]; then - (echo $CI_PRIVATE_TOKEN_SIF | oras login -u ${CIENV_SINGULARITYROBOT} --password-stdin gitlab-registry.cern.ch) || fail "[publish_oras.sh] cannot login to gitlab-registry.cern.ch" + (echo $CI_PRIVATE_TOKEN_SIF | singularity remote login -u ${CIENV_SINGULARITYROBOT} --password-stdin oras://gitlab-registry.cern.ch) || fail "[publish_oras.sh] cannot login to gitlab-registry.cern.ch" else # Maintain registry.cern.ch for legacy reasons - (echo $CI_SING_TOKEN | oras login -u ${CIENV_SINGULARITYROBOT} --password-stdin registry.cern.ch) || fail "[publish_oras.sh] cannot login to ${CIENV_SINGULARITYREGISTRY}" + (echo $CI_SING_TOKEN | singularity remote login -u ${CIENV_SINGULARITYROBOT} --password-stdin oras://registry.cern.ch) || fail "[publish_oras.sh] cannot login to ${CIENV_SINGULARITYREGISTRY}" fi # Loop on built architectures, as declared in HEPWL_BUILDARCH -- GitLab From 7094ce6cd806cccf0dab031ddd0987ea99d070e7 Mon Sep 17 00:00:00 2001 From: David Southwick <david.southwick@cern.ch> Date: Tue, 6 Dec 2022 17:52:45 +0100 Subject: [PATCH 2/4] BMK-1061 singularity push instead of ORAS --- build-executor/utility.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-executor/utility.sh b/build-executor/utility.sh index 797afb7d..cfbcc7a0 100644 --- a/build-executor/utility.sh +++ b/build-executor/utility.sh @@ -514,9 +514,9 @@ function docker_to_sif(){ execute_command_retry singularity build ${HEPWL_DOCKERIMAGENAME}.sif docker-daemon://${CIENV_DOCKERREGISTRY}/${HEPWL_DOCKERIMAGENAME}:${thetag} || fail "[publish_oras.sh] cannot build singularity image" - execute_command_retry oras push ${CIENV_SINGULARITYREGISTRY}/${HEPWL_DOCKERIMAGENAME}:${thetag} ${HEPWL_DOCKERIMAGENAME}.sif:application/vnd.sylabs.sif.layer.v1.sif || fail "[publish_oras.sh] cannot push sif image" + execute_command_retry singularity push ${HEPWL_DOCKERIMAGENAME}.sif oras://${CIENV_SINGULARITYREGISTRY}/${HEPWL_DOCKERIMAGENAME}:${thetag} || fail "[publish_oras.sh] cannot push sif image" if [[ ${HEPWL_DOCKERIMAGETAG} =~ ^v[0-9]*\.[0-9]*$ ]]; then - execute_command_retry oras push ${CIENV_SINGULARITYREGISTRY}/${HEPWL_DOCKERIMAGENAME}:latest_${somearch} ${HEPWL_DOCKERIMAGENAME}.sif:application/vnd.sylabs.sif.layer.v1.sif || fail "[publish_oras.sh] cannot push sif image" + execute_command_retry singularity push ${HEPWL_DOCKERIMAGENAME}.sif oras://${CIENV_SINGULARITYREGISTRY}/${HEPWL_DOCKERIMAGENAME}:latest_${somearch} || fail "[publish_oras.sh] cannot push sif image" fi rm -f ${HEPWL_DOCKERIMAGENAME}.sif done -- GitLab From 06c07fb0dcb99c07eb41e2d923f6857d0e3918df Mon Sep 17 00:00:00 2001 From: Domenico Giordano <domenico.giordano@cern.ch> Date: Thu, 25 May 2023 12:05:33 +0200 Subject: [PATCH 3/4] [BMK-1061] use multiarch images for oras stage --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1918cdd0..99c2334f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,7 +49,7 @@ announce: ######################################## .publish_oras: stage: publish_oras - image: $CI_REGISTRY_IMAGE/hep-workload-builder:latest + image: $CI_REGISTRY_IMAGE/hep-workload-builder-multiarch:latest tags: - hep-benchmarks - x86_64 @@ -74,6 +74,9 @@ publish_oras_x86: publish_oras_aarch64: variables: CIENV_PROCESS_ARCH: "aarch64" + tags: + - hep-benchmarks + - aarch64 extends: .publish_oras ######################################## @@ -142,7 +145,7 @@ test-sing-img-aarch64: variables: # This is needed because of the GPU build arg JOBBUILDARG: buildgpu - CIENV_BUILDIMG: $CI_REGISTRY_IMAGE/hep-workload-builder:latest + CIENV_BUILDIMG: $CI_REGISTRY_IMAGE/hep-workload-builder-multiarch:latest script: - export CIENV_BUILDIMG - echo $JOBBUILDARG -- GitLab From 5403f8eaa7f6e2b9fabe00786b7ddb2bbd1a41f2 Mon Sep 17 00:00:00 2001 From: hep-wl-merge-request <project60276_bot1@example.com> Date: Tue, 30 May 2023 15:24:35 +0200 Subject: [PATCH 4/4] update WL list --- WL_list.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WL_list.md b/WL_list.md index cbf695b7..9bc1210b 100644 --- a/WL_list.md +++ b/WL_list.md @@ -159,7 +159,7 @@ [hello_world-cs8-ma_code]: https://gitlab.cern.ch/hep-benchmarks/hep-workloads/-/blob/master/hello/world-cs8-ma/hello-world-cs8-ma [hello_world-cs8-ma_sif]: https://gitlab.cern.ch/hep-benchmarks/hep-workloads-sif/container_registry/?search%5B%5D=hello-world-cs8-ma-bmk [hello_world-cs8-ma_img]: docker://gitlab-registry.cern.ch/hep-benchmarks/hep-workloads/hello-world-cs8-ma-bmk:latest -[hello_world-cs8-ma_pipelink]: https://gitlab.cern.ch/hep-benchmarks/hep-workloads/-/pipelines/5635245 +[hello_world-cs8-ma_pipelink]: https://gitlab.cern.ch/hep-benchmarks/hep-workloads/-/pipelines/5656367 [hello_world-cs8-ma_pipeline]: https://gitlab.cern.ch/hep-benchmarks/hep-workloads/badges/qa-build-hello-world-cs8-ma/pipeline.svg [igwn_pe_code]: https://gitlab.cern.ch/hep-benchmarks/hep-workloads/-/blob/master/igwn/pe/igwn-pe -- GitLab