From 995af296482af80b0dbe49fe28c36483c85c43d5 Mon Sep 17 00:00:00 2001
From: "Patrick L.S. Connor" <connorpa@mail.desy.de>
Date: Thu, 18 Jul 2024 09:30:43 +0200
Subject: [PATCH 1/5] temporary fix for Core CI (waiting for Corevim main.tex
and Corecmake --build build -j)
---
.gitlab-ci.yml | 117 +++++++++++++++----------------------------------
1 file changed, 36 insertions(+), 81 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b1d4ab05..fa7aa533 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,11 +1,8 @@
-image: gitlab-registry.cern.ch/ci-tools/ci-worker:cc7
-
before_script:
- - source /cvmfs/cms.cern.ch/cmsset_default.sh
- - shopt -s expand_aliases # allows aliases like cmsenv to be used
- - yum install -y openssl-devel python3 zsh
+ - yum install -y cmake python3 zsh which libtirpc libicu libglvnd-glx pcre2-utf16 libXrender libSM glibc-devel zlib-devel git-lfs
+ - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-el9-gcc13-opt/setup.sh
-installation_cmake:
+installation:
stage: build
variables:
GIT_STRATEGY: none # Will be checked out by the installer
@@ -15,12 +12,12 @@ installation_cmake:
paths:
- Installer/
script:
- - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-centos7-gcc12-opt/setup.sh
# Get the installer
- git clone https://gitlab.cern.ch/lmoureau/Darwin.git -b feature/cmake Installer # TODO
# Compile everything (with ninja!)
- cmake Installer -B build -G Ninja -DCORE_GIT_REPOSITORY="$CI_REPOSITORY_URL" -DCORE_GIT_TAG="$CI_COMMIT_REF_NAME"
- cmake --build build --target install -j
+ # TODO: doxygen
test_cmake:
stage: test
@@ -31,52 +28,17 @@ test_cmake:
artifacts:
when: always
paths:
- - Installer/Core.build/bin/printDarwinSoftwareVersion
- Installer/Core.build/test_report.xml
- Installer/Core.build/Testing/Temporary/LastTest.log
reports:
junit: Installer/Core.build/test_report.xml
script:
- - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-centos7-gcc12-opt/setup.sh
- - cd Installer/Core.build
+ - cd Installer
+ - source tools/setup.sh
+ - printDarwinSoftwareVersion
+ - cd Core.build
- ctest --output-junit test_report.xml -j$(nproc)
-
-installation:
- stage: build
- variables:
- GIT_STRATEGY: clone
- tags:
- - cvmfs
- artifacts:
- paths:
- - DAS/
- script:
- - yum install -y cmake3
- # All the cd and mv gymnastics below is there to make sure install.sh
- # picks up the version of Core we want to test.
- # We start within a copy of the right version of Core. Rename it so we
- # can move it more easily later.
- - REPO=$PWD
- - cd ..
- - mv $REPO $REPO.bak
- # Create an empty dir with the same name so GitLab copies artifacts.
- - mkdir $REPO
- - cd $REPO
- # Now get the installer and run it.
- - git clone https://gitlab.cern.ch/cms-analysis/general/DasAnalysisSystem/gitlab-profile.git DAS
- - cd DAS
- - ./install.sh
- - cd CMSSW_*; cmsenv; cd ..
- # Delete the Core fetched by the installer and replace it with our own.
- - rm -rf CMSSW_*/src/Core
- - mv $REPO.bak CMSSW_12_4_0/src/Core
- # Finally compile everything.
- - cd CMSSW_*
- - scram b -j$(nproc)
- # And generate the docs.
- - src/Core/CommonTools/doc/mkDoc
-
clang-format:
stage: test
variables:
@@ -85,39 +47,37 @@ clang-format:
- cvmfs
allow_failure: true
script:
- - cd DAS/CMSSW_*
- - cmsenv
- echo $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
- echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
- git-clang-format --diff $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
- git-clang-format --diff $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
- "[[ -z \"$(git-clang-format -q --diff $CI_MERGE_REQUEST_TARGET_BRANCH_NAME)\" ]]"
-analysis:
- retry: 2
- stage: test
- variables:
- GIT_STRATEGY: none
- tags:
- - cvmfs
- allow_failure: true
- artifacts:
- when: on_failure
- paths:
- - _test
- script:
- - TEST_FOLDER=$PWD/_test
- - |
- (cd DAS/CMSSW_*/src/Core/Ntupliser/test
- cmsenv
- ./mkFastSimTestSample.sh $TEST_FOLDER
- ./mkNtuple.sh $TEST_FOLDER/mc $TEST_FOLDER/MINIAODSIMv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root yes
- ./mkNtuple.sh $TEST_FOLDER/data $TEST_FOLDER/MINIAODv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root no)
- - find $TEST_FOLDER
- - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-centos7-gcc12-opt/setup.sh
- - source Installer/tools/setup.sh
- - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/mc $TEST_FOLDER/mc/ntuple.root yes
- - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/data $TEST_FOLDER/data/ntuple.root no
+# TODO: first solve https://gitlab.cern.ch/cms-analysis/general/DasAnalysisSystem/Core/-/merge_requests/170
+#analysis:
+# retry: 2
+# stage: test
+# variables:
+# GIT_STRATEGY: none
+# tags:
+# - cvmfs
+# allow_failure: true
+# artifacts:
+# when: on_failure
+# paths:
+# - _test
+# script:
+# - TEST_FOLDER=$PWD/_test
+# - |
+# (cd DAS/CMSSW_*/src/Core/Ntupliser/test
+# cmsenv
+# ./mkFastSimTestSample.sh $TEST_FOLDER
+# ./mkNtuple.sh $TEST_FOLDER/mc $TEST_FOLDER/MINIAODSIMv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root yes
+# ./mkNtuple.sh $TEST_FOLDER/data $TEST_FOLDER/MINIAODv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root no)
+# - find $TEST_FOLDER
+# - source Installer/tools/setup.sh
+# - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/mc $TEST_FOLDER/mc/ntuple.root yes
+# - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/data $TEST_FOLDER/data/ntuple.root no
pages:
stage: deploy
@@ -132,17 +92,12 @@ pages:
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
tags:
- cvmfs
- before_script:
- - yum install -y zsh ghostscript
- script:
- - source /cvmfs/sft.cern.ch/lcg/releases/gcc/9.3.0-467e1/x86_64-centos7/setup.sh
- - source /cvmfs/sft.cern.ch/lcg/views/LCG_100/x86_64-centos7-gcc9-opt/setup.sh
+ script: # TODO: first solve https://gitlab.cern.ch/cms-analysis/general/DasAnalysisSystem/Core/-/merge_requests/152
+ - yum install -y ghostscript
- export PATH=/cvmfs/sft.cern.ch/lcg/external/texlive/latest/bin/x86_64-linux:$PATH
- doxygen --version
- - cd DAS/CMSSW_*
- VERSION=$CI_COMMIT_SHA
- CMSSW_BASE=$PWD
- doxygen src/Core/CommonTools/doc/Doxyfile
+ doxygen Core/CommonTools/doc/Doxyfile
- mv doc/html $OLDPWD/public
- make -C doc/latex || cat doc/latex/refman.log
- mv doc/latex/ $OLDPWD
--
GitLab
From 95cf8a8a5a3743185488ec4ef77734299bc16d44 Mon Sep 17 00:00:00 2001
From: "Patrick L.S. Connor" <connorpa@mail.desy.de>
Date: Thu, 18 Jul 2024 09:46:00 +0200
Subject: [PATCH 2/5] trying CI without ninja
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fa7aa533..fe151786 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,8 +14,8 @@ installation:
script:
# Get the installer
- git clone https://gitlab.cern.ch/lmoureau/Darwin.git -b feature/cmake Installer # TODO
- # Compile everything (with ninja!)
- - cmake Installer -B build -G Ninja -DCORE_GIT_REPOSITORY="$CI_REPOSITORY_URL" -DCORE_GIT_TAG="$CI_COMMIT_REF_NAME"
+ # Compile everything
+ - cmake Installer -B build -DCORE_GIT_REPOSITORY="$CI_REPOSITORY_URL" -DCORE_GIT_TAG="$CI_COMMIT_REF_NAME"
- cmake --build build --target install -j
# TODO: doxygen
--
GitLab
From 23d4ab334ceb3dae69a768bd88aa227fbf55c807 Mon Sep 17 00:00:00 2001
From: "Patrick L.S. Connor" <connorpa@mail.desy.de>
Date: Thu, 18 Jul 2024 10:00:03 +0200
Subject: [PATCH 3/5] controlling the number of parallel tasks
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fe151786..3558cddc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ installation:
- git clone https://gitlab.cern.ch/lmoureau/Darwin.git -b feature/cmake Installer # TODO
# Compile everything
- cmake Installer -B build -DCORE_GIT_REPOSITORY="$CI_REPOSITORY_URL" -DCORE_GIT_TAG="$CI_COMMIT_REF_NAME"
- - cmake --build build --target install -j
+ - cmake --build build --target install -j$(nproc)
# TODO: doxygen
test_cmake:
--
GitLab
From 2b4b89c4c3ba9891d1e5200cd144ddb2b19b3c0f Mon Sep 17 00:00:00 2001
From: "Patrick L.S. Connor" <connorpa@mail.desy.de>
Date: Thu, 18 Jul 2024 10:55:17 +0200
Subject: [PATCH 4/5] changing Installer version
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3558cddc..f8a8028a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,7 +13,7 @@ installation:
- Installer/
script:
# Get the installer
- - git clone https://gitlab.cern.ch/lmoureau/Darwin.git -b feature/cmake Installer # TODO
+ - git clone https://gitlab.cern.ch/cms-analysis/general/DasAnalysisSystem/gitlab-profile.git -b micromamba Installer # TODO: change to main branch once the whole migration is done
# Compile everything
- cmake Installer -B build -DCORE_GIT_REPOSITORY="$CI_REPOSITORY_URL" -DCORE_GIT_TAG="$CI_COMMIT_REF_NAME"
- cmake --build build --target install -j$(nproc)
--
GitLab
From ce05f5ab380e565de08e5e927b46943510657e42 Mon Sep 17 00:00:00 2001
From: "Patrick L.S. Connor" <connorpa@mail.desy.de>
Date: Fri, 19 Jul 2024 08:41:17 +0200
Subject: [PATCH 5/5] Addressing Louis' comments
---
.gitlab-ci.yml | 54 +++++++++++++++++++++++++-------------------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f8a8028a..f2ec2c68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,6 @@
before_script:
- - yum install -y cmake python3 zsh which libtirpc libicu libglvnd-glx pcre2-utf16 libXrender libSM glibc-devel zlib-devel git-lfs
- - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-el9-gcc13-opt/setup.sh
+ - yum install -y cmake python3 zsh which libtirpc libicu libglvnd-glx pcre2-utf16 libXrender libSM glibc-devel zlib-devel git-lfs
+ - source /cvmfs/sft.cern.ch/lcg/views/LCG_104/x86_64-el9-gcc13-opt/setup.sh
installation:
stage: build
@@ -53,31 +53,31 @@ clang-format:
- git-clang-format --diff $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
- "[[ -z \"$(git-clang-format -q --diff $CI_MERGE_REQUEST_TARGET_BRANCH_NAME)\" ]]"
-# TODO: first solve https://gitlab.cern.ch/cms-analysis/general/DasAnalysisSystem/Core/-/merge_requests/170
-#analysis:
-# retry: 2
-# stage: test
-# variables:
-# GIT_STRATEGY: none
-# tags:
-# - cvmfs
-# allow_failure: true
-# artifacts:
-# when: on_failure
-# paths:
-# - _test
-# script:
-# - TEST_FOLDER=$PWD/_test
-# - |
-# (cd DAS/CMSSW_*/src/Core/Ntupliser/test
-# cmsenv
-# ./mkFastSimTestSample.sh $TEST_FOLDER
-# ./mkNtuple.sh $TEST_FOLDER/mc $TEST_FOLDER/MINIAODSIMv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root yes
-# ./mkNtuple.sh $TEST_FOLDER/data $TEST_FOLDER/MINIAODv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root no)
-# - find $TEST_FOLDER
-# - source Installer/tools/setup.sh
-# - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/mc $TEST_FOLDER/mc/ntuple.root yes
-# - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/data $TEST_FOLDER/data/ntuple.root no
+analysis:
+ allow_failure: true # TODO: finalise https://gitlab.cern.ch/cms-analysis/general/DasAnalysisSystem/Core/-/merge_requests/170
+ retry: 2
+ stage: test
+ variables:
+ GIT_STRATEGY: none
+ tags:
+ - cvmfs
+ allow_failure: true
+ artifacts:
+ when: on_failure
+ paths:
+ - _test
+ script:
+ - TEST_FOLDER=$PWD/_test
+ - |
+ (cd DAS/CMSSW_*/src/Core/Ntupliser/test
+ cmsenv
+ ./mkFastSimTestSample.sh $TEST_FOLDER
+ ./mkNtuple.sh $TEST_FOLDER/mc $TEST_FOLDER/MINIAODSIMv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root yes
+ ./mkNtuple.sh $TEST_FOLDER/data $TEST_FOLDER/MINIAODv2-DYToLL_M-50_13TeV_pythia8_cff-UL18.root no)
+ - find $TEST_FOLDER
+ - source Installer/tools/setup.sh
+ - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/mc $TEST_FOLDER/mc/ntuple.root yes
+ - Installer/Core/Ntupliser/test/runAnalysis.sh $TEST_FOLDER/data $TEST_FOLDER/data/ntuple.root no
pages:
stage: deploy
--
GitLab