From d7eacbeab0acb976ee91d19aeae2969dc0158492 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Tue, 16 May 2023 11:33:13 +0200 Subject: [PATCH 1/7] CI: disable Python2 tests and jobs --- .gitlab-ci.yml | 112 ++----------------------------------------------- 1 file changed, 3 insertions(+), 109 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ffc17b28b..057e9c3e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,16 +34,6 @@ variables: DIRAC_DEPRECATED_FAIL: "1" -.rule_release_tags_py2_template: &rule_release_tags - rules: - - if: '$CI_PROJECT_PATH != "CLICdp/iLCDirac/ILCDIRAC"' - when: never - - if: '$CI_COMMIT_TAG =~ /.*-pre[0-9]*|.*a[0-9]*/' - when: never - - if: '$CI_COMMIT_TAG =~ /v[0-9]+\.[0-9]+\.[0-9]+/' - when: never - - if: '$CI_COMMIT_TAG =~ /v[0-9]*r[0-9]*.*/' - .rule_release_tags_py3_template: &rule_release_tags_py3 rules: - if: '$CI_PROJECT_PATH != "CLICdp/iLCDirac/ILCDIRAC"' @@ -56,12 +46,6 @@ variables: when: never - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/' -.rule_pre_tags_py2_template: &rule_pre_tags - rules: - - if: '$CI_PROJECT_PATH != "CLICdp/iLCDirac/ILCDIRAC"' - when: never - - if: '$CI_COMMIT_TAG =~ /.*-pre[0-9]*/' - .rule_pre_tags_py3_template: &rule_pre_tags_py3 rules: - if: '$CI_PROJECT_PATH != "CLICdp/iLCDirac/ILCDIRAC"' @@ -169,11 +153,6 @@ variables: image: registry.cern.ch/docker.io/library/python:3.9 -install:CC7: - <<: *install - <<: *cc7docker - - install:Py3: <<: *cc7docker stage: install @@ -255,13 +234,6 @@ code_quality: when: never - if: '$SWEEP != "true"' - -unittest:CC7: - <<: *unittest - <<: *cc7docker - needs: ["install:CC7"] - - unittest:Py3: <<: *unittest <<: *cc7docker @@ -294,7 +266,7 @@ checkFormat: allow_failure: true <<: *retry_policy stage: unittest - needs: ["install:CC7"] + needs: ["install:Py3"] <<: *rule_normal script: - export HOME=/root/ @@ -319,13 +291,6 @@ checkFormat: - .gitlab-ci.d/run-pylint.sh <<: *retry_policy -pylint:CC7: - variables: - GIT_STRATEGY: none - PYLINT_RCFILE: ".gitlab-ci.d/DIRAC.pylint.rc" - <<: *cc7docker - <<: *pylint - needs: ["install:CC7"] pylint:Py3: variables: @@ -335,22 +300,6 @@ pylint:Py3: <<: *pylint needs: ["install:Py3"] -pylint:PY3K: - <<: *cc7docker - stage: integration - variables: - GIT_STRATEGY: none - <<: *rule_normal - script: - - export HOME=/root/ - - tar xzf installation.tar.gz - - cd installation/ILCDIRAC - - source ../bashrc - - unset PYTHONOPTIMIZE - - .gitlab-ci.d/run-pylint-py3k.sh - <<: *retry_policy - needs: ["install:CC7"] - .jobTests_template: &jobTests stage: integration @@ -403,29 +352,15 @@ pylint:PY3K: when: on_failure expire_in: 2 days -.cc7: &cc7integration - <<: *cc7docker - needs: ["install:CC7", "unittest:CC7"] - .py3: &py3integration <<: *cc7docker needs: ["install:Py3", "unittest:Py3"] -jobTests:CC7: - <<: *jobTests - <<: *rule - <<: *cc7integration - jobTests:Py3: <<: *jobTests <<: *rule <<: *py3integration -prodTests:CC7: - <<: *prodTests - <<: *rule - <<: *cc7integration - prodTests:Py3: <<: *prodTests <<: *rule @@ -452,12 +387,6 @@ prodTests:Py3: <<: *retry_policy <<: *rule_schedule -setests:CC7: - <<: *seTests - <<: *rule - <<: *cc7docker - needs: ["install:CC7"] - setests:Py3: <<: *seTests <<: *rule @@ -514,18 +443,6 @@ compiledoc: expire_in: 2 hours <<: *retry_policy -testmaketarballs: - <<: *tarball - <<: *rule - variables: - ILCDIRAC_TAG: v32r0p2 - -maketarballs: - <<: *tarball - <<: *rule_all_tags - variables: - ILCDIRAC_TAG: ${CI_COMMIT_TAG} - .wheel_template: &wheel <<: *py39docker @@ -549,12 +466,13 @@ testCreateWheels: <<: *rule_normal <<: *wheel + createWheels: <<: *rule_all_tags_py3 <<: *wheel -makeILCConfig: +.makeILCConfig: <<: *cc7docker stage: create needs: [] @@ -648,30 +566,6 @@ deploytag:pre: <<: *rule_pre_tags needs: ["maketarballs"] -deployToCVMFS: - stage: deployToCVMFS - tags: - - cvmfs - <<: *rule_release_tags - needs: ["deploytag:prod"] - script: - - sudo -u cvclicdp -i scripts/installILCDIRAC.sh $CI_COMMIT_TAG $CERT_PASSPHRASE - before_script: [] - after_script: [] - - -deployToCVMFS:pre: - stage: deployToCVMFS - tags: - - cvmfs - <<: *rule_pre_tags - needs: ["deploytag:pre"] - script: - - sudo -u cvclicdp -i scripts/installILCDIRAC.sh $CI_COMMIT_TAG $CERT_PASSPHRASE PRE_RELEASE - before_script: [] - after_script: [] - - .deployPy3_template: &deploy_py3 stage: deployToCVMFS before_script: [] -- GitLab From 9a47cb4fadc2b3a40176dd28136392b9f2fac138 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Fri, 26 May 2023 13:15:44 +0200 Subject: [PATCH 2/7] CI: remove obsolete install template --- .gitlab-ci.yml | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 057e9c3e4..a3a51f807 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,44 +104,6 @@ variables: - api_failure - runner_unsupported -.install_template: &install - stage: install - script: - - cd ../ - - rm -rf !(ILCDIRAC) .installCache - - mv ILCDIRAC ILCDIRAC_commit - - mkdir ILCDIRAC; cd ILCDIRAC - - mkdir installation; cd installation - - wget -O dirac-install -np https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py --no-check-certificate - - chmod +x dirac-install - - ./dirac-install -V ILCDIRAC -r $ILCDIRAC_VERSION --dirac-os --dirac-os-version=$DIRACOS_VERSION - - rm -rf DIRAC - - git clone https://github.com/DIRACGrid/DIRAC.git --depth 1 --single-branch -b $DIRAC_BRANCH DIRAC - - rm -rf ILCDIRAC - - mv ../../ILCDIRAC_commit ILCDIRAC - - rm -rf .installCache - - echo "export PYTHONPATH=\$DIRAC/ILCDIRAC/src:\$DIRAC/DIRAC/src:\$DIRAC/ILCDIRAC/:\$PYTHONPATH" >> bashrc - - echo "export X509_CERT_DIR=/builds/${CI_PROJECT_PATH}/installation/DIRAC/src/etc/grid-security/certificates" >> bashrc - - echo "export X509_VOMSES=/builds/${CI_PROJECT_PATH}/installation/DIRAC/src/etc/grid-security/vomses" >> bashrc - - echo "export X509_VOMS_DIR=/builds/${CI_PROJECT_PATH}/installation/DIRAC/src/etc/grid-security/vomsdir" >> bashrc - - source bashrc - - dirac-deploy-scripts - - unset REQUESTS_CA_BUNDLE - - pip install --upgrade pip - - pip uninstall -y distribute - - pip install --upgrade setuptools || echo "done" - - pip install --upgrade setuptools_scm || echo "done scm" - - pip install --upgrade pylint mock MySQL-python pytest-cov pytest-randomly pytest-mock pytest-timeout pytest-xdist flake8 flake8-docstrings flake8-commas caniusepython3 parameterized future suds_jurko sphinx-tabs - - cd .. - - tar czf installation.tar.gz installation - artifacts: - paths: - - installation.tar.gz - expire_in: 2 hours - rules: - - if: '$SWEEP != "true"' - <<: *retry_policy - .cc7docker_template: &cc7docker tags: - docker -- GitLab From 0fbe7e7d5e02b07c060a008e6d5879fca9fb0dc3 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Fri, 26 May 2023 17:05:50 +0200 Subject: [PATCH 3/7] CI: remove more obsolete templates and rules --- .gitlab-ci.yml | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3a51f807..2e1f5c6e2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -434,26 +434,12 @@ createWheels: <<: *wheel -.makeILCConfig: - <<: *cc7docker - stage: create - needs: [] - <<: *rule_release_tags - script: - - mkdir -p ILCDIRACTars/defaults; echo -e "Releases = https://gitlab.cern.ch/CLICdp/iLCDirac/ILCDIRAC/raw/master/releases.cfg\nUploadCommand = ( cd %OUTLOCATION% ; tar -cf - *.tar.gz *.md5 *.cfg *.pdf *.html ) | ssh \`whoami\`@lxplus.cern.ch \'cd /eos/project/l/lcd-web/www/ILCDIRACTars && tar -xvf - && ls *.tar.gz > tars.list\'\nBaseURL = http://lcd-data.web.cern.ch/lcd-data/ILCDIRACTars\nLocalInstallation\n{\n Project = ILCDIRAC\n ExternalsType = client\n PythonVersion = 27\n Release = $CI_COMMIT_TAG\n DIRACOS = $DIRACOS_VERSION\n Setup = ILC-Production\n ConfigurationServer = dips://voilcdirac13.cern.ch:9135/Configuration/Server, dips://voilcdirac14.cern.ch:9135/Configuration/Server\n}\n" > ILCDIRACTars/defaults/ilc.cfg - artifacts: - paths: - - ILCDIRACTars - expire_in: 2 hours - <<: *retry_policy - - deploydoc: stage: deploy tags: - docker image: gitlab-registry.cern.ch/ci-tools/ci-web-deployer:latest - <<: *rule_release_tags + <<: *rule_release_tags_py3 needs: ["compiledoc"] variables: EOS_PATH: "/eos/project/l/lcd-web/www/doc/ilcdiracdoc/" @@ -503,31 +489,6 @@ docProd: EOS_PATH: "/eos/project/l/lcd-web/www/doc/ilcdiracdoc/" CI_OUTPUT_DIR: "ilcdiracdoc" - -.deploytag_template: &deploytag - stage: deploy - tags: - - docker - image: gitlab-registry.cern.ch/ci-tools/ci-web-deployer:latest - variables: - EOS_PATH: "/eos/project/l/lcd-web/www/ILCDIRACTars/" - CI_OUTPUT_DIR: "ILCDIRACTars" - script: - - deploy-eos - before_script: [] - after_script: [] - <<: *retry_policy - -deploytag:prod: - <<: *deploytag - <<: *rule_release_tags - needs: ["maketarballs", "makeILCConfig"] - -deploytag:pre: - <<: *deploytag - <<: *rule_pre_tags - needs: ["maketarballs"] - .deployPy3_template: &deploy_py3 stage: deployToCVMFS before_script: [] -- GitLab From 243ec8c184963c3b0789f8851c4d38987504d2d9 Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 29 May 2023 11:27:48 +0200 Subject: [PATCH 4/7] CI: cleanup variables, install latest DIRAC --- .gitlab-ci.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e1f5c6e2..88850cada 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,8 +9,8 @@ workflow: - if: '$CI_PIPELINE_SOURCE == "schedule"' before_script: - - echo $LCG_BUNDLE_VERSION $ILCDIRAC_VERSION $DIRAC_BRANCH $SETUP - - source /cvmfs/clicdp.cern.ch/software/git/2.13.2/x86_64-slc6-gcc7-opt/setup.sh + - echo ${SETUP} ${DIRAC_RELEASE} ${ILCDIRAC_TEST_CONFIG} ${ILCDIRAC_BRANCH} + - source /cvmfs/clicdp.cern.ch/software/git/2.13.2/x86_64-centos7-gcc7-opt/setup.sh - shopt -s extglob stages: @@ -23,12 +23,8 @@ stages: - sweep variables: - DIRACOS_VERSION: "master" - - ILCDIRAC_VERSION: "v32r0p3" ILCDIRAC_TEST_CONFIG: "dips://voilcdiracconfig.cern.ch:9135/Configuration/Server" - DIRAC_BRANCH: "rel-v7r3" - DIRAC_TAG: "v7r3p34" + DIRAC_RELEASE: "7.3" SETUP: "ILC-Development" ILCDIRAC_BRANCH: "Rel-v34r0" DIRAC_DEPRECATED_FAIL: "1" @@ -133,6 +129,7 @@ install:Py3: - mv ../ILCDIRAC . - ls -la - pip install ./ILCDIRAC[testing] + - pip install -U DIRAC~=${DIRAC_RELEASE} - cd .. - tar czf installation.tar.gz installation artifacts: -- GitLab From 1c11ce3078d81b0dbd752ac51fcf99e3d0f5c5cb Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 29 May 2023 11:30:35 +0200 Subject: [PATCH 5/7] CI: disable broken code_quality and checkFormat --- .gitlab-ci.yml | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88850cada..9c9ec963a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -140,10 +140,10 @@ install:Py3: - if: '$SWEEP != "true"' <<: *retry_policy -include: - - template: Code-Quality.gitlab-ci.yml +# include: +# - template: Code-Quality.gitlab-ci.yml -code_quality: +.code_quality: stage: integration tags: - overlay @@ -160,7 +160,6 @@ code_quality: - if: $CI_COMMIT_TAG when: never - if: '$SWEEP != "true"' - <<: *retry_policy @@ -220,7 +219,7 @@ checkLicense: - git diff --quiet # fail if there are changed files -checkFormat: +.checkFormat: <<: *cc7docker allow_failure: true <<: *retry_policy @@ -383,25 +382,6 @@ compiledoc: allow_failure: true <<: *retry_policy -.tarball_template: &tarball - stage: create - before_script: [] - after_script: [] - needs: [] - tags: - - docker - image: ghcr.io/diracgrid/management/dirac-distribution:latest - script: - - export HOME=/root/ - - echo -e "DefaultModules = ILCDIRAC\n\nSources\n{\n ILCDIRAC = https://gitlab.cern.ch/CLICdp/iLCDirac/ILCDIRAC.git\n}\nReleases\n{\n $ILCDIRAC_TAG\n {\n Modules = ILCDIRAC:$ILCDIRAC_TAG\n Depends = DIRAC:$DIRAC_TAG\n }\n}\n" > thisrelease.cfg - - cat thisrelease.cfg - - /dirac-distribution.py -l ILCDIRAC -r $ILCDIRAC_TAG -C thisrelease.cfg -D ILCDIRACTars - artifacts: - paths: - - ILCDIRACTars - expire_in: 2 hours - <<: *retry_policy - .wheel_template: &wheel <<: *py39docker -- GitLab From 22a98d0ce3de5b9c75381014357999a8cf376e4d Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 29 May 2023 11:30:44 +0200 Subject: [PATCH 6/7] CI: tweak docs creation --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c9ec963a..21c45fbe4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -363,14 +363,13 @@ compiledoc: - tar xzf installation.tar.gz - cd installation/ILCDIRAC - source ../bashrc - - source /cvmfs/clicdp.cern.ch/software/git/2.13.2/x86_64-slc6-gcc7-opt/setup.sh + - source /cvmfs/clicdp.cern.ch/software/git/2.13.2/x86_64-centos7-gcc7-opt/setup.sh - unset REQUESTS_CA_BUNDLE - rm -rf ILCDIRAC ## removes just the folder with the links - cd docs - export DIRAC=$PWD/../../DIRAC - export ILCDIRAC=$PWD/../../ILCDIRAC - export PYTHONPATH=$PWD/source:$PYTHONPATH - - bash patchSphinx.sh - SPHINXOPTS=-wsphinxWarnings make completeDocumentation - mv build/html ../../../ilcdiracdoc - if [ -s sphinxWarnings ]; then cat sphinxWarnings; echo "Warnings When Creating Doc"; exit 1; fi -- GitLab From 531cfd6be54c2dc1472e3a127e39c5a20f122c7d Mon Sep 17 00:00:00 2001 From: Andre Sailer <andre.philippe.sailer@cern.ch> Date: Mon, 29 May 2023 11:31:12 +0200 Subject: [PATCH 7/7] setup: use latest DIRAC DIRAC 7.3.34 has broken m2crypto, BIO error when uploading proxy --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index aea00d44d..972bf3acf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ package_dir= =src packages = find: install_requires = - DIRAC == 7.3.34 + DIRAC == 7.3.36 requests six zip_safe = False -- GitLab