From c9c64bb2993f019f2254ad3e1fb2657b3398d462 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Thu, 20 Dec 2018 15:48:34 +0100 Subject: [PATCH 1/3] Update version of VTune Aplifier used in builds --- jenkins/utils.d/set_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/utils.d/set_common.sh b/jenkins/utils.d/set_common.sh index 5ad43858..4a3506ea 100644 --- a/jenkins/utils.d/set_common.sh +++ b/jenkins/utils.d/set_common.sh @@ -151,7 +151,7 @@ set_common [--build] [--test]" export LD_LIBRARY_PATH=${LD_LIBRARY_PATH+${LD_LIBRARY_PATH}:}/cvmfs/sft.cern.ch/lcg/contrib/bintuils/2.28/${LCG_hostos}/lib # add Intel VTune to the search path - export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/cvmfs/projects.cern.ch/intelsw/psxe/linux/x86_64/2017/vtune_amplifier_xe + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/cvmfs/projects.cern.ch/intelsw/psxe/linux/x86_64/2019/vtune_amplifier # special hack to get a dev version of the CMake configuration files export CMAKE_PREFIX_PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/cmake:${CMAKE_PREFIX_PATH} -- GitLab From cc3e2de3c3713743c7389de4d73bc714fe0c1288 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Thu, 20 Dec 2018 15:57:11 +0100 Subject: [PATCH 2/3] Use LbEnv.sh instead of LbLogin.sh --- jenkins/utils.d/build_slot.sh | 6 ---- jenkins/utils.d/set_common.sh | 36 ++---------------------- python/LbRPMTools/LHCbRPMSpecBuilder.py | 6 ++-- testdata/rpm/PARAM_TMVAWeights.spec | 4 +-- testdata/rpm/PARAM_TMVAWeights_rel5.spec | 4 +-- 5 files changed, 10 insertions(+), 46 deletions(-) diff --git a/jenkins/utils.d/build_slot.sh b/jenkins/utils.d/build_slot.sh index d9089ad6..a8baebb6 100644 --- a/jenkins/utils.d/build_slot.sh +++ b/jenkins/utils.d/build_slot.sh @@ -143,12 +143,6 @@ build_slot flavour slot slot_build_id platform time lbn-rpm ${loglevel_opt} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file} --platform "${platform}" createrepo ${directory}/rpms if [[ "$platform" != *-slc5-* ]] ; then - # remove old copies (we want always a fresh install) - rm -rf ~/.local/{bin,lib} ~/.cache/pip - # get latest pip and lbinstall - curl -O https://bootstrap.pypa.io/get-pip.py - python get-pip.py --user - ~/.local/bin/pip install --extra-index-url https://lhcb-pypi.web.cern.ch/lhcb-pypi/simple --trusted-host lhcb-pypi.web.cern.ch --user lbinstall -U lbn-rpm-validator -d --build-folder=${directory}/rpms --repo-url=file:${directory}/rpms ${platform} fi fi diff --git a/jenkins/utils.d/set_common.sh b/jenkins/utils.d/set_common.sh index 4a3506ea..2ed1db07 100644 --- a/jenkins/utils.d/set_common.sh +++ b/jenkins/utils.d/set_common.sh @@ -117,46 +117,16 @@ set_common [--build] [--test]" echo EOS destination: $EOS_ARTIFACTS_DIR echo =================================================================== - LbScriptsVersion=dev - LbLoginPath=/cvmfs/lhcb.cern.ch/lib/lhcb/LBSCRIPTS/${LbScriptsVersion}/InstallArea/scripts/LbLogin.sh - - # FIXME: workaround for LBCORE-769 - if ( echo $platform | grep -q slc5 ) ; then - export PATH=/cvmfs/lhcb.cern.ch/lib/lcg/external/Python/2.7.3/x86_64-slc5-gcc46-opt/bin:$PATH - LbLoginPath="$LbLoginPath --cmakevers=3.3.2" - fi + LbEnvPath=/cvmfs/lhcb.cern.ch/lib/LbEnv-dev.sh if [ "${special_config}" == "true" ] ; then export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \\n :) - # FIXME: this is usually set by the "group login" script, but it is not - # called on lxbuild (it is needed to get the right ICC environment) - export GROUP_DIR=/afs/cern.ch/group/z5 - export LOGIN_POST_SCRIPT=${GROUP_DIR}/post/login - export CMTEXTRATAGS=host-$(echo $platform | cut -d: -f2 | cut -d- -f2) - . $LbLoginPath - export CMTCONFIG=${platform} - # COMPILER_PATH (may be set by LbLogin) create troubles - unset COMPILER_PATH - # FIXME: variable not set in CVMFS LbLogin.sh (but should not be used by builds) - export LHCBTAR=/afs/cern.ch/lhcb/distribution - - # FIXME: temporary workaround for a working gdb in CentOS7 (see LBCORE-1261) - if ( echo $platform | grep -q centos7 ) ; then - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH+${LD_LIBRARY_PATH}:}/afs/cern.ch/user/m/marcocle/public/centos7_lib - fi - - # FIXME: make sure that if we need to call something from /cvmfs/sft.cern.ch/lcg/contrib/bintuils/2.28/xyz/bin - # we get the libraries they need - export PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/scripts:$PATH - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH+${LD_LIBRARY_PATH}:}/cvmfs/sft.cern.ch/lcg/contrib/bintuils/2.28/${LCG_hostos}/lib + . $LbEnvPath -c ${platform} # add Intel VTune to the search path export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/cvmfs/projects.cern.ch/intelsw/psxe/linux/x86_64/2019/vtune_amplifier - - # special hack to get a dev version of the CMake configuration files - export CMAKE_PREFIX_PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/cmake:${CMAKE_PREFIX_PATH} else - . $LbLoginPath + . $LbEnvPath fi if [ "${USER}" != "lblocal" ] ; then diff --git a/python/LbRPMTools/LHCbRPMSpecBuilder.py b/python/LbRPMTools/LHCbRPMSpecBuilder.py index 083f8629..05bdd11e 100644 --- a/python/LbRPMTools/LHCbRPMSpecBuilder.py +++ b/python/LbRPMTools/LHCbRPMSpecBuilder.py @@ -1041,7 +1041,7 @@ echo "Creating link in update.d" mkdir -p -v $PREFIX/etc/update.d ln -s $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/Update.py $PREFIX/etc/update.d/%{package}_Update.py echo "Running Update script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null echo "Now using python:" which python echo "PYTHONPATH: $PYTHONPATH" @@ -1052,7 +1052,7 @@ fi if [ -f $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/PostInstall.py ]; then echo "Running PostInstall script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null python $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/PostInstall.py fi @@ -1219,7 +1219,7 @@ echo "Creating link in update.d" mkdir -p -v $PREFIX/etc/update.d ln -s $PREFIX/lhcb/%{projectUp}/%{projectUp}_%{lbversion}/%{project}Sys/cmt/Update.py $PREFIX/etc/update.d/%{package}_Update.py echo "Running Update script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null echo "Now using python:" which python echo "PYTHONPATH: $PYTHONPATH" diff --git a/testdata/rpm/PARAM_TMVAWeights.spec b/testdata/rpm/PARAM_TMVAWeights.spec index efe08fc0..01d85658 100644 --- a/testdata/rpm/PARAM_TMVAWeights.spec +++ b/testdata/rpm/PARAM_TMVAWeights.spec @@ -71,7 +71,7 @@ echo "Creating link in update.d" mkdir -p -v $PREFIX/etc/update.d ln -s $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/Update.py $PREFIX/etc/update.d/%{package}_Update.py echo "Running Update script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null echo "Now using python:" which python echo "PYTHONPATH: $PYTHONPATH" @@ -82,7 +82,7 @@ fi if [ -f $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/PostInstall.py ]; then echo "Running PostInstall script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null python $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/PostInstall.py fi diff --git a/testdata/rpm/PARAM_TMVAWeights_rel5.spec b/testdata/rpm/PARAM_TMVAWeights_rel5.spec index eef276eb..5d1ed12f 100644 --- a/testdata/rpm/PARAM_TMVAWeights_rel5.spec +++ b/testdata/rpm/PARAM_TMVAWeights_rel5.spec @@ -71,7 +71,7 @@ echo "Creating link in update.d" mkdir -p -v $PREFIX/etc/update.d ln -s $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/Update.py $PREFIX/etc/update.d/%{package}_Update.py echo "Running Update script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null echo "Now using python:" which python echo "PYTHONPATH: $PYTHONPATH" @@ -82,7 +82,7 @@ fi if [ -f $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/PostInstall.py ]; then echo "Running PostInstall script" -. $PREFIX/LbLogin.sh --silent --mysiteroot=$PREFIX +. $PREFIX/LbEnv.sh --siteroot $PREFIX 2>/dev/null python $PREFIX/lhcb/%{versiondir}/%{lbversion}/cmt/PostInstall.py fi -- GitLab From 4de57272f4a54402558dcaab90882334a1b58be4 Mon Sep 17 00:00:00 2001 From: Marco Clemencic <marco.clemencic@cern.ch> Date: Thu, 10 Jan 2019 09:21:37 +0100 Subject: [PATCH 3/3] Skip LbEnv/LbLogin when starting containers --- jenkins/docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/docker.sh b/jenkins/docker.sh index a395b681..be65616b 100755 --- a/jenkins/docker.sh +++ b/jenkins/docker.sh @@ -45,6 +45,7 @@ cmd="${main_dir}/jenkins/lb-docker-run \ $propagate_private \ --network host \ --no-interactive \ + --no-lbenv \ --update" for mockCopy in python scripts jenkins setup.sh ; do -- GitLab