Commit 0615e504 authored by Marco Clemencic's avatar Marco Clemencic
Browse files

Merge branch 'testing-v2'

Conflicts:
	jenkins/build.sh
	jenkins/enabled_slots.sh
	python/LbNightlyTools/EnabledSlots.py
	python/LbNightlyTools/SlotBuildId.py
	python/LbNightlyTools/tests/test_enabled_slots_script.py
parents e9e222ce 0bc7c1a8
......@@ -10,84 +10,20 @@
# or submit itself to any jurisdiction. #
###############################################################################
# Clean LD_LIBRARY_PATH of /gcc/ entries
# (see comment on issue LBCORE-109 http://cern.ch/go/PLQ7)
export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \\n :)
# Set common environment
set_config=1
. $(dirname $0)/common.sh
day=$(date +%a)
timestamp=$(date -I)
deploybase=$(dirname /data/${ARTIFACTS_DIR})
# special hack to get a dev version of the CMake configuration files
export CMAKE_PREFIX_PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/cmake:$CMAKE_PREFIX_PATH
# ensure that the distcc lock directory exists
if [ -n "$DISTCC_DIR" ] ; then
mkdir -pv $DISTCC_DIR
fi
# ensure that Coverity is on the PATH
if [ -e /build/coverity/static-analysis/bin ] ; then
export PATH=/build/coverity/static-analysis/bin:/build/coverity:$PATH
fi
config_file=${ARTIFACTS_DIR}/slot-config.json
if [ "${os_label}" = "coverity" ] ; then
coverity_opt='--coverity'
# Coverity builds to not need to trigger tests
with_tests=no
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
# create moving symlinks in the artifacts deployment directory (ASAP)
# (ignore errors, see <https://its.cern.ch/jira/browse/LBCORE-153>)
ssh buildlhcb.cern.ch "mkdir -pv ${deploybase} ; ln -svfT ${slot_build_id} ${deploybase}/${day} ; ln -svfT ${slot_build_id} ${deploybase}/${timestamp} ; ln -svfT ${slot_build_id} ${deploybase}/Today" || true
submit_opt="--submit --flavour ${flavour}"
rsync_opt="--rsync-dest buildlhcb.cern.ch:${deploybase}/${slot_build_id}"
fi
# Notify the system of the builds that need to be tested.
if [ "${with_tests}" != "no" ] ; then
lbn-list-expected-builds --slot-build-id ${slot_build_id} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" --platforms "${platform}" -o expected_builds.json ${config_file}
if [ "$JENKINS_MOCK" != "true" ] ; then
datadir=${JENKINS_HOME}/nightlies/${flavour}/running_builds
scp expected_builds.json buildlhcb.cern.ch:${datadir}/expected_builds.${slot}.${slot_build_id}.${platform}.json
fi
fi
time lbn-build --no-distcc --verbose --jobs 8 --timeout 18000 --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" --clean ${submit_opt} ${rsync_opt} ${coverity_opt} ${config_file}
if [ "${flavour}" = "release" ] ; then
# Prepare the RPMs
time lbn-rpm --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file} --platform "${platform}"
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
rsync --archive --whole-file --partial-dir=.rsync-partial.$(hostname).$$ --delay-updates --rsh=ssh "${ARTIFACTS_DIR}/" "buildlhcb.cern.ch:${deploybase}/${slot_build_id}"
fi
# if possible and requested, generate glimpse indexes and upload them to buildlhcb
if [ "${flavour}" = "release" -o -n "${run_indexer}" ] ; then
if which glimpseindex &> /dev/null ; then
# clean up the build dir before indexing
lbn-build --verbose --clean --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" --clean ${config_file}
time lbn-index --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file}
if [ "${flavour}" = "release" ] ; then
time lbn-rpm --glimpse --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file}
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
rsync --archive --whole-file --partial-dir=.rsync-partial.$(hostname).$$ --delay-updates --rsh=ssh "${ARTIFACTS_DIR}/" "buildlhcb.cern.ch:${deploybase}/${slot_build_id}"
fi
fi
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
# Clean up
rm -rf ${ARTIFACTS_DIR} build
fi
. $(dirname $0)/utils.sh
set_common --build
get_artifact \
--get-config \
--get-sources \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")" \
"${ARTIFACTS_DIR}"
build_slot \
"${flavour}" \
"${slot}" \
"${slot_build_id}" \
"${platform}" \
--build-dir "${ARTIFACTS_DIR}" \
${os_label:+--os-label "${os_label}"}
......@@ -10,59 +10,39 @@
# or submit itself to any jurisdiction. #
###############################################################################
# Set common environment
. $(dirname $0)/common.sh
if [ "$JENKINS_MOCK" != "true" -o ! -e configs ] ; then
# Get the slot configuration files from Subversion
lbn-get-configs
fi
if [ "${slot}" = "lhcb-release" ] ; then
if [ -z "${build_tool}" ] ; then
build_tool=cmt
fi
if [ -n "${platforms}" ] ; then
lbn-gen-release-config --build-tool="${build_tool}" --platforms="${platforms}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
else
lbn-gen-release-config --build-tool="${build_tool}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
fi
fi
. $(dirname $0)/utils.sh
# this allow to bypass the configurations in SVN
if [ -e slot-config.json ] ; then
cp -f -v slot-config.json configs/${slot}.json
fi
# Set common environment
set_common
if [ -e configs/${slot}.json ] ; then
config_file=configs/${slot}.json
else
config_file=configs/configuration.xml#${slot}
fi
get_configs_folder --dest-dir "configs"
if [ "$JENKINS_MOCK" != "true" ] ; then
submit_opt="--submit --flavour ${flavour}"
if [ "${no_checkout}" == "true" ] ; then
no_checkout_opt="--no-checkout"
fi
if [ "${flavour}" = "release" ] ; then
ignore_error_opt=--no-ignore-checkout-errors
checkout_slot \
"${flavour}" \
"${slot}" \
"${slot_build_id}" \
--config-dir "configs" \
--dest-dir "${ARTIFACTS_DIR}" \
${build_tool:+--build-tool "${build_tool}"} \
${paltforms:+--platforms "${platforms}"} \
${packages_list:+--packages-list "${packages_list}"} \
${projects_list:+--projects-list "${projects_list}"} \
${no_checkout_opt}
if [ "${no_checkout}" != "true" ] ; then
push_artifact \
"${ARTIFACTS_DIR}" \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
lbn-checkout --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${submit_opt} ${ignore_error_opt} ${config_file}
check_preconditions \
"${config_file_checkout}" \
"${slot}" \
"${slot_build_id}" \
${platforms:+--platforms "${platforms}"}
# We need to copy the configuration at the end because
# StachCkeckout.py cleans the artifacts before starting
cp ${config_file%%#*} ${ARTIFACTS_DIR}
cp ${env_log} ${ARTIFACTS_DIR}
echo "$BUILD_URL" > ${ARTIFACTS_DIR}/checkout_job_url.txt
if [ "${flavour}" = "release" ] ; then
# Now preparing the RPM with the project source
time lbn-rpm --shared --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file}
if [ -n "${packages_list}" ] ; then
time lbn-rpm --datapkg --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file}
fi
fi
# Cleaning up
rm -rf tmp
......@@ -31,6 +31,9 @@ export LHCB_NIGHTLY_MAX_THREADS=1
export ARTIFACTS_DIR=${ARTIFACTS_DIR:-artifacts/${flavour}/${slot}/${slot_build_id}}
mkdir -p ${ARTIFACTS_DIR}
export RSYNC_SERVER=${RSYNC_SERVER:-*********} # TODO
export RSYNC_WORKDIR=${RSYNC_WORKDIR:-*********} # TODO
export RSYNC_DIR=${RSYNC_DIR:-${RSYNC_SERVER}:${RSYNC_WORKDIR}/${flavour}/${slot}/${slot_build_id}}
export TMPDIR=${WORKSPACE}/tmp
mkdir -p ${TMPDIR}
......@@ -80,12 +83,6 @@ if [ $(python -c 'import sys; print "%d%d" % sys.version_info[:2]') = 24 ] ; the
. SetupProject.sh LCGCMT 66 Python
fi
if [ -e ${ARTIFACTS_DIR}/${slot}.json ] ; then
config_file=${ARTIFACTS_DIR}/${slot}.json
else
config_file=${ARTIFACTS_DIR}/configuration.xml#${slot}
fi
if klist -5 > /dev/null 2>&1 ; then
kinit -R
klist -5
......
......@@ -10,15 +10,14 @@
# or submit itself to any jurisdiction. #
###############################################################################
# hack because of a bug with non-writable home (this script is run by tomcat)
export HOME=$PWD
. $(dirname $0)/utils.sh
# Set common environment
. $(dirname $0)/common.sh
set_common
if [ "$JENKINS_MOCK" != "true" -o ! -e configs ] ; then
# Get the slot configuration files from Subversion
lbn-get-configs
fi
get_configs_folder --dest-dir "configs"
extract_enabled_slots \
"${flavour}" \
${slots:+--slots "${slots}"} \
--config-dir "configs"
lbn-enabled-slots --verbose 'slot-params-{0}.txt' ${slots}
\ No newline at end of file
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
function manage_user_launch {
local DESCRIPTION="DESCRIPTION : \
Function to manage slot launch by user"
local USAGE="USAGE : \
manage_user_launch flavour slots
[--slot-build-id <id>]
[--no-checkout]
[--rebuild-last-id]"
local nb_param=0
local slot_build_id_opt=""
local no_checkout_opt=""
local rebuild_last_id_opt=""
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--slot-build-id")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local slot_build_id_opt="--slot-build-id $2"
fi
shift ;;
"--no-checkout")
local no_checkout_opt="--no-checkout" ;;
"--rebuild-last-id")
local rebuild_last_id_opt="--rebuild-last-id" ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slots="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
if [ "${nb_param}" != "2" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
if [ "${slot_build_id_opt}" != "" -a "${rebuild_last_id_opt}" != "" ] ; then
echo "ERROR : $0 can't accept --slot-build-id and --rebuild-last-id"
exit 1
fi
lbn-user-launch --verbose "${flavour}" "slot-params-{0}.txt" "${slots}" ${slot_build_id_opt} ${no_checkout_opt} ${rebuild_last_id_opt}
}
......@@ -46,6 +46,9 @@ export JOB_NAME=${JOB_NAME:-nightly-test-slot-build-platform}
guessed_label=${platform#*-}
guessed_label=${guessed_label%%-*}
export os_label=${os_label:-${guessed_label}}
export JENKINS_HOME=${JENKINS_HOME:-jenkins_home}
export RSYNC_SERVER=${RSYNC_SERVER:-${USERNAME}@${HOSTNAME}}
export RSYNC_WORKDIR=${RSYNC_WORKDIR:-$(cd $(dirname $0)/.. ;pwd)/rsync}
# this variable might be used inside the Jenkins scripts to avoid some ops
export JENKINS_MOCK=true
......
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
rm expected_builds.json
rm -f build.sh.*.env
rm -rf artifacts build tmp
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
rm -f slot-precondition-*.txt slot-build-*.txt
rm -f checkout.sh.*.env check_preconditions.sh.*.env
rm -rf artifacts tmp
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
rm -f slot-params-*.txt
rm -f enabled_slots.sh.*.env
rm -rf artifacts ${JENKINS_HOME} tmp
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
rm -f preconditions.sh.*.env
rm -rf artifacts tmp
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
ssh ${RSYNC_SERVER} rm -rf ${RSYNC_WORKDIR}
......@@ -10,12 +10,13 @@
# or submit itself to any jurisdiction. #
###############################################################################
# hack because of a bug with non-writable home (this script is run by tomcat)
export HOME=$PWD
. $(dirname $0)/utils.sh
# Set common environment
. $(dirname $0)/common.sh
set_common
export CMTCONFIG=$platform
get_artifact \
--get-config \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")" \
"${ARTIFACTS_DIR}"
lbn-check-preconditions --verbose ${config_file}
execute_preconditions "${config_file}"
......@@ -10,13 +10,10 @@
# or submit itself to any jurisdiction. #
###############################################################################
# Clean LD_LIBRARY_PATH of /gcc/ entries
# (see comment on issue LBCORE-109 http://cern.ch/go/PLQ7)
export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \\n :)
. $(dirname $0)/utils.sh
set_common --build
# Set common environment
set_config=1
. $(dirname $0)/common.sh
day=$(date +%a)
deploybase=$(dirname /data/${ARTIFACTS_DIR})
......
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
. $(dirname $0)/utils.sh
set_common
get_configs_folder --dest-dir "configs"
if [ "${rebuild_last_id}" == "true" ] ; then
rebuild_last_id_opt="--rebuild-last-id"
fi
if [ "${no_checkout}" == "true" ] ; then
no_checkout_opt="--no-checkout"
fi
manage_user_launch \
"${flavour}" \
"${slots}" \
${slot_build_id:+--slot-build-id "${slot_build_id}"} \
${rebuild_last_id_opt} \
${no_checkout_opt}
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
function build_slot {
loglevel_opt="--debug"
local DESCRIPTION="DESCRIPTION : \
Function to build a slot on a specify platform"
local USAGE="USAGE : \
build_slot flavour slot slot_build_id platform
[--build-dir <dir>]
[--os-label <label>]"
local nb_param=0
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--build-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local directory="$2"
fi
shift ;;
"--os-label")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local os_label="$2"
fi
shift ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slot="$1" ;;
"2")
local slot_build_id="$1" ;;
"3")
local platform="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
if [ "${nb_param}" != "4" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
# special hack to get a dev version of the CMake configuration files
export CMAKE_PREFIX_PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/cmake:$CMAKE_PREFIX_PATH
# ensure that the distcc lock directory exists
if [ -n "$DISTCC_DIR" ] ; then
mkdir -pv $DISTCC_DIR
fi
# ensure that Coverity is on the PATH
if [ -e /build/coverity/static-analysis/bin ] ; then
export PATH=/build/coverity/static-analysis/bin:/build/coverity:$PATH
fi
local config_file=${directory}/slot-config.json
if [ "${os_label}" = "coverity" ] ; then
coverity_opt='--coverity'
# Coverity builds to not need to trigger tests
with_tests=no
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
# create moving symlinks in the artifacts deployment directory (ASAP)
# (ignore errors, see <https://its.cern.ch/jira/browse/LBCORE-153>)
create_alias
submit_opt="--submit --flavour ${flavour}"
rsync_opt="--rsync-dest $(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
# Notify the system of the builds that need to be tested.
if [ "${with_tests}" != "no" ] ; then
lbn-list-expected-builds --slot-build-id ${slot_build_id} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --platforms "${platform}" -o expected_builds.json ${config_file}
if [ "$JENKINS_MOCK" != "true" ] ; then
datadir=${JENKINS_HOME}/nightlies/${flavour}/running_builds
scp expected_builds.json buildlhcb.cern.ch:${datadir}/expected_builds.${slot}.${slot_build_id}.${platform}.json
fi
fi
time lbn-build --no-distcc ${loglevel_opt} --jobs 8 --timeout 18000 --build-id "${slot}.${</