Commit d043993a authored by cpomies's avatar cpomies
Browse files

Change all Function to add more parameters and options

parent 7f6dd290
......@@ -13,6 +13,17 @@
. $(dirname $0)/utils.sh
set_common --build
get_config_file "${flavour}" "${slot}" "${slot_build_id}" "${ARTIFACTS_DIR}"
get_sources "${flavour}" "${slot}" "${slot_build_id}" "${ARTIFACTS_DIR}"
build_slot "${flavour}" "${slot}" "${slot_build_id}" "${platform}" "${ARTIFACTS_DIR}" "${os_label}"
get_artifact \
--get-config \
--get-sources \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")" \
"${ARTIFACTS_DIR}"
build_slot \
"${flavour}" \
"${slot}" \
"${slot_build_id}" \
"${platform}" \
--build-dir "${ARTIFACTS_DIR}" \
${os_label:+--os-label "${os_label}"}
......@@ -14,18 +14,28 @@
# Set common environment
set_common
get_configs_folder
get_configs_folder --dest-dir "configs"
checkout_slot \
"${flavour}" \
"${slot}" \
"${slot_build_id}" \
--config-dir "configs" \
--dest-dir "${ARTIFACTS_DIR}" \
${build_tool:+--build-tool "${build_tool}"} \
${paltforms:+--platforms "${platforms}"} \
${packages_list:+--packages-list "${packages_list}"} \
${projects_list:+--projects-list "${projects_list}"}
push_artifact \
"${ARTIFACTS_DIR}" \
"${build_tool}" \
"${platforms}" \
"${packages_list}" \
"${peojects_list}"
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
push_artifact "${ARTIFACTS_DIR}" "${flavour}" "${slot}" "${slot_build_id}"
check_preconditions "${config_file_checkout}" "${slot}" "${slot_build_id}" "${platforms}"
check_preconditions \
"${config_file_checkout}" \
"${slot}" \
"${slot_build_id}" \
${platforms:+--platforms "${platforms}"}
......@@ -13,5 +13,10 @@
. $(dirname $0)/utils.sh
set_common
get_configs_folder
extract_enabled_slots "${flavour}" "${slots}"
get_configs_folder --dest-dir "configs"
extract_enabled_slots \
"${flavour}" \
${slots:+--slots "${slots}"} \
--config-dir "configs"
......@@ -13,5 +13,10 @@
. $(dirname $0)/utils.sh
set_common
get_config_file "${flavour}" "${slot}" "${slot_build_id}" "${ARTIFACTS_DIR}"
get_artifact \
--get-config \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")" \
"${ARTIFACTS_DIR}"
execute_preconditions "${config_file}"
......@@ -11,92 +11,139 @@
function build_slot {
local USAGE="build_slot flavour slot slot_build_id platform directory [os_label]"
if [ $# < 5 ] ; then
echo "ERROR : Usage : ${USAGE}"
exit 1
fi
local flavour="$1"
local slot="$2"
local slot_build_id="$3"
local platform="$4"
local directory="$5"
local os_label="$6"
if [ "$SET_COMMON" != "true" -o "$GET_CONFIG_FILE" != "true" -o "$GET_SOURCES" != "true" -o "$SET_SPECIAL_CONFIG" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON, SET_SPECIAL_CONFIG, GET_SOURCES and GET_CONFIG_FILE set with true"
exit 1
fi
local DESCRIPTION="DESCRIPTION : \
Function to build a slot on a specify platform"
local USAGE="USAGE : \
build_slot flavour slot slot_build_id platform
[--build-dir <dir>]
[--os-label <label>]"
local nb_param=0
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--build-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local directory="$2"
fi
shift ;;
"--os-label")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local os_label="$2"
fi
shift ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slot="$1" ;;
"2")
local slot_build_id="$1" ;;
"3")
local platform="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
if [ "${nb_param}" != "4" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
# special hack to get a dev version of the CMake configuration files
export CMAKE_PREFIX_PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/cmake:$CMAKE_PREFIX_PATH
# ensure that the distcc lock directory exists
if [ -n "$DISTCC_DIR" ] ; then
mkdir -pv $DISTCC_DIR
mkdir -pv $DISTCC_DIR
fi
# ensure that Coverity is on the PATH
if [ -e /build/coverity/static-analysis/bin ] ; then
export PATH=/build/coverity/static-analysis/bin:/build/coverity:$PATH
export PATH=/build/coverity/static-analysis/bin:/build/coverity:$PATH
fi
local config_file=${directory}/slot-config.json
if [ "${os_label}" = "coverity" ] ; then
coverity_opt='--coverity'
# Coverity builds to not need to trigger tests
with_tests=no
coverity_opt='--coverity'
# Coverity builds to not need to trigger tests
with_tests=no
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
# create moving symlinks in the artifacts deployment directory (ASAP)
# (ignore errors, see <https://its.cern.ch/jira/browse/LBCORE-153>)
create_alias
submit_opt="--submit --flavour ${flavour}"
rsync_opt="--rsync-dest $(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
# create moving symlinks in the artifacts deployment directory (ASAP)
# (ignore errors, see <https://its.cern.ch/jira/browse/LBCORE-153>)
create_alias
submit_opt="--submit --flavour ${flavour}"
rsync_opt="--rsync-dest $(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
# Notify the system of the builds that need to be tested.
if [ "${with_tests}" != "no" ] ; then
lbn-list-expected-builds --slot-build-id ${slot_build_id} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --platforms "${platform}" -o expected_builds.json ${config_file}
if [ "$JENKINS_MOCK" != "true" ] ; then
datadir=${JENKINS_HOME}/nightlies/${flavour}/running_builds
scp expected_builds.json buildlhcb.cern.ch:${datadir}/expected_builds.${slot}.${slot_build_id}.${platform}.json
fi
lbn-list-expected-builds --slot-build-id ${slot_build_id} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --platforms "${platform}" -o expected_builds.json ${config_file}
if [ "$JENKINS_MOCK" != "true" ] ; then
datadir=${JENKINS_HOME}/nightlies/${flavour}/running_builds
scp expected_builds.json buildlhcb.cern.ch:${datadir}/expected_builds.${slot}.${slot_build_id}.${platform}.json
fi
fi
time lbn-build --no-distcc --verbose --jobs 8 --timeout 18000 --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --clean ${submit_opt} ${rsync_opt} ${coverity_opt} ${config_file}
if [ "${flavour}" = "release" ] ; then
# Prepare the RPMs
time lbn-rpm --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file} --platform "${platform}"
# Prepare the RPMs
time lbn-rpm --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file} --platform "${platform}"
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
push_artifact "${directory}" "${flavour}" "${slot}" "${slot_build_id}"
push_artifact "${directory}" "$(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
# if possible and requested, generate glimpse indexes and upload them to buildlhcb
if [ "${flavour}" = "release" -o -n "${run_indexer}" ] ; then
if which glimpseindex &> /dev/null ; then
# clean up the build dir before indexing
lbn-build --verbose --clean --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --clean ${config_file}
time lbn-index --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file}
if [ "${flavour}" = "release" ] ; then
time lbn-rpm --glimpse --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file}
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
push_artifact "${directory}" "${flavour}" "${slot}" "${slot_build_id}"
fi
fi
if which glimpseindex &> /dev/null ; then
# clean up the build dir before indexing
lbn-build --verbose --clean --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --clean ${config_file}
time lbn-index --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file}
if [ "${flavour}" = "release" ] ; then
time lbn-rpm --glimpse --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file}
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
push_artifact "${directory}" "$(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
fi
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
# Clean up
rm -rf ${directory} build
# Clean up
rm -rf ${directory} build
fi
}
\ No newline at end of file
}
......@@ -10,28 +10,66 @@
###############################################################################
function check_preconditions {
local USAGE="check_preconditions config_file slot slot_build_id [platforms]"
if [ $# != 3 -a $# != 4 ] ; then
echo "ERROR : Usage : ${USAGE}"
exit 1
fi
local DESCRIPTION="DESCRIPTION : \
Function to check if we have precondition on an specify slot"
local USAGE="USAGE : \
check_preconditions config_file slot slot_build_id
[--platforms <platforms>]"
if [ "$SET_COMMON" != "true" -o "$CONFIG_FILE_CHECKOUT" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON and CONFIG_FILE_CHECKOUT set with true"
exit 1
fi
local nb_param=0
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--platforms")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local platforms="$2"
fi
shift ;;
local config="$1"
local slot="$2"
local slot_build_id="$3"
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local config="$1" ;;
"1")
local slot="$1" ;;
"2")
local slot_build_id="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
local platforms_opt=''
if [ $# == 4 -a "$4" != "" ] ; then
lbn-check-preconditions --verbose "${config}" "$slot" "$slot_build_id" --platforms "$4"
else
lbn-check-preconditions --verbose "${config}" "$slot" "$slot_build_id"
shift
done
if [ "${nb_param}" != "3" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
if [ "$SET_COMMON" != "true" -o "$CONFIG_FILE_CHECKOUT" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON and CONFIG_FILE_CHECKOUT set with true"
exit 1
fi
lbn-check-preconditions --verbose "${config}" "$slot" "$slot_build_id" ${platforms:+--platforms "${platforms}"}
}
......@@ -11,73 +11,161 @@
function checkout_slot {
local USAGE="checkout_slot flavour slot slot_build_id directory [build_tool platforms packages_list projects_list]"
if [ $# < 4 ] ; then
echo "ERROR : Usage : ${USAGE}"
exit 1
fi
local flavour=$1
local slot=$2
local slot_build_id=$3
local directory=$4
local build_tool=$5
local platforms=$6
local packages_list=$7
local projects_list=$8
local DESCRIPTION="DESCRIPTION : \
Function to checkout a specific slot"
local USAGE="USAGE : \
checkout_slot flavour slot slot_build_id
[--config-dir <dir>]
[--dest-dir <dir>]
[--build-tool <tool>]
[--platforms <platforms>]
[--packages-list <packages>]
[--projects-list <projects>]"
local nb_param=0
local config_dir="."
local dest_dir="."
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--config-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local config_dir="$2"
fi
shift ;;
"--dest-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local dest_dir="$2"
fi
shift ;;
"--build-tool")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local build_tool="$2"
fi
shift ;;
"--platforms")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local platforms="$2"
fi
shift ;;
"--packages-list")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local packages_list="$2"
fi
shift ;;
"--projects-list")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local projects_list="$2"
fi
shift ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slot="$1" ;;
"2")
local slot_build_id="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
if [ "${nb_param}" != "3" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
if [ "$SET_COMMON" != "true" -o "$GET_CONFIGS_FOLDER" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON and GET_CONFIGS_FOLDER set with true"
exit 1
echo "ERROR : $0 need SET_COMMON and GET_CONFIGS_FOLDER set with true"
exit 1
fi
if [ "${slot}" = "lhcb-release" ] ; then
if [ -z "${build_tool}" ] ; then
build_tool=cmt
fi
if [ -n "${platforms}" ] ; then
lbn-gen-release-config --build-tool="${build_tool}" --platforms="${platforms}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
else
lbn-gen-release-config --build-tool="${build_tool}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
fi
if [ -z "${build_tool}" ] ; then
build_tool=cmt
fi
if [ -n "${platforms}" ] ; then
lbn-gen-release-config --build-tool="${build_tool}" --platforms="${platforms}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
else
lbn-gen-release-config --build-tool="${build_tool}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
fi
fi
# this allow to bypass the configurations in SVN
if [ -e slot-config.json ] ; then
cp -f -v slot-config.json configs/${slot}.json
cp -f -v slot-config.json "${config_dir}/${slot}.json"
fi
if [ -e configs/${slot}.json ] ; then
config_file_checkout=configs/${slot}.json
if [ -e "${config_dir}/${slot}.json" ] ; then
config_file_checkout="${config_dir}/${slot}.json"
else
config_file_checkout=configs/configuration.xml#${slot}
config_file_checkout="${config_dir}/configuration.xml#${slot}"
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
submit_opt="--submit --flavour ${flavour}"
submit_opt="--submit --flavour ${flavour}"
fi
if [ "${flavour}" = "release" ] ; then
ignore_error_opt=--no-ignore-checkout-errors
ignore_error_opt=--no-ignore-checkout-errors
fi
lbn-checkout --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${submit_opt} ${ignore_error_opt} ${config_file_checkout}
lbn-checkout --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${submit_opt} ${ignore_error_opt} ${config_file_checkout}
# We need to copy the configuration at the end because
# StachCkeckout.py cleans the artifacts before starting
cp ${config_file_checkout%%#*} ${ARTIFACTS_DIR}
cp ${env_log} ${ARTIFACTS_DIR}
echo "$BUILD_URL" > ${ARTIFACTS_DIR}/checkout_job_url.txt
cp ${config_file_checkout%%#*} ${dest_dir}
cp ${env_log} ${dest_dir}
echo "$BUILD_URL" > ${dest_dir}/checkout_job_url.txt
if [ "${flavour}" = "release" ] ; then
# Now preparing the RPM with the project source
time lbn-rpm --shared --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file_checkout}
if [ -n "${packages_list}" ] ; then
time lbn-rpm --datapkg --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file_checkout}
fi
# Now preparing the RPM with the project source
time lbn-rpm --shared --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${config_file_checkout}
if [ -n "${packages_list}" ] ; then
time lbn-rpm --datapkg --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${config_file_checkout}
fi
fi
rm -rf tmp
......
......@@ -11,19 +11,50 @@
function execute_preconditions {
local USAGE="execute_preconditions config_file"
local DESCRIPTION="DESCRIPTION : \
Function to execute precondition contain in a specify file"
local USAGE="USAGE : \
execute_preconditions config_file"
if [ $# != 1 ] ; then
echo "ERROR : Usage : ${USAGE}"
exit 1
fi
local nb_param=0
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local config="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac