Skip to content

GitLab

Commit 1a0644b8 authored by cpomies's avatar cpomies
Browse files

add user launch and change checkout to accept no-checkout

parent 918abfa6
Hide whitespace changes
Inline Side-by-side
jenkins/build.sh
View file @ 1a0644b8
......@@ -15,8 +15,8 @@
set_common --build
get_artifact \
--get-config \
--get-sources \
--get-config \
--get-sources \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")" \
"${ARTIFACTS_DIR}"
......
jenkins/checkout.sh
View file @ 1a0644b8
......@@ -17,20 +17,29 @@ set_common
get_configs_folder --dest-dir "configs"
checkout_slot \
"${flavour}" \
"${slot}" \
"${slot_build_id}" \
--config-dir "configs" \
--dest-dir "${ARTIFACTS_DIR}" \
${build_tool:+--build-tool "${build_tool}"} \
${paltforms:+--platforms "${platforms}"} \
${packages_list:+--packages-list "${packages_list}"} \
${projects_list:+--projects-list "${projects_list}"}
push_artifact \
"${ARTIFACTS_DIR}" \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
if [ "${no_checkout}" == "true" ] ; then
no_checkout_opt="--no-checkout"
fi
checkout_slot \
"${flavour}" \
"${slot}" \
"${slot_build_id}" \
--config-dir "configs" \
--dest-dir "${ARTIFACTS_DIR}" \
${build_tool:+--build-tool "${build_tool}"} \
${paltforms:+--platforms "${platforms}"} \
${packages_list:+--packages-list "${packages_list}"} \
${projects_list:+--projects-list "${projects_list}"} \
${no_checkout_opt}
if [ "${no_checkout}" != "true" ] ; then
push_artifact \
"${ARTIFACTS_DIR}" \
"$(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
fi
check_preconditions \
"${config_file_checkout}" \
......
jenkins/test.sh
View file @ 1a0644b8
......@@ -10,13 +10,10 @@
# or submit itself to any jurisdiction. #
###############################################################################
# Clean LD_LIBRARY_PATH of /gcc/ entries
# (see comment on issue LBCORE-109 http://cern.ch/go/PLQ7)
export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \\n :)
. $(dirname $0)/utils.sh
set_common --build
# Set common environment
set_config=1
. $(dirname $0)/common.sh
day=$(date +%a)
deploybase=$(dirname /data/${ARTIFACTS_DIR})
......
jenkins/user_launch.sh 0 → 100644
View file @ 1a0644b8
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
. $(dirname $0)/utils.sh
set_common
get_configs_folder --dest-dir "configs"
extract_enabled_slots \
"${flavour}" \
"${slots}"} \
${slot_build_id:+--slot-build-id "${slot_build_id}"} \
${rebuild_last_id:+--rebuild-last-id} \
${no_checkout:+--no-checkout}
jenkins/utils.d/checkout_slot.sh
View file @ 1a0644b8
......@@ -11,7 +11,7 @@
function checkout_slot {
local DESCRIPTION="DESCRIPTION : \
local DESCRIPTION="DESCRIPTION : \
Function to checkout a specific slot"
local USAGE="USAGE : \
checkout_slot flavour slot slot_build_id
......@@ -20,155 +20,167 @@ checkout_slot flavour slot slot_build_id
[--build-tool <tool>]
[--platforms <platforms>]
[--packages-list <packages>]
[--projects-list <projects>]"
[--peojects-list <projects>]
[--no-checkout]"
local nb_param=0
local config_dir="."
local dest_dir="."
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--config-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local config_dir="$2"
fi
shift ;;
"--dest-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local dest_dir="$2"
fi
shift ;;
"--build-tool")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local build_tool="$2"
fi
shift ;;
"--platforms")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local platforms="$2"
fi
shift ;;
"--packages-list")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local packages_list="$2"
fi
shift ;;
"--projects-list")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local projects_list="$2"
fi
shift ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slot="$1" ;;
"2")
local slot_build_id="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
local no_checkout=false ;;
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--config-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local config_dir="$2"
fi
shift ;;
"--dest-dir")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local dest_dir="$2"
fi
shift ;;
"--build-tool")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local build_tool="$2"
fi
shift ;;
"--platforms")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local platforms="$2"
fi
shift ;;
"--packages-list")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local packages_list="$2"
fi
shift ;;
"--projects-list")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local projects_list="$2"
fi
shift ;;
"--no-checkout")
local no_checkout=true ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slot="$1" ;;
"2")
local slot_build_id="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
if [ "${no_checkout}" != "true" ]; then
if [ "${nb_param}" != "3" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
if [ "$SET_COMMON" != "true" -o "$GET_CONFIGS_FOLDER" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON and GET_CONFIGS_FOLDER set with true"
exit 1
fi
if [ "$SET_COMMON" != "true" -o "$GET_CONFIGS_FOLDER" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON and GET_CONFIGS_FOLDER set with true"
exit 1
fi
if [ "${slot}" = "lhcb-release" ] ; then
if [ -z "${build_tool}" ] ; then
build_tool=cmt
fi
if [ -n "${platforms}" ] ; then
lbn-gen-release-config --build-tool="${build_tool}" --platforms="${platforms}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
else
lbn-gen-release-config --build-tool="${build_tool}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
fi
fi
if [ "${slot}" = "lhcb-release" ] ; then
if [ -z "${build_tool}" ] ; then
build_tool=cmt
fi
if [ -n "${platforms}" ] ; then
lbn-gen-release-config --build-tool="${build_tool}" --platforms="${platforms}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
else
lbn-gen-release-config --build-tool="${build_tool}" -o configs/${slot}.json --packages "${packages_list}" ${projects_list}
fi
fi
# this allow to bypass the configurations in SVN
if [ -e slot-config.json ] ; then
cp -f -v slot-config.json "${config_dir}/${slot}.json"
if [ -e slot-config.json ] ; then
cp -f -v slot-config.json "${config_dir}/${slot}.json"
fi
fi
if [ -e "${config_dir}/${slot}.json" ] ; then
config_file_checkout="${config_dir}/${slot}.json"
config_file_checkout="${config_dir}/${slot}.json"
else
config_file_checkout="${config_dir}/configuration.xml#${slot}"
config_file_checkout="${config_dir}/configuration.xml#${slot}"
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
submit_opt="--submit --flavour ${flavour}"
fi
if [ "${no_checkout}" != "true" ]; then
if [ "${flavour}" = "release" ] ; then
ignore_error_opt=--no-ignore-checkout-errors
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
submit_opt="--submit --flavour ${flavour}"
fi
if [ "${flavour}" = "release" ] ; then
ignore_error_opt=--no-ignore-checkout-errors
fi
lbn-checkout --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${submit_opt} ${ignore_error_opt} ${config_file_checkout}
lbn-checkout --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${submit_opt} ${ignore_error_opt} ${config_file_checkout}
# We need to copy the configuration at the end because
# StachCkeckout.py cleans the artifacts before starting
cp ${config_file_checkout%%#*} ${dest_dir}
cp ${env_log} ${dest_dir}
echo "$BUILD_URL" > ${dest_dir}/checkout_job_url.txt
cp ${config_file_checkout%%#*} ${dest_dir}
cp ${env_log} ${dest_dir}
echo "$BUILD_URL" > ${dest_dir}/checkout_job_url.txt
if [ "${flavour}" = "release" ] ; then
if [ "${flavour}" = "release" ] ; then
# Now preparing the RPM with the project source
time lbn-rpm --shared --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${config_file_checkout}
if [ -n "${packages_list}" ] ; then
time lbn-rpm --datapkg --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${config_file_checkout}
fi
fi
time lbn-rpm --shared --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${config_file_checkout}
if [ -n "${packages_list}" ] ; then
time lbn-rpm --datapkg --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${dest_dir}" ${config_file_checkout}
fi
fi
rm -rf tmp
rm -rf tmp
fi
export config_file_checkout=${config_file_checkout}
export CONFIG_FILE_CHECKOUT=true
......
jenkins/utils.d/extract_enabled_slots.sh
View file @ 1a0644b8
......@@ -16,7 +16,7 @@ Function to extract all enabled slots from configs file"
local USAGE="USAGE : \
extract_enabled_slots flavour
[--config-dir <dir>]
[--slots <slots>] flavour"
[--slots <slots>]"
local nb_param=0
local slots=""
......
jenkins/utils.d/manage_user_launch.sh 0 → 100644
View file @ 1a0644b8
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
function manage_user_launch {
local DESCRIPTION="DESCRIPTION : \
Function to manage slot launch by user"
local USAGE="USAGE : \
manage_user_launch flavour slots
[--slot-build-id <id>]
[--no-checkout]
[--rebuild-last-id]"
local nb_param=0
local slot_build_id_opt=""
local no_checkout_opt=""
local rebuild_last_id_opt=""
while (( "$#" )); do
if [[ "$1" =~ ^- ]] ; then
case "$1" in
"--slot-build-id")
if [[ "$2" = "" || "$2" =~ ^- ]] ; then
echo "ERROR : Option $1 need an argument"
exit 3
else
local slot_build_id_opt="--slot-build-id $2"
fi
shift ;;
"--no-checkout")
local no_checkout_opt="--no-checkout" ;;
"--rebuild-last-id")
local rebuild_last_id_opt="--rebuild-last-id" ;;
"-h" | "--help")
echo ${DESCRIPTION}
echo ${USAGE}
exit 0;;
*)
echo "ERROR : Option $1 unknow in $0"
echo ${USAGE}
exit 2
esac
else
case "${nb_param}" in
"0")
local flavour="$1" ;;
"1")
local slots="$1" ;;
*)
echo "ERROR : Too much parameter"
echo ${USAGE}
exit 1
esac
local nb_param=$((nb_param+1))
fi
shift
done
if [ "${nb_param}" != "2" ] ; then
echo "ERROR : Need more parameter"
echo ${USAGE}
exit 1
fi
if [ "${slot_build_id_opt}" != "" -a "${rebuild_last_id_opt}" != "" ] ; then
echo "ERROR : $0 can't accept --slot-build-id and --rebuild-last-id"
exit 1
fi
lbn-user-launch --verbose "${flavour}" "slot-params-{0}.txt" "${slots}" ${slot_build_id_opt} ${no_checkout_opt} ${rebuild_last_id_opt}
}
jenkins/utils.d/set_common.sh
View file @ 1a0644b8
......@@ -13,13 +13,13 @@
# Common set up for all the Jenkins scripts
#
DESCRIPTION="DESCRIPTION : \
function set_common {
local DESCRIPTION="DESCRIPTION : \
Function to define common set up for all the Jenckins scripts"
USAGE="USAGE : \
local USAGE="USAGE : \
set_common [--build] [--test]"
function set_common {
local special_config=false
while (( "$#" )); do
......
python/LbNightlyTools/EnabledSlots.py
View file @ 1a0644b8
......@@ -31,13 +31,13 @@ class Script(LbUtils.Script.PlainScript):
This file contain the slot name and the slot build id
The slot build id is extract with the function get_ids
'''
__usage__ = '%prog [options] flavour <output_file.txt> [<slot1> <slot2> ...]'
__usage__ = '%prog [options] flavour output_file.txt'
__version__ = ''
def defineOpts(self):
self.parser.add_option('--config-dir',
action='store',
help='Directoryto find configurations files')
help='Directory to find configurations files')
self.parser.add_option('--slots',
action='store',
......
python/LbNightlyTools/SlotBuildId.py
View file @ 1a0644b8
......@@ -99,3 +99,44 @@ def get_ids(slots, flavour):
xml_parse.write(slot_id_file)
return res
def get_last_ids(slots, flavour):
slot_id_dir = os.path.join(os.environ['JENKINS_HOME'],
'nightlies',
flavour)
# slot_id_dir = 'configs'
slot_id_file = os.path.join(slot_id_dir, 'slot_ids.xml')
if os.path.isfile(slot_id_file):
try:
xml_parse = ET.parse(slot_id_file)
root = xml_parse.getroot()
except:
logging.error('Can''t find or open %s', slot_id_file)
sys.exit(1)
else:
logging.error('get_last_ids need %s to work', slot_id_file)
sys.exit(2)
res = {}
all_slots = dict((el.get('name'), el) for el in root.findall("slot"))
for slot_name in slots:
slot = all_slots.get(slot_name)
if slot is not None:
slot_id = slot.get('last_id')
if not slot_id:
logging.error('No current_id on the slot %s', slot_name)
sys.exit(3)
slot_id = int(slot_id)
else:
logging.error('Slot %s have no entrie in %s',
slot_name,
slot_id_file)
sys.exit(4)
res[slot_name] = slot_id
return res
python/LbNightlyTools/UserLaunch.py 0 → 100644
View file @ 1a0644b8
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Simple script to manage the request from on user job
'''
__author__ = 'Colas Pomies <colas.pomies@cern.ch>'
from LbNightlyTools.Utils import JobParams
from LbNightlyTools.SlotBuildId import get_ids, get_last_ids
import LbUtils.Script
class Script(LbUtils.Script.PlainScript):
'''
Script to create one file for all enable slots or for slots in parameters
This file contain the slot name and the slot build id
The slot build id is extract with the function get_ids
'''
__usage__ = '%prog [options] flavour output_checkout.txt slots'
__version__ = ''
def defineOpts(self):