Skip to content

GitLab

Commit acef3901 authored by cpomies's avatar cpomies
Browse files

Changed build_slot to acept parameter 

Created function for create alias
parent 21530419
Hide whitespace changes
Inline Side-by-side
jenkins/build.sh
View file @ acef3901
......@@ -15,4 +15,4 @@
set_common --build
get_config_file "${flavour}" "${slot}" "${slot_build_id}" "${ARTIFACTS_DIR}"
get_sources "${flavour}" "${slot}" "${slot_build_id}" "${ARTIFACTS_DIR}"
build_slot
build_slot "${flavour}" "${slot}" "${slot_build_id}" "${platform}" "${ARTIFACTS_DIR}" "${os_label}"
jenkins/utils.d/build_slot.sh
View file @ acef3901
function build_slot {
local USAGE="build_slot flavour slot slot_build_id platform directory [os_label]"
if [ $# < 5 ] ; then
echo "ERROR : Usage : ${USAGE}"
exit 1
fi
local flavour="$1"
local slot="$2"
local slot_build_id="$3"
local platform="$4"
local directory="$5"
local os_label="$6"
if [ "$SET_COMMON" != "true" -o "$GET_CONFIG_FILE" != "true" -o "$GET_SOURCES" != "true" -o "$SET_SPECIAL_CONFIG" != "true" ] ; then
echo "ERROR : $0 need SET_COMMON, SET_SPECIAL_CONFIG, GET_SOURCES and GET_CONFIG_FILE set with true"
exit 1
......@@ -7,10 +21,6 @@ function build_slot {
export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \\n :)
day=$(date +%a)
timestamp=$(date -I)
deploybase=$(dirname ${RSYNC_WORKDIR})
# special hack to get a dev version of the CMake configuration files
export CMAKE_PREFIX_PATH=/afs/cern.ch/work/m/marcocle/workspace/LbScripts/LbUtils/cmake:$CMAKE_PREFIX_PATH
......@@ -24,7 +34,7 @@ function build_slot {
export PATH=/build/coverity/static-analysis/bin:/build/coverity:$PATH
fi
config_file=${ARTIFACTS_DIR}/slot-config.json
local config_file=${directory}/slot-config.json
if [ "${os_label}" = "coverity" ] ; then
coverity_opt='--coverity'
......@@ -35,50 +45,49 @@ function build_slot {
if [ "$JENKINS_MOCK" != "true" ] ; then
# create moving symlinks in the artifacts deployment directory (ASAP)
# (ignore errors, see <https://its.cern.ch/jira/browse/LBCORE-153>)
ssh "${RSYNC_SERVER}" "mkdir -pv ${deploybase} ; ln -svfT ${slot_build_id} ${deploybase}/${day} ; ln -svfT ${slot_build_id} ${deploybase}/${timestamp}" || true
create_alias
submit_opt="--submit --flavour ${flavour}"
rsync_opt="--rsync-dest ${RSYNC_DIR}"
rsync_opt="--rsync-dest $(get_remote_directory "$flavour" "$slot" "$slot_build_id")"
fi
# Notify the system of the builds that need to be tested.
if [ "${with_tests}" != "no" ] ; then
lbn-list-expected-builds --slot-build-id ${slot_build_id} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" --platforms "${platform}" -o expected_builds.json ${config_file}
lbn-list-expected-builds --slot-build-id ${slot_build_id} --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --platforms "${platform}" -o expected_builds.json ${config_file}
if [ "$JENKINS_MOCK" != "true" ] ; then
datadir=${JENKINS_HOME}/nightlies/${flavour}/running_builds
scp expected_builds.json buildlhcb.cern.ch:${datadir}/expected_builds.${slot}.${slot_build_id}.${platform}.json
fi
fi
time lbn-build --no-distcc --verbose --jobs 8 --timeout 18000 --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" --clean ${submit_opt} ${rsync_opt} ${coverity_opt} ${config_file}
time lbn-build --no-distcc --verbose --jobs 8 --timeout 18000 --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --clean ${submit_opt} ${rsync_opt} ${coverity_opt} ${config_file}
if [ "${flavour}" = "release" ] ; then
# Prepare the RPMs
time lbn-rpm --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file} --platform "${platform}"
time lbn-rpm --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file} --platform "${platform}"
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
push_artifact "${ARTIFACTS_DIR}" "${flavour}" "${slot}" "${slot_build_id}"
push_artifact "${directory}" "${flavour}" "${slot}" "${slot_build_id}"
fi
# if possible and requested, generate glimpse indexes and upload them to buildlhcb
if [ "${flavour}" = "release" -o -n "${run_indexer}" ] ; then
if which glimpseindex &> /dev/null ; then
# clean up the build dir before indexing
lbn-build --verbose --clean --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" --clean ${config_file}
time lbn-index --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file}
lbn-build --verbose --clean --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" --clean ${config_file}
time lbn-index --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file}
if [ "${flavour}" = "release" ] ; then
time lbn-rpm --glimpse --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${ARTIFACTS_DIR}" ${config_file}
time lbn-rpm --glimpse --verbose --build-id "${slot}.${slot_build_id}" --artifacts-dir "${directory}" ${config_file}
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
push_artifact "${ARTIFACTS_DIR}" "${flavour}" "${slot}" "${slot_build_id}"
push_artifact "${directory}" "${flavour}" "${slot}" "${slot_build_id}"
fi
fi
fi
if [ "$JENKINS_MOCK" != "true" ] ; then
# Clean up
rm -rf ${ARTIFACTS_DIR} build
rm -rf ${directory} build
fi
}
\ No newline at end of file
jenkins/utils.d/get_remote_directory.sh → jenkins/utils.d/manage_remote_directory.sh
View file @ acef3901
......@@ -21,3 +21,18 @@ function get_remote_directory {
echo "${RSYNC_DIR}"
}
function create_alias {
local day=$(date +%a)
local timestamp=$(date -I)
local RSYNC_DIR="${RSYNC_WORKDIR}/${flavour}/${slot}"
local COMMAND="mkdir -pv ${RSYNC_DIR} ; ln -svfT ${slot_build_id} ${RSYNC_DIR}/${day} ; ln -svfT ${slot_build_id} ${RSYNC_DIR}/${timestamp}"
if [ ! -d "${RSYNC_WORKDIR}" ] ; then
local COMMAND="ssh ${RSYNC_SERVER} ${COMMAND} || true"
fi
${COMMAND}
}
\ No newline at end of file
python/LbNightlyTools/tests/test_enabled_slots_script.py
View file @ acef3901
......@@ -31,7 +31,6 @@ def setup():
global _env_bk
_env_bk = dict(os.environ)
os.environ['JENKINS_HOME'] = 'JENKINS_HOME'
os.environ['flavour'] = 'flavour'
def teardown():
global _env_bk
......@@ -51,7 +50,7 @@ def test_wrong_number_argument():
def test_no_data():
with TemporaryDir(chdir=True):
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -60,7 +59,7 @@ def test_no_file():
with TemporaryDir(chdir=True):
os.makedirs('./configs')
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -78,9 +77,7 @@ def test_one_file_json_chmod_111():
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
os.chmod('configs/lhcb-TEST.json', 0111)
slots = EnabledSlots.Script().extract_from_json('lhcb-*.json')
EnabledSlots.Script().write_files(slots, 'slot-param-{0}.txt')
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -99,7 +96,7 @@ def test_one_file_json_disabled_flase():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -119,7 +116,7 @@ def test_one_file_json_disabled_true():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -138,7 +135,7 @@ def test_one_file_json_no_disabled():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -157,7 +154,7 @@ def test_one_file_json_no_slot():
os.makedirs('./configs')
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST.json'))
assert json.load(open('configs/lhcb-TEST.json')) == conf_data
......@@ -184,7 +181,7 @@ def test_two_file_json():
slot_file.write(json.dumps(conf_data1))
with open('configs/lhcb-TEST2.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data2))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/lhcb-TEST1.json'))
assert exists(join('configs/lhcb-TEST2.json'))
......@@ -205,7 +202,7 @@ def test_one_job_xml_disbaled_false():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 1
......@@ -223,7 +220,7 @@ def test_one_job_xml_disabled_true():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 0
......@@ -241,7 +238,7 @@ def test_one_job_xml_no_disabled():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 1
......@@ -261,7 +258,7 @@ def test_two_job_xml():
os.makedirs('./configs')
with open('configs/configuration.xml', 'w') as cfg_file:
cfg_file.write(test_xml)
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 2
......@@ -289,7 +286,7 @@ def test_same_job_xml_and_json():
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert exists(join('configs/lhcb-TEST.json'))
......@@ -318,7 +315,7 @@ def test_different_job_xml_and_json():
with open('configs/lhcb-TEST.json', 'w') as slot_file:
slot_file.write(json.dumps(conf_data))
retval = EnabledSlots.Script().run(['slot-param-{0}.txt'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt'])
assert retval == 0
assert exists(join('configs/configuration.xml'))
assert exists(join('configs/lhcb-TEST.json'))
......@@ -327,7 +324,7 @@ def test_different_job_xml_and_json():
def test_one_job_param():
with TemporaryDir(chdir=True):
retval = EnabledSlots.Script().run(['slot-param-{0}.txt', 'lhcb-test'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt', 'lhcb-test'])
assert retval == 0
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 1
......@@ -335,6 +332,6 @@ def test_one_job_param():
def test_two_job_param():
with TemporaryDir(chdir=True):
retval = EnabledSlots.Script().run(['slot-param-{0}.txt', 'lhcb-test', 'lhcb-test2'])
retval = EnabledSlots.Script().run(['testing', 'slot-param-{0}.txt', 'lhcb-test', 'lhcb-test2'])
assert retval == 0
assert len([x for x in os.listdir('.') if re.match(r'^slot-param-.*\.txt', x)]) == 2
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment