Commit 133047a9 authored by Pere Mato Vila's avatar Pere Mato Vila
Browse files

Added pipelines for Jenkins jobs

parent f362d2c5
//----------------------------------------------------------------------------------------------------------------------
// This declarative Jenkins pipeline encodes all the steps required for the nightly/continuous of a single platform.
// Other jobs may call this pipeline to execute the build, test and installation of a set platforms.
//
// Author: Pere Mato
//----------------------------------------------------------------------------------------------------------------------
pipeline {
parameters {
string(name: 'EXTERNALS', defaultValue: 'devgeantv/latest', description: 'LCG software stack in CVMFS')
choice(name: 'MODE', choices: ['experimental', 'nightly', 'continuous'], description: 'CDash mode')
string(name: 'ExtraCMakeOptions', defaultValue: '', description: 'CMake extra configuration options')
string(name: 'LABEL', defaultValue: 'centos7', description: 'Jenkins label for physical nodes or container image for docker')
choice(name: 'COMPILER', choices: ['gcc7', 'gcc8', 'gcc9', 'gcc10', 'clang8', 'clang10', 'native'])
choice(name: 'BUILDTYPE', choices: ['Release', 'Debug'])
choice(name: 'OPTION', choices: ['default', 'SPEC', 'AVX', 'GDML'])
choice(name: 'BACKEND', choices: ['scalar', 'vc'])
string(name: 'DOCKER_LABEL', defaultValue: 'docker-host-noafs', description: 'Label for the the nodes able to launch docker images')
string(name: 'SourceBranch', defaultValue: 'master', description: 'Source branch in repository')
string(name: 'TargetBranch', defaultValue: 'master', description: 'Target branch in repository')
}
environment {
CMAKE_INSTALL_PREFIX = 'install'
CMAKE_SOURCE_DIR = 'vecgeom'
CMAKE_BINARY_DIR = 'build'
}
agent none
stages {
//------------------------------------------------------------------------------------------------------------------
//---Build & Test stages--------------------------------------------------------------------------------------------
//------------------------------------------------------------------------------------------------------------------
stage('Prepa'){
steps {
init()
}
}
stage('InDocker') {
when {
beforeAgent true
expression { params.LABEL =~ 'centos|ubuntu' && !(params.LABEL =~ 'physical')}
}
agent {
docker {
image "gitlab-registry.cern.ch/sft/docker/$LABEL"
label "$DOCKER_LABEL"
args """-v /cvmfs:/cvmfs
-v /ccache:/ccache
-v /ec:/ec
-e SHELL -e gitlabMergedByUser -e gitlabMergeRequestIid
--net=host
--hostname ${LABEL}-docker
"""
}
}
stages {
stage('Build&Test') {
steps {
buildAndTest()
}
post {
success {
deleteDir()
}
}
}
}
}
stage('InBareMetal') {
when {
beforeAgent true
expression { params.LABEL =~ 'cuda|physical' }
}
agent {
label "$LABEL"
}
stages {
stage('Build&Test') {
steps {
buildAndTest()
}
post {
success {
deleteDir()
}
}
}
}
}
}
}
def init() {
currentBuild.displayName = "#${BUILD_NUMBER}" + ' ' + params.OPTION + '-' + params.BACKEND + '-' + params.LABEL + '-' + params.COMPILER + '-' + params.BUILDTYPE
}
def buildAndTest() {
sh label: 'build_and_test', script: """
source /cvmfs/sft.cern.ch/lcg/views/${EXTERNALS}/x86_64-centos7-${COMPILER}-opt/setup.sh
env | sort | sed 's/:/:? /g' | tr '?' '\n'
ctest -VV -S vecgeom/jenkins/vecgeom-ctest.cmake,$MODE
"""
}
\ No newline at end of file
####################################################################
################################################################################
# Before run should be exported next variables:
# $CTEST_BUILD_OPTIONS // CMake flags for VecGeom build
# $CMAKE_SOURCE_DIR // CMake source directory
# $CMAKE_BINARY_DIR // CMake binary directory
# $CMAKE_BUILD_TYPE // CMake build type: Debug, Release
# $CMAKE_INSTALL_PREFIX // Installation prefix for CMake (Jenkins trigger)
# CC and CXX (In Jenkins this step has been done authomaticly)
# Enviroment for name of build for CERN CDash:
# $LABEL // Name of node (Jenkins trigger)
# Name of $BACKEND // Backend for VecGeom (CUDA/Vc/Scalar/..)
# CMAKE_SOURCE_DIR // CMake source directory
# CMAKE_BINARY_DIR // CMake binary directory
# BUILDTYPE // CMake build type: Debug, Release
# CMAKE_INSTALL_PREFIX // Installation prefix for CMake (Jenkins trigger)
# LABEL // Name of node (Jenkins trigger)
# BACKEND // Backend for VecGeom (CUDA/Vc/Scalar/..)
#
# Optional variables:
# CTEST_SITE // Name of the host computer
# ExtraCMakeOptions // Addional options
cmake_minimum_required(VERSION 2.8)
set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS "1000")
set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "1000")
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "50000")
###################################################################
macro(CheckExitCode)
if(NOT ${ExitCode} EQUAL 0)
return(${ExitCode})
endif()
endmacro(CheckExitCode)
####################################################################
################################################################################
# Build name settings
find_program(UNAME NAMES uname)
macro(getuname name flag)
exec_program("${UNAME}" ARGS "${flag}" OUTPUT_VARIABLE "${name}")
endmacro(getuname)
getuname(osname -s)
getuname(osrel -r)
getuname(cpu -m)
exec_program(${UNAME} ARGS -m OUTPUT_VARIABLE arch)
if (DEFINED ENV{BACKEND})
set(CTEST_BUILD_NAME "$ENV{OPTION}-${cpu}+$ENV{BACKEND}-$ENV{LABEL}-$ENV{COMPILER}-$ENV{CMAKE_BUILD_TYPE}")
set(CTEST_BUILD_NAME "$ENV{OPTION}-${arch}+$ENV{BACKEND}-$ENV{LABEL}-$ENV{COMPILER}-$ENV{BUILDTYPE}")
else()
set(CTEST_BUILD_NAME "$ENV{OPTION}-${cpu}-$ENV{LABEL}-$ENV{COMPILER}-$ENV{CMAKE_BUILD_TYPE}")
set(CTEST_BUILD_NAME "$ENV{OPTION}-${arch}-$ENV{LABEL}-$ENV{COMPILER}-$ENV{BUILDTYPE}")
endif()
if(DEFINED ENV{gitlabMergedByUser} AND DEFINED ENV{gitlabMergeRequestIid})
set(CTEST_BUILD_NAME "$ENV{gitlabMergedByUser}#$ENV{gitlabMergeRequestIid}-${CTEST_BUILD_NAME}")
endif()
message("CTEST name: ${CTEST_BUILD_NAME}")
find_program(HOSTNAME_CMD NAMES hostname)
exec_program(${HOSTNAME_CMD} ARGS OUTPUT_VARIABLE HOSTNAME)
IF(NOT DEFINED CTEST_SITE)
SET(CTEST_SITE "${HOSTNAME}")
ENDIF(NOT DEFINED CTEST_SITE)
if(DEFINED ENV{CTEST_SITE})
set(CTEST_SITE $ENV{CTEST_SITE})
elseif(DEFINED ENV{container} AND DEFINED ENV{NODE_NAME})
set(CTEST_SITE "$ENV{NODE_NAME}-$ENV{container}")
else()
find_program(HOSTNAME_CMD NAMES hostname)
exec_program(${HOSTNAME_CMD} ARGS OUTPUT_VARIABLE CTEST_SITE)
endif()
#######################################################
################################################################################
# Build dashboard model setup
if("$ENV{MODE}" STREQUAL "")
set(CTEST_MODE Experimental)
else()
......@@ -68,42 +55,38 @@ else()
SET(MODEL Experimental)
endif()
find_program(CTEST_COMMAND_BIN NAMES ctest)
SET (CTEST_COMMAND
"$CTEST_COMMAND_BIN -D ${MODEL}")
#######################################################
set(WITH_COVERAGE FALSE)
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "5000")
set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE "5000")
#######################################################
#set(CTEST_USE_LAUNCHERS 1)
#if(NOT "${CTEST_CMAKE_GENERATOR}" MATCHES "Make")
# set(CTEST_USE_LAUNCHERS 0)
#endif()
#set(ENV{CTEST_USE_LAUNCHERS_DEFAULT} ${CTEST_USE_LAUNCHERS})
################################################################################
# Use multiple CPU cores to build
include(ProcessorCount)
ProcessorCount(N)
if(NOT N EQUAL 0)
set(ctest_test_args ${ctest_test_args} PARALLEL_LEVEL ${N})
endif()
################################################################################
# CTest/CMake settings
set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS "1000")
set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "1000")
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE "50000")
set(CTEST_TEST_TIMEOUT 900)
set(CTEST_BUILD_CONFIGURATION "$ENV{CMAKE_BUILD_TYPE}")
set(CTEST_BUILD_CONFIGURATION "$ENV{BUILDTYPE}")
set(CMAKE_INSTALL_PREFIX "$ENV{CMAKE_INSTALL_PREFIX}")
set(CTEST_SOURCE_DIRECTORY "$ENV{CMAKE_SOURCE_DIR}")
set(CTEST_BINARY_DIRECTORY "$ENV{CMAKE_BINARY_DIR}")
set(CTEST_INSTALL_PREFIX "$ENV{CMAKE_INSTALL_PREFIX}")
set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
set(CTEST_BUILD_OPTIONS "$ENV{CTEST_BUILD_OPTIONS}")
# Fix set of CMake options
set(CTEST_BUILD_FLAGS "-j${N}")
################################################################################
# Fixed set of CMake options
set(config_options -DCMAKE_INSTALL_PREFIX=${CTEST_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION}
-DBUILDTYPE=${CTEST_BUILD_CONFIGURATION}
-DCTEST=ON
-DBENCHMARK=ON
-DROOT=ON
-DCUDA_VOLUME_SPECIALIZATION=OFF
$ENV{ExtraCMakeOptions})
################################################################################
# Options depending on compiler/label/etc.
if("$ENV{LABEL}" MATCHES cuda)
list(APPEND config_options -DCUDA=ON)
......@@ -112,7 +95,7 @@ if("$ENV{LABEL}" MATCHES cuda)
endif()
if (DEFINED ENV{BACKEND})
list(APPEND config_options -DBACKEND=$ENV{BACKEND})
list(APPEND config_options -DBACKEND=$ENV{BACKEND})
endif()
list(APPEND config_options -DCMAKE_CXX_STANDARD=17)
......@@ -126,12 +109,9 @@ elseif("$ENV{OPTION}" STREQUAL "AVX")
elseif("$ENV{OPTION}" STREQUAL "SSE3")
list(APPEND config_options -DVECGEOM_VECTOR=sse3)
endif()
ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY})
#########################################################
################################################################################
# git command configuration
find_program(CTEST_GIT_COMMAND NAMES git)
if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}")
set(CTEST_CHECKOUT_COMMAND "${CTEST_GIT_COMMAND} clone https://gitlab.cern.ch/VecGeom/VecGeom.git ${CTEST_SOURCE_DIRECTORY}")
......@@ -140,35 +120,22 @@ endif()
set(CTEST_GIT_UPDATE_COMMAND "${CTEST_GIT_COMMAND}")
if(${MODEL} MATCHES Nightly OR Experimental)
if(NOT "$ENV{GIT_COMMIT}" STREQUAL "")
set(CTEST_CHECKOUT_COMMAND "cmake -E chdir ${CTEST_SOURCE_DIRECTORY} ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_PREVIOUS_COMMIT}")
set(CTEST_GIT_UPDATE_CUSTOM ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_COMMIT})
endif()
if(NOT "$ENV{GIT_COMMIT}" STREQUAL "")
set(CTEST_CHECKOUT_COMMAND "cmake -E chdir ${CTEST_SOURCE_DIRECTORY} ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_PREVIOUS_COMMIT}")
set(CTEST_GIT_UPDATE_CUSTOM ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_COMMIT})
endif()
else()
if(NOT "$ENV{GIT_COMMIT}" STREQUAL "")
set(CTEST_CHECKOUT_COMMAND "cmake -E chdir ${CTEST_SOURCE_DIRECTORY} ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_COMMIT}")
set(CTEST_GIT_UPDATE_CUSTOM ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_COMMIT})
endif()
if(NOT "$ENV{GIT_COMMIT}" STREQUAL "")
set(CTEST_CHECKOUT_COMMAND "cmake -E chdir ${CTEST_SOURCE_DIRECTORY} ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_COMMIT}")
set(CTEST_GIT_UPDATE_CUSTOM ${CTEST_GIT_COMMAND} checkout -f $ENV{GIT_COMMIT})
endif()
endif()
#########################################################
################################################################################
## Output language
set($ENV{LC_MESSAGES} "en_EN")
#########################################################
# Use multiple CPU cores to build
include(ProcessorCount)
ProcessorCount(N)
if(NOT N EQUAL 0)
if(NOT WIN32)
# reduce the number if parallel compilations (SPEC mode takes lots of memory)
math(EXPR M ${N}/2)
set(CTEST_BUILD_FLAGS -j${M})
endif(NOT WIN32)
set(ctest_test_args ${ctest_test_args} PARALLEL_LEVEL ${N})
endif()
##########################################################
################################################################################
# Print summary information.
foreach(v
CTEST_SITE
......@@ -181,21 +148,21 @@ foreach(v
CTEST_CONFIGURE_COMMAND
CTEST_SCRIPT_DIRECTORY
CTEST_BUILD_FLAGS
WITH_MEMCHECK
WITH_COVERAGE
)
set(vars "${vars} ${v}=[${${v}}]\n")
endforeach(v)
message("Dashboard script configuration (check if everything is declared correctly):\n${vars}\n")
#######################################################
################################################################################
# Test custom update with a dashboard script.
message("Running CTest Dashboard Script (custom update)...")
include("${CTEST_SOURCE_DIRECTORY}/CTestConfig.cmake")
# ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY})
file(REMOVE_RECURSE ${CTEST_BINARY_DIRECTORY})
ctest_start(${MODEL} TRACK ${MODEL})
ctest_update(SOURCE ${CTEST_SOURCE_DIRECTORY})
message("Updated.")
ctest_configure(BUILD ${CTEST_BINARY_DIRECTORY}
SOURCE ${CTEST_SOURCE_DIRECTORY}
......@@ -208,7 +175,9 @@ ctest_build(BUILD ${CTEST_BINARY_DIRECTORY}
APPEND)
ctest_submit(PARTS Build)
ctest_test(BUILD ${CTEST_BINARY_DIRECTORY} APPEND)
ctest_test(BUILD ${CTEST_BINARY_DIRECTORY}
${ctest_test_args}
APPEND)
ctest_submit(PARTS Test)
if(${MODEL} MATCHES NightlyMemoryCheck)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment