diff --git a/jenkins/docker.sh b/jenkins/docker.sh new file mode 100755 index 0000000000000000000000000000000000000000..fbfcf3f65177c8a222fbdd1787eb1dc3089c056d --- /dev/null +++ b/jenkins/docker.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +if [ -z $main_dir ] +then + scripts_dir=$(cd $(dirname $0) ; pwd) + main_dir=$(cd ${scripts_dir}/.. ; pwd) +fi + +if [ -z $KRB5CCNAME ] +then + KRB5CCNAME="FILE:/tmp/$UID" +fi + +if [ -e ~/private/couchdb-admin ] ; then + export COUCHDB_ADMIN=$(cat ~/private/couchdb-admin) +fi + +if [[ "$os_label" = *docker* ]] +then + export os_label=$(echo $platform | cut -d: -f2 | cut -d- -f2) + export platform=$(echo $platform | cut -d: -f1) +fi + +cmd="${main_dir}/jenkins/lb-docker-run \ + -v /afs \ + -e COUCHDB_ADMIN \ + -e slot \ + -e platform \ + -e project \ + -e flavour \ + -e NODE_NAME \ + -e slot_build_id \ + -e WORKSPACE=/workspace \ + -e JOB_NAME \ + -e BUILD_NUMBER \ + -e BUILD_URL \ + -e os_label \ + -e JENKINS_HOME \ + -e RSYNC_SERVER \ + -e RSYNC_WORKDIR \ + -e JENKINS_MOCK \ + -e projects_list \ + -e USER \ + --privileged \ + --kerberos \ + --docker-hostname \ + --no-interactive" + +for mockCopy in python scripts jenkins setup.sh +do + if [ ! -e ${WORKSPACE}/$mockCopy ] + then + cmd="$cmd -v ${main_dir}/$mockCopy:/workspace/$mockCopy" + fi +done + +if [[ $(hostname -f) == *.cern.ch ]] ; then + dns_opt="--dns 137.138.17.5" +fi + +$cmd ${dns_opt} --$os_label "/bin/lb-su" $1 diff --git a/jenkins/lb-docker-run b/jenkins/lb-docker-run new file mode 100755 index 0000000000000000000000000000000000000000..6c7a32e15039b769a5f59becee46663081deb154 --- /dev/null +++ b/jenkins/lb-docker-run @@ -0,0 +1,114 @@ +#!/usr/bin/env python +import os +import sys +import pwd +import grp +import optparse +import socket +from subprocess import Popen + +cwd = os.getcwd() +cwd_stat = os.stat(cwd) +userName = pwd.getpwuid(cwd_stat.st_uid).pw_name +grpName = grp.getgrgid(cwd_stat.st_gid).gr_name + +parser = optparse.OptionParser() +parser.add_option('--slc6', action='store_const', const='slc6', dest='os', help="Run an slc6 container, default one") +parser.add_option('--slc5', action='store_const', const='slc5', dest='os', help="Run an slc5 container") +parser.add_option('--centos7', action='store_const', const='centos7', dest='os', help="Run an centos7 container") +parser.add_option('--os', action='store', help="OS you want to run", + choices=['slc5', 'slc6', 'centos7']) +parser.add_option('--no-LbLogin', action='store_const', const='no', dest='LbLogin', help="Do not call LbLogin on start") +parser.add_option('--no-lb-su', action='store_const', const='no', dest='lb_su', help="Do not switch user") +parser.add_option('--home', action='store_true', dest='home', help="Set a persistent home for this image") +parser.add_option('-c', action='store', dest='cmd', help="Run CMD when container is started") +parser.add_option('-e', '--env', action='append', dest='envVar', help="Set environment variables") +parser.add_option('-v', '--volume', action='append', dest='volume', help="Bind mount a volume") +parser.add_option('-p', '--publish', action='append', dest='port', help="Publish a container's port(s) to the host") +parser.add_option('-u', '--user', action='store', dest='user', help="Username or UID (format: <name|uid>[:<group|gid>])") +parser.add_option('-w', '--workdir', action='store', dest='workdir', help="Working directory inside the container") +parser.add_option('--dns', action='store', dest='dns', help="Set custom DNS servers") +parser.add_option('--privileged', action='store_true', dest='privileged', help="Give extended privileges to this container") +parser.add_option('--no-interactive', action='store_false', dest='interactive', help="Do not give an interactive shell to this container") +parser.add_option('--ssh-agent', action='store_true', dest='sshAgent', help="Forward host ssh-agent to the container") +parser.add_option('--docker-hostname', action='store_true', dest='hostname', help="Pass to the container the host's hostname") +parser.add_option('--kerberos', action='store_true', dest='kerberos', help="Forward the the kerberos ticket to the container") + +parser.set_defaults(os='slc6', + LbLogin='yes', + lb_su='yes', + home=False, + privileged=False, + interactive=True, + sshAgent=False, + hostname=False, + kerberos=False, + envVar=[], + port=[], + volume=[cwd + ':/workspace', '/cvmfs/lhcb.cern.ch:/cvmfs/lhcb.cern.ch', '/cvmfs/lhcbdev.cern.ch:/cvmfs/lhcbdev.cern.ch']) + +options, args = parser.parse_args() + +if options.cmd and args: + parser.error('you cannot -c in conjunction with arguments') + +if not args: + args.append('/bin/bash') + +if options.home == True: + homedir = os.getenv('HOME') + '/.local/docker/' + options.os + '-build-home/' + if not os.path.exists(homedir): + os.makedirs(homedir) + options.volume.extend([homedir + ':/home/' + userName]) + +cmd = ['docker', 'run', '--rm', + '-e', 'dockerLb=' + options.LbLogin + ':' + options.lb_su] + +if options.interactive == True: + cmd.extend(['-it']) + +if options.privileged == True: + cmd.extend(['--privileged']) + +if options.sshAgent == True: + options.volume.extend([os.getenv('SSH_AUTH_SOCK') + ':/ssh-agent']) + options.envVar.extend(['SSH_AUTH_SOCK=/ssh-agent']) + +if options.hostname == True: + options.envVar.extend(['docker_hostname=' + socket.gethostname()]) + +if options.kerberos == True: + options.envVar.extend(['KRB5CCNAME']) + options.volume.extend([os.getenv('KRB5CCNAME').split(':')[1]]) + +if options.user: + cmd.extend(['-u', options.user]) +else: + options.envVar.extend(['WORK_USER=' + userName + ':' + grpName]) + +for env in options.envVar: + cmd.extend(['-e', env]) + +for vol in options.volume: + if vol.find(':') >= 0: + cmd.extend(['-v', vol]) + else: + cmd.extend(['-v', vol + ':' + vol]) + +for port in options.port: + cmd.extend(['-p', port]) + +if options.workdir: + cmd.extend(['-w', options.workdir]) + +if options.dns: + cmd.extend(['--dns', options.dns]) + +cmd.extend(['lhcbdev/{0}-build:latest'.format(options.os)]) +cmd.extend(args) + +if options.cmd: + cmd.extend(['-c', options.cmd]) + +os.system('docker pull lhcbdev/{0}-build:latest'.format(options.os)) +sys.exit(Popen(cmd).wait()) diff --git a/jenkins/mock.sh b/jenkins/mock.sh index d736501b703ef3d99b24037f0ed7a380ae96e872..f3a2b647c2db7ca26003f44aad4d27c8b2444e12 100755 --- a/jenkins/mock.sh +++ b/jenkins/mock.sh @@ -19,7 +19,6 @@ # <slot> is a slot name # <platform> is the platform id string - # Check arguments if [ $# -lt 3 ] ; then @@ -39,8 +38,8 @@ platform=$3 project=$4 flavour=${flavour:-mock} -scripts_dir=$(cd $(dirname $0) ; pwd) -main_dir=$(cd ${scripts_dir}/.. ; pwd) +export scripts_dir=$(cd $(dirname $0) ; pwd) +export main_dir=$(cd ${scripts_dir}/.. ; pwd) # Prepare Jenkins-like environment export slot @@ -68,9 +67,12 @@ if [ ! -e $command ] ; then fi if [ ! -d $WORKSPACE ] ; then - mkdir -p $WORKSPACE - echo ". ${main_dir}/setup.sh" > $WORKSPACE/setup.sh + mkdir -p $WORKSPACE fi cd $WORKSPACE -exec $command + +cmd=/workspace/jenkins/${step}.sh +exec ${scripts_dir}/docker.sh $cmd + +#EOF diff --git a/jenkins/utils.d/set_common.sh b/jenkins/utils.d/set_common.sh index 4cd362cd644ab55dd9e77f6a6bec1ebe49d2eaa2..f1bbe07b80ccb6aed785c4fe0b739a173487af01 100644 --- a/jenkins/utils.d/set_common.sh +++ b/jenkins/utils.d/set_common.sh @@ -53,6 +53,18 @@ set_common [--build] [--test]" export HOME=$PWD fi + if [[ "${special_config}" == "true" && $(pwd) != "/workspace" && "${NODE_LABELS}" == *docker* ]] ; then + exec $(dirname $0)/docker.sh "$0" + fi + + if [ -n "${COUCHDB_ADMIN}" ] ; then + if [ ! -e ~/private/couchdb-admin ] ; then + mkdir -p ~/private + echo "${COUCHDB_ADMIN}" > ~/private/couchdb-admin + fi + unset COUCHDB_ADMIN + fi + export CMTCONFIG=${platform} # default (backward-compatible) build flavour if [ "${flavour}" == "" ] ; then diff --git a/python/LbNightlyTools/CheckSlotPreconditions.py b/python/LbNightlyTools/CheckSlotPreconditions.py index e2e616faa0c469e883197dfb45e1d0de39143903..0f69b20e5fb0beb609d1ef46404f6ab369272307 100644 --- a/python/LbNightlyTools/CheckSlotPreconditions.py +++ b/python/LbNightlyTools/CheckSlotPreconditions.py @@ -67,7 +67,10 @@ class Script(LbUtils.Script.PlainScript): label = '-build' for platform in platforms: - os_label = platform.split('-')[1] + label + if os.environ.get('os_label') == 'docker': + os_label = 'docker' + label + else: + os_label = platform.split('-')[1] + label output_file_name = output_file.format(slot.name, platform) open(output_file_name, 'w') \ .write(str(JobParams(slot=slot.name, diff --git a/python/LbNightlyTools/Scripts/Build.py b/python/LbNightlyTools/Scripts/Build.py index 4e78c1eb8498c19a0a6f3c9885ca8d1d5193b323..e4c4c6e3f9b67c6cec6542e09c645a9fbad89a3f 100644 --- a/python/LbNightlyTools/Scripts/Build.py +++ b/python/LbNightlyTools/Scripts/Build.py @@ -53,6 +53,8 @@ __log__ = logging.getLogger(__name__) LOAD_AVERAGE_SCALE = 1.2 GB_PER_BUILD_JOB = 1.5 +HOSTNAME = os.environ.get('docker_hostname', socket.gethostname()) + def listAllFiles(path, excl=None): ''' @@ -366,7 +368,7 @@ string(REPLACE "$${NIGHTLY_BUILD_ROOT}" "$${CMAKE_CURRENT_LIST_DIR}" platform=self.json_tmpl['platform']) self.dump_json({'type': 'job-start', - 'host': socket.gethostname(), + 'host': HOSTNAME, 'build_number': os.environ.get('BUILD_NUMBER', 0), 'started': self.starttime.isoformat()}, update=False) @@ -913,7 +915,7 @@ class BuildReporter(object): version=self.project.version, slot=self.slot.name, slot_build_id=self.slot.build_id, - host=socket.gethostname(), + host=HOSTNAME, platform=self.platform, logfile_links=dumps(logfile_links, indent=2), code_links=dumps(code_links, indent=2), diff --git a/python/LbNightlyTools/Utils.py b/python/LbNightlyTools/Utils.py index d5a50c23d51700fe5a5e252cfdc947316c28e292..a93435270dd655726e07d30d11fc4cd3f990fcf9 100644 --- a/python/LbNightlyTools/Utils.py +++ b/python/LbNightlyTools/Utils.py @@ -409,10 +409,18 @@ class Dashboard(object): self._log = logging.getLogger('Dashboard') if submit and credentials is None: + cred_data = None if os.path.exists(self.CRED_FILE): + self._log.debug('taking credentials from %s', self.CRED_FILE) + cred_data = open(self.CRED_FILE).read() + elif os.environ.get('COUCHDB_ADMIN'): + self._log.debug('taking credentials from COUCHDB_ADMIN') + cred_data = os.environ['COUCHDB_ADMIN'] + + if cred_data: # make a tuple with the first two lines of the file credentials = tuple([l.strip() - for l in open(self.CRED_FILE)][:2]) + for l in cred_data.splitlines()][:2]) else: self._log.debug('no couchdb credentials found') @@ -432,8 +440,8 @@ class Dashboard(object): except (couchdb.ResourceNotFound, couchdb.ServerError, couchdb.Unauthorized, - socket.error): - self._log.warning('failed to access %s%s', server, db) + socket.error), x: + self._log.warning('failed to access %s%s (%s)', server, db, x) # ignore connection failures self.db = None else: