Commit ef81aa8b authored by cpomies's avatar cpomies
Browse files

Added files to manage rsync

parent 92e379ba
......@@ -18,6 +18,11 @@ export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \
set_config=1
. $(dirname $0)/common.sh
. lbn-rsync-get-config
lbn-rsync-get-sources
exit 0
day=$(date +%a)
timestamp=$(date -I)
deploybase=$(dirname /data/${ARTIFACTS_DIR})
......
......@@ -64,5 +64,9 @@ if [ "${flavour}" = "release" ] ; then
fi
fi
lbn-rsync-push-artifact
lbn-check-preconditions --verbose ${config_file}
# Cleaning up
rm -rf tmp
......@@ -31,6 +31,9 @@ export LHCB_NIGHTLY_MAX_THREADS=1
export ARTIFACTS_DIR=${ARTIFACTS_DIR:-artifacts/${flavour}/${slot}/${slot_build_id}}
mkdir -p ${ARTIFACTS_DIR}
export RSYNC_SERVER=${RSYNC_SERVER:-*********} # TODO
export RSYNC_WORKDIR=${RSYNC_WORKDIR:-*********} # TODO
export RSYNC_DIR=${RSYNC_DIR:-${RSYNC_SERVER}:${RSYNC_WORKDIR}/${flavour}/${slot}/${slot_build_id}}
export TMPDIR=${WORKSPACE}/tmp
mkdir -p ${TMPDIR}
......@@ -78,12 +81,6 @@ if [ $(python -c 'import sys; print "%d%d" % sys.version_info[:2]') = 24 ] ; the
. SetupProject.sh LCGCMT 66 Python
fi
if [ -e ${ARTIFACTS_DIR}/${slot}.json ] ; then
config_file=${ARTIFACTS_DIR}/${slot}.json
else
config_file=${ARTIFACTS_DIR}/configuration.xml#${slot}
fi
if klist -5 > /dev/null 2>&1 ; then
kinit -R
klist -5
......
......@@ -46,6 +46,9 @@ export JOB_NAME=${JOB_NAME:-nightly-test-slot-build-platform}
guessed_label=${platform#*-}
guessed_label=${guessed_label%%-*}
export os_label=${os_label:-${guessed_label}}
export JENKINS_HOME=${JENKINS_HOME:-jenkins_home}
export RSYNC_SERVER=${RSYNC_SERVER:-${USERNAME}@${HOSTNAME}}
export RSYNC_WORKDIR=${RSYNC_WORKDIR:-$(cd $(dirname $0)/.. ;pwd)/rsync}
# this variable might be used inside the Jenkins scripts to avoid some ops
export JENKINS_MOCK=true
......
......@@ -16,6 +16,6 @@ export HOME=$PWD
# Set common environment
. $(dirname $0)/common.sh
export CMTCONFIG=$platform
. lbn-rsync-get-config
lbn-preconditions --verbose ${config_file}
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
'''
Contains scripts fo push or get artifact with rsync
'''
__author__ = 'Colas Pomies <colas.pomies@cern.ch>'
import os
import LbUtils.Script
from socket import gethostname
from LbNightlyTools.Utils import timeout_call as call, ensureDirs
def get_cmd_rsync(src, dest, includes = [], excludes = []):
cmd = ['rsync', '-P', '--archive', '--whole-file',
'--partial-dir=.rsync-partial.%s.%d' %
(gethostname(), os.getpid()),
'--delay-updates', '--rsh=ssh']
if includes:
for include in includes:
cmd.append('--include=%s' % include)
if excludes:
for exclude in excludes:
cmd.append('--exclude=%s' % exclude)
cmd.append(src)
cmd.append(dest)
return cmd
class Script(LbUtils.Script.PlainScript):
__usage__ = '%prog [options]'
__version__ = ''
def defineOpts(self):
""" User options -- has to be overridden """
from LbNightlyTools.ScriptsCommon import (addBasicOptions,
addDashboardOptions)
addBasicOptions(self.parser)
addDashboardOptions(self.parser)
self.parser.add_option('--get-config',
action='store_true',
dest='get_config',
help='Synchronize configs files')
self.parser.add_option('--get-sources',
action='store_true',
dest='get_sources',
help='Synchronize sources files')
self.parser.add_option('-d', '--destination',
action='store',
help='Destination folder')
self.parser.add_option('-s', '--source',
action='store',
help='Source folder')
self.parser.set_defaults(get_config=False,
get_sources=False,
source=None,
destination=None)
def main(self):
if len(self.args) != 0:
self.parser.error('wrong number of arguments')
opts = self.options
if not opts.source:
self.log.error("Source folder needs to be defined with -s or --source")
return 1
if not opts.destination:
self.log.error("Destination folder needs to be defined with -d or --destination")
return 1
includes_param = []
excludes_param = []
if opts.get_config:
includes_param.append("*.json")
includes_param.append("*.xml")
excludes_param = ["*"]
if opts.get_sources:
includes_param.append("*.src.*")
includes_param.append("checkout_job_url.txt")
excludes_param = ["*"]
if ':' in opts.destination:
host, path = opts.destination.split(':', 1)
call(['ssh', host, 'mkdir -pv "%s"' % path])
else:
ensureDirs([opts.destination])
cmd = get_cmd_rsync(
opts.source + '/',
opts.destination,
includes_param,
excludes_param)
self.log.info("Rsync call : " + " ".join(cmd))
return call(cmd)
#!/usr/bin/env python
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
import LbUtils.Log
LbUtils.Log._default_log_format = '%(asctime)s:' + LbUtils.Log._default_log_format
from LbNightlyTools.RsyncArtifact import Script
import sys
sys.exit(Script().run())
\ No newline at end of file
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
lbn-manage-rsync --verbose --get-config --source "${RSYNC_DIR}" --destination "${ARTIFACTS_DIR}"
if [ -e ${ARTIFACTS_DIR}/${slot}.json ] ; then
export config_file=${ARTIFACTS_DIR}/${slot}.json
else
export config_file=${ARTIFACTS_DIR}/configuration.xml#${slot}
fi
\ No newline at end of file
......@@ -10,12 +10,4 @@
# or submit itself to any jurisdiction. #
###############################################################################
# hack because of a bug with non-writable home (this script is run by tomcat)
export HOME=$PWD
# Set common environment
. $(dirname $0)/common.sh
export CMTCONFIG=$platform
lbn-check-preconditions --verbose ${config_file}
lbn-manage-rsync --verbose --get-sources --source "${RSYNC_DIR}" --destination "${ARTIFACTS_DIR}"
\ No newline at end of file
#!/bin/bash
###############################################################################
# (c) Copyright 2013 CERN #
# #
# This software is distributed under the terms of the GNU General Public #
# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". #
# #
# In applying this licence, CERN does not waive the privileges and immunities #
# granted to it by virtue of its status as an Intergovernmental Organization #
# or submit itself to any jurisdiction. #
###############################################################################
lbn-manage-rsync --verbose --source "${ARTIFACTS_DIR}" --destination "${RSYNC_DIR}"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment