Added files to manage rsync

jenkins/build.sh

@@ -18,6 +18,11 @@ export LD_LIBRARY_PATH=$(echo $LD_LIBRARY_PATH | tr : \\n | grep -v /gcc/ | tr \
 set_config=1
 . $(dirname $0)/common.sh
 
+. lbn-rsync-get-config
+lbn-rsync-get-sources
+
+exit 0
+
 day=$(date +%a)
 timestamp=$(date -I)
 deploybase=$(dirname /data/${ARTIFACTS_DIR})

jenkins/checkout.sh

@@ -64,5 +64,9 @@ if [ "${flavour}" = "release" ] ; then
   fi
 fi
 
+lbn-rsync-push-artifact
+
+lbn-check-preconditions --verbose ${config_file}
+
 # Cleaning up
 rm -rf tmp

jenkins/common.sh

@@ -31,6 +31,9 @@ export LHCB_NIGHTLY_MAX_THREADS=1
 
 export ARTIFACTS_DIR=${ARTIFACTS_DIR:-artifacts/${flavour}/${slot}/${slot_build_id}}
 mkdir -p ${ARTIFACTS_DIR}
+export RSYNC_SERVER=${RSYNC_SERVER:-*********} # TODO
+export RSYNC_WORKDIR=${RSYNC_WORKDIR:-*********} # TODO
+export RSYNC_DIR=${RSYNC_DIR:-${RSYNC_SERVER}:${RSYNC_WORKDIR}/${flavour}/${slot}/${slot_build_id}}
 export TMPDIR=${WORKSPACE}/tmp
 mkdir -p ${TMPDIR}
 
@@ -78,12 +81,6 @@ if [ $(python -c 'import sys; print "%d%d" % sys.version_info[:2]') = 24 ] ; the
   . SetupProject.sh LCGCMT 66 Python
 fi
 
-if [ -e ${ARTIFACTS_DIR}/${slot}.json ] ; then
-  config_file=${ARTIFACTS_DIR}/${slot}.json
-else
-  config_file=${ARTIFACTS_DIR}/configuration.xml#${slot}
-fi
-
 if klist -5 > /dev/null 2>&1 ; then
   kinit -R
   klist -5

jenkins/mock.sh

@@ -46,6 +46,9 @@ export JOB_NAME=${JOB_NAME:-nightly-test-slot-build-platform}
 guessed_label=${platform#*-}
 guessed_label=${guessed_label%%-*}
 export os_label=${os_label:-${guessed_label}}
+export JENKINS_HOME=${JENKINS_HOME:-jenkins_home}
+export RSYNC_SERVER=${RSYNC_SERVER:-${USERNAME}@${HOSTNAME}}
+export RSYNC_WORKDIR=${RSYNC_WORKDIR:-$(cd $(dirname $0)/.. ;pwd)/rsync}
 # this variable might be used inside the Jenkins scripts to avoid some ops
 export JENKINS_MOCK=true
 

jenkins/preconditions.sh

@@ -16,6 +16,6 @@ export HOME=$PWD
 # Set common environment
 . $(dirname $0)/common.sh
 
-export CMTCONFIG=$platform
+. lbn-rsync-get-config
 
 lbn-preconditions --verbose ${config_file}

python/LbNightlyTools/RsyncArtifact.py

+###############################################################################
+# (c) Copyright 2013 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+'''
+Contains scripts fo push or get artifact with rsync
+'''
+__author__ = 'Colas Pomies <colas.pomies@cern.ch>'
+
+import os
+import LbUtils.Script
+
+from socket import gethostname
+from LbNightlyTools.Utils import timeout_call as call, ensureDirs
+
+def get_cmd_rsync(src, dest, includes = [], excludes = []):
+
+    cmd = ['rsync', '-P', '--archive', '--whole-file',
+           '--partial-dir=.rsync-partial.%s.%d' %
+           (gethostname(), os.getpid()),
+           '--delay-updates', '--rsh=ssh']
+
+    if includes:
+        for include in includes:
+            cmd.append('--include=%s' % include)
+
+    if excludes:
+        for exclude in excludes:
+            cmd.append('--exclude=%s' % exclude)
+
+    cmd.append(src)
+    cmd.append(dest)
+
+    return cmd
+
+class Script(LbUtils.Script.PlainScript):
+
+    __usage__ = '%prog [options]'
+    __version__ = ''
+
+    def defineOpts(self):
+        """ User options -- has to be overridden """
+        from LbNightlyTools.ScriptsCommon import (addBasicOptions,
+                                                  addDashboardOptions)
+        addBasicOptions(self.parser)
+        addDashboardOptions(self.parser)
+
+        self.parser.add_option('--get-config',
+                               action='store_true',
+                               dest='get_config',
+                               help='Synchronize configs files')
+
+        self.parser.add_option('--get-sources',
+                               action='store_true',
+                               dest='get_sources',
+                               help='Synchronize sources files')
+
+        self.parser.add_option('-d', '--destination',
+                               action='store',
+                               help='Destination folder')
+
+        self.parser.add_option('-s', '--source',
+                               action='store',
+                               help='Source folder')
+
+        self.parser.set_defaults(get_config=False,
+                                 get_sources=False,
+                                 source=None,
+                                 destination=None)
+
+    def main(self):
+
+        if len(self.args) != 0:
+            self.parser.error('wrong number of arguments')
+
+        opts = self.options
+
+        if not opts.source:
+            self.log.error("Source folder needs to be defined with -s or --source")
+            return 1
+
+        if not opts.destination:
+            self.log.error("Destination folder needs to be defined with -d or --destination")
+            return 1
+
+        includes_param = []
+        excludes_param = []
+
+        if opts.get_config:
+            includes_param.append("*.json")
+            includes_param.append("*.xml")
+            excludes_param = ["*"]
+        if opts.get_sources:
+            includes_param.append("*.src.*")
+            includes_param.append("checkout_job_url.txt")
+            excludes_param = ["*"]
+
+        if ':' in opts.destination:
+            host, path = opts.destination.split(':', 1)
+            call(['ssh', host, 'mkdir -pv "%s"' % path])
+        else:
+            ensureDirs([opts.destination])
+
+        cmd = get_cmd_rsync(
+            opts.source + '/',
+            opts.destination,
+            includes_param,
+            excludes_param)
+
+        self.log.info("Rsync call : " + " ".join(cmd))
+        return call(cmd)

scripts/lbn-manage-rsync

+#!/usr/bin/env python
+###############################################################################
+# (c) Copyright 2013 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+import LbUtils.Log
+LbUtils.Log._default_log_format = '%(asctime)s:' + LbUtils.Log._default_log_format
+
+from LbNightlyTools.RsyncArtifact import Script
+import sys
+sys.exit(Script().run())
\ No newline at end of file

scripts/lbn-rsync-get-config

+#!/bin/bash
+###############################################################################
+# (c) Copyright 2013 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+
+lbn-manage-rsync --verbose --get-config --source "${RSYNC_DIR}" --destination "${ARTIFACTS_DIR}"
+
+if [ -e ${ARTIFACTS_DIR}/${slot}.json ] ; then
+	export config_file=${ARTIFACTS_DIR}/${slot}.json
+else
+	export config_file=${ARTIFACTS_DIR}/configuration.xml#${slot}
+fi
\ No newline at end of file

jenkins/check_preconditions.sh → scripts/lbn-rsync-get-sources

@@ -10,12 +10,4 @@
 # or submit itself to any jurisdiction.                                       #
 ###############################################################################
 
-# hack because of a bug with non-writable home (this script is run by tomcat)
-export HOME=$PWD
-
-# Set common environment
-. $(dirname $0)/common.sh
-
-export CMTCONFIG=$platform
-
-lbn-check-preconditions --verbose ${config_file}
+lbn-manage-rsync --verbose --get-sources --source "${RSYNC_DIR}" --destination "${ARTIFACTS_DIR}"
\ No newline at end of file

scripts/lbn-rsync-push-artifact

+#!/bin/bash
+###############################################################################
+# (c) Copyright 2013 CERN                                                     #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+
+lbn-manage-rsync --verbose --source "${ARTIFACTS_DIR}" --destination "${RSYNC_DIR}"