From 245f134f09f71aa5b4ef30dcf92275e2377ac893 Mon Sep 17 00:00:00 2001
From: Atlas Librarian <atlas-software.librarian@cern.ch>
Date: Thu, 12 Oct 2017 17:43:25 +0200
Subject: [PATCH] retry git checkout/fetch in checkout_atlasexternals.sh and
 checkout_Gaudi.sh (ATLINFR-1817)

Former-commit-id: ead4c942efb63be4ffa616d79d473b419c32f7fb
---
 Build/AtlasBuildScripts/checkout_Gaudi.sh     | 22 ++++++++++++++++---
 .../checkout_atlasexternals.sh                | 22 ++++++++++++++++---
 Projects/Athena/build_externals.sh            | 12 ++++++----
 3 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/Build/AtlasBuildScripts/checkout_Gaudi.sh b/Build/AtlasBuildScripts/checkout_Gaudi.sh
index 15851ec1313..80dce55c133 100755
--- a/Build/AtlasBuildScripts/checkout_Gaudi.sh
+++ b/Build/AtlasBuildScripts/checkout_Gaudi.sh
@@ -27,6 +27,22 @@ usage() {
 	echo "  in the source directory is left untouched."
 }
 
+_max_retry_=5
+
+_retry_ () {
+    local cmd="$*"
+    local n=0
+    while ! $cmd ; do
+        if test $n -eq $_max_retry_ ; then
+            echo "ERROR: $cmd FAILED $_max_retry_ times. EXIT(1)" >&2
+            exit 1
+        fi
+        echo "WARNING: $cmd FAILED, retry in 30 sec ... "
+        sleep 30s
+        n=`expr $n + 1`
+    done
+}
+
 # Parse the command line arguments:
 TAGBRANCH=""
 SOURCEDIR=""
@@ -86,12 +102,12 @@ echo "   from: $GAUDIURL"
 
 if [ ! -d "${SOURCEDIR}" ]; then
     # Clone the repository:
-    git clone ${GAUDIURL} ${SOURCEDIR}
+    _retry_ git clone ${GAUDIURL} ${SOURCEDIR}
 else
     echo "${SOURCEDIR} already exists -> assume previous checkout"
 fi
 
 # Get the appropriate version of it:
 cd ${SOURCEDIR}
-git fetch --prune origin
-git checkout -f ${TAGBRANCH}
+_retry_ git fetch --prune origin
+_retry_ git checkout -f ${TAGBRANCH}
diff --git a/Build/AtlasBuildScripts/checkout_atlasexternals.sh b/Build/AtlasBuildScripts/checkout_atlasexternals.sh
index a6a6627ed26..52fd9cd5e4c 100755
--- a/Build/AtlasBuildScripts/checkout_atlasexternals.sh
+++ b/Build/AtlasBuildScripts/checkout_atlasexternals.sh
@@ -27,6 +27,22 @@ usage() {
 	echo "  in the source directory is left untouched."
 }
 
+_max_retry_=5
+
+_retry_ () {
+    local cmd="$*"
+    local n=0
+    while ! $cmd ; do
+        if test $n -eq $_max_retry_ ; then
+            echo "ERROR: $cmd FAILED $_max_retry_ times. EXIT(1)" >&2
+            exit 1
+        fi
+        echo "WARNING: $cmd FAILED, retry in 30 sec ... "
+        sleep 30s
+        n=`expr $n + 1`
+    done
+}
+
 # Parse the command line arguments:
 TAGBRANCH=""
 SOURCEDIR=""
@@ -86,12 +102,12 @@ echo "   from: $EXTERNALSURL"
 
 if [ ! -d "${SOURCEDIR}" ]; then
     # Clone the repository:
-    git clone ${EXTERNALSURL} ${SOURCEDIR}
+    _retry_ git clone ${EXTERNALSURL} ${SOURCEDIR}
 else
     echo "${SOURCEDIR} already exists -> assume previous checkout"
 fi
 
 # Get the appropriate version of it:
 cd ${SOURCEDIR}
-git fetch --prune origin
-git checkout -f ${TAGBRANCH}
+_retry_ git fetch --prune origin
+_retry_ git checkout -f ${TAGBRANCH}
diff --git a/Projects/Athena/build_externals.sh b/Projects/Athena/build_externals.sh
index 46392165be7..540d24b93ba 100755
--- a/Projects/Athena/build_externals.sh
+++ b/Projects/Athena/build_externals.sh
@@ -119,9 +119,11 @@ fi
 AthenaExternalsVersion=$(awk '/^AthenaExternalsVersion/{print $3}' ${thisdir}/externals.txt)
 
 # Check out AthenaExternals from the right branch/tag:
-${scriptsdir}/checkout_atlasexternals.sh \
+error_stamp=`mktemp .tmp.error.XXXXX` ; rm -f $error_stamp
+{ ${scriptsdir}/checkout_atlasexternals.sh \
     -t ${AthenaExternalsVersion} \
-    -s ${BUILDDIR}/src/AthenaExternals 2>&1 | tee ${BUILDDIR}/src/checkout.AthenaExternals.log 
+    -s ${BUILDDIR}/src/AthenaExternals 2>&1 || touch $error_stamp ; } | tee ${BUILDDIR}/src/checkout.AthenaExternals.log 
+test -f $error_stamp && { echo "ERROR: checkout_atlasexternals.sh FAILED. EXIT(1)" >&2 ; exit 1; }
 
 # Build AthenaExternals:
 export NICOS_PROJECT_HOME=$(cd ${BUILDDIR}/install;pwd)/AthenaExternals
@@ -140,9 +142,11 @@ platform=$(cd ${BUILDDIR}/install/AthenaExternals/${NICOS_PROJECT_VERSION}/Insta
 GaudiVersion=$(awk '/^GaudiVersion/{print $3}' ${thisdir}/externals.txt)
 
 # Check out Gaudi from the right branch/tag:
-${scriptsdir}/checkout_Gaudi.sh \
+error_stamp=`mktemp .tmp.error.XXXXX` ; rm -f $error_stamp
+{ ${scriptsdir}/checkout_Gaudi.sh \
     -t ${GaudiVersion} \
-    -s ${BUILDDIR}/src/GAUDI 2>&1 | tee ${BUILDDIR}/src/checkout.GAUDI.log
+    -s ${BUILDDIR}/src/GAUDI 2>&1 || touch $error_stamp ; } | tee ${BUILDDIR}/src/checkout.GAUDI.log
+test -f $error_stamp && { echo "ERROR: checkout_atlasexternals.sh FAILED. EXIT(1)" >&2 ; exit 1; }
 
 # Build Gaudi:
 export NICOS_PROJECT_HOME=$(cd ${BUILDDIR}/install;pwd)/GAUDI
-- 
GitLab