retry git checkout/fetch in checkout_atlasexternals.sh and checkout_Gaudi.sh (ATLINFR-1817)

Former-commit-id: ead4c942

Build/AtlasBuildScripts/checkout_Gaudi.sh

@@ -27,6 +27,22 @@ usage() {
 	echo "  in the source directory is left untouched."
 }
 
+_max_retry_=5
+
+_retry_ () {
+    local cmd="$*"
+    local n=0
+    while ! $cmd ; do
+        if test $n -eq $_max_retry_ ; then
+            echo "ERROR: $cmd FAILED $_max_retry_ times. EXIT(1)" >&2
+            exit 1
+        fi
+        echo "WARNING: $cmd FAILED, retry in 30 sec ... "
+        sleep 30s
+        n=`expr $n + 1`
+    done
+}
+
 # Parse the command line arguments:
 TAGBRANCH=""
 SOURCEDIR=""
@@ -86,12 +102,12 @@ echo "   from: $GAUDIURL"
 
 if [ ! -d "${SOURCEDIR}" ]; then
     # Clone the repository:
-    git clone ${GAUDIURL} ${SOURCEDIR}
+    _retry_ git clone ${GAUDIURL} ${SOURCEDIR}
 else
     echo "${SOURCEDIR} already exists -> assume previous checkout"
 fi
 
 # Get the appropriate version of it:
 cd ${SOURCEDIR}
-git fetch --prune origin
-git checkout -f ${TAGBRANCH}
+_retry_ git fetch --prune origin
+_retry_ git checkout -f ${TAGBRANCH}

Build/AtlasBuildScripts/checkout_atlasexternals.sh

@@ -27,6 +27,22 @@ usage() {
 	echo "  in the source directory is left untouched."
 }
 
+_max_retry_=5
+
+_retry_ () {
+    local cmd="$*"
+    local n=0
+    while ! $cmd ; do
+        if test $n -eq $_max_retry_ ; then
+            echo "ERROR: $cmd FAILED $_max_retry_ times. EXIT(1)" >&2
+            exit 1
+        fi
+        echo "WARNING: $cmd FAILED, retry in 30 sec ... "
+        sleep 30s
+        n=`expr $n + 1`
+    done
+}
+
 # Parse the command line arguments:
 TAGBRANCH=""
 SOURCEDIR=""
@@ -86,12 +102,12 @@ echo "   from: $EXTERNALSURL"
 
 if [ ! -d "${SOURCEDIR}" ]; then
     # Clone the repository:
-    git clone ${EXTERNALSURL} ${SOURCEDIR}
+    _retry_ git clone ${EXTERNALSURL} ${SOURCEDIR}
 else
     echo "${SOURCEDIR} already exists -> assume previous checkout"
 fi
 
 # Get the appropriate version of it:
 cd ${SOURCEDIR}
-git fetch --prune origin
-git checkout -f ${TAGBRANCH}
+_retry_ git fetch --prune origin
+_retry_ git checkout -f ${TAGBRANCH}

Projects/Athena/build_externals.sh

@@ -119,9 +119,11 @@ fi
 AthenaExternalsVersion=$(awk '/^AthenaExternalsVersion/{print $3}' ${thisdir}/externals.txt)
 
 # Check out AthenaExternals from the right branch/tag:
-${scriptsdir}/checkout_atlasexternals.sh \
+error_stamp=`mktemp .tmp.error.XXXXX` ; rm -f $error_stamp
+{ ${scriptsdir}/checkout_atlasexternals.sh \
     -t ${AthenaExternalsVersion} \
-    -s ${BUILDDIR}/src/AthenaExternals 2>&1 | tee ${BUILDDIR}/src/checkout.AthenaExternals.log 
+    -s ${BUILDDIR}/src/AthenaExternals 2>&1 || touch $error_stamp ; } | tee ${BUILDDIR}/src/checkout.AthenaExternals.log 
+test -f $error_stamp && { echo "ERROR: checkout_atlasexternals.sh FAILED. EXIT(1)" >&2 ; exit 1; }
 
 # Build AthenaExternals:
 export NICOS_PROJECT_HOME=$(cd ${BUILDDIR}/install;pwd)/AthenaExternals
@@ -140,9 +142,11 @@ platform=$(cd ${BUILDDIR}/install/AthenaExternals/${NICOS_PROJECT_VERSION}/Insta
 GaudiVersion=$(awk '/^GaudiVersion/{print $3}' ${thisdir}/externals.txt)
 
 # Check out Gaudi from the right branch/tag:
-${scriptsdir}/checkout_Gaudi.sh \
+error_stamp=`mktemp .tmp.error.XXXXX` ; rm -f $error_stamp
+{ ${scriptsdir}/checkout_Gaudi.sh \
     -t ${GaudiVersion} \
-    -s ${BUILDDIR}/src/GAUDI 2>&1 | tee ${BUILDDIR}/src/checkout.GAUDI.log
+    -s ${BUILDDIR}/src/GAUDI 2>&1 || touch $error_stamp ; } | tee ${BUILDDIR}/src/checkout.GAUDI.log
+test -f $error_stamp && { echo "ERROR: checkout_atlasexternals.sh FAILED. EXIT(1)" >&2 ; exit 1; }
 
 # Build Gaudi:
 export NICOS_PROJECT_HOME=$(cd ${BUILDDIR}/install;pwd)/GAUDI