Merge branch 'copy-rpm-eos-retry' into 'master'

RPM to EOS copy: 5 retries for mkdir and cp before counting failure as an error

See merge request atlas/athena!10083

(cherry picked from commit e74ddf61 [formerly 492b0b9a])

c1147375 RPM to EOS copy: 5 retries for mkdir and cp before counting failure as an error
356c6297 RPM to EOS copy: 6 retries (>10m tot) for mkdir and cp before counting failure as an error

Former-commit-id: 85302a414ffc07d1267541d7827a86723987f347

Build/AtlasBuildScripts/copy_rpm_eos.sh

@@ -65,9 +65,25 @@ echo "====================================================="
 
 DESTDIR=${DESTDIR}/${BRANCH}/${ARCH}/${DDAY}
 
+_retry_() {
+    local cmd="$*" dt=16 retr=0
+    while ! $cmd ; do
+        if test $retr -ge 6 ; then
+            echo "ERROR: 6 retries of $cmd FAILED ... " >&2
+            return 1
+        fi
+        echo "WARNING: $cmd failed, waiting $dt sec ..."
+        sleep ${dt}s
+        dt=`expr $dt + $dt`
+        retr=`expr $retr + 1`
+    done
+    return 0
+}
+
+
 if [ ! -d ${DESTDIR} ] ; then 
    echo "mkdir -p ${DESTDIR}"
-   mkdir -p ${DESTDIR} 
+   _retry_ mkdir -p ${DESTDIR} 
    if [ ! -d ${DESTDIR} ] ; then ((ERROR_COUNT++)) ; fi  #avoid false positive eos error if the directory was actually created
 fi
 
@@ -79,7 +95,7 @@ if [ "${#arr_rpm[@]}" -le 0 ]; then
    for ele in "${arr_rpm[@]}" 
    do
       echo "Info: copying $ele to ${DESTDIR}"                                                 
-      cp -a $ele ${DESTDIR} || ((ERROR_COUNT++))
+      _retry_ cp -a $ele ${DESTDIR} || ((ERROR_COUNT++))
    done
 fi 
 echo "====================================================="