diff --git a/lxsoftbin/new-lxsoft-sync-mirrors b/lxsoftbin/new-lxsoft-sync-mirrors
index 498bff06d9173dcf792232eba148d451951c46a4..19ab8c02cd5b9ad35ac6b2192656a94bb7c112a1 100755
--- a/lxsoftbin/new-lxsoft-sync-mirrors
+++ b/lxsoftbin/new-lxsoft-sync-mirrors
@@ -7,9 +7,6 @@
[[ $EUID -eq 0 ]] && echo "Do not run it as root. Exit!" && exit 1
#MIRRORSL="rsync.scientificlinux.org::scientific"
-# https://admin.fedoraproject.org/mirrormanager/mirrors/EPEL/7
-#MIRRORFEDORA="rsync://fedora.mirrors.ovh.net/download.fedora.redhat.com/"
-MIRRORFEDORA="rsync://ftp.uni-stuttgart.de/epel"
MIRRORDAG="rsync://apt.sw.be/pub/freshrpms/pub/dag"
MIRRORATRPMS="rsync://rsync.hrz.tu-chemnitz.de/ftp/pub/linux/ATrpms/"
#MIRRORELREPO="elrepo.reloumirrors.net::elrepo"
@@ -48,7 +45,7 @@ help() {
echo " -v = more verbose sync"
echo " -q = be quiet"
echo " -n = noaction"
- echo " -s ARG , where ARG centos, dag, jpackage, atrpms, fedora-epel, elrepo, repoforge, emi, linuxtech, centos, centosvault, centosalt, centosdebuginfo"
+ echo " -s ARG , where ARG centos, dag, jpackage, atrpms, elrepo, repoforge, emi, linuxtech, centos, centosvault, centosalt, centosdebuginfo"
exit 0
}
@@ -63,7 +60,7 @@ done
RSYNC_OPTS="$RSYNC_OPTS $RSYNC_VERBOSE"
-[ "$SYNC" == "all" ] && TODO="dag jpackage fedora-epel atrpms elrepo repoforge emi centosvault centosdebuginfo" || TODO=$SYNC
+[ "$SYNC" == "all" ] && TODO="dag jpackage atrpms elrepo repoforge emi centosvault centosdebuginfo" || TODO=$SYNC
[ $VERBOSE -eq 1 ] && RSYNC_OPTS="$RSYNC_OPTS --progress"
[ $CHECKSUM -eq 1 ] && RSYNC_OPTS="$RSYNC_OPTS -c"
[ $NOACTION -eq 1 ] && RSYNC_OPTS="$RSYNC_OPTS -n"
@@ -123,33 +120,6 @@ for COMP in ${TODO}; do
RSYNC_OPTS="$RSYNC_OPTS --exclude=1.0/ --exclude=1.5/ --exclude=1.6/ --exclude=1.7/"
;;
- # "fedora-linux-releases" ) TOP_REMOTE="$MIRRORFEDORA/linux/releases";
- # TOP_LOCAL="/mnt/data1/dist/fedora/linux/releases";
- # RSYNC_OPTS="$RSYNC_OPTS --include='23/***' --exclude='*' -H --numeric-ids --delete" ;;
- #
- # "fedora-linux-development" ) TOP_REMOTE="$MIRRORFEDORA/linux/development";
- # TOP_LOCAL="/mnt/data1/dist/fedora/linux/development";
- # RSYNC_OPTS="$RSYNC_OPTS --include='rawhide/***' --exclude='*' --exclude='rawhide/Spins/' -H --numeric-ids --delete" ;;
- #
- # "fedora-linux-updates" ) TOP_REMOTE="$MIRRORFEDORA/linux/updates";
- # TOP_LOCAL="/mnt/data1/dist/fedora/linux/updates";
- # RSYNC_OPTS="$RSYNC_OPTS --include='23/***' --exclude='*' -H --numeric-ids --delete";;
-
- "fedora-epel")
- #TOP_REMOTE="rsync://mirrors.ircam.fr/fedora-epel"
- #TOP_REMOTE="rsync://mirror.vutbr.cz/epel"
- #TOP_REMOTE="rsync://mirror.vutbr.cz/epel"
- #TOP_REMOTE="rsync://ftp.heanet.ie/pub/fedora/epel"
- #TOP_REMOTE="rsync://epel.mirrors.ovh.net/fedora-epel" # broken since 03.12 (checked 11.12.2017)
- #TOP_REMOTE="rsync://rsync.informatik.rwth-aachen.de/fedora-epel" # broken 13.12.2017
- #TOP_REMOTE="rsync://mirror.in2p3.fr/pub/epel"
- #TOP_REMOTE="rsync://ftp.fau.de/epel"
- #TOP_REMOTE="rsync://archive.linux.duke.edu/fedora-epel"
- TOP_REMOTE="rsync://ftp-stud.hs-esslingen.de/epel"
- TOP_LOCAL="/mnt/data1/dist/epel"
- RSYNC_OPTS="$RSYNC_OPTS --exclude=ppc/ --exclude=ppc64/ --exclude=s390x/ --exclude=ppc64le/ --exclude=.~tmp~ -H --numeric-ids --delete --exclude=5/ --exclude=5Server/ --exclude=5Client/"
- ;;
-
"atrpms")
TOP_REMOTE="$MIRRORATRPMS"
TOP_LOCAL="/mnt/data1/dist/atrpms"
diff --git a/lxsoftbin/sync-epel b/lxsoftbin/sync-epel
new file mode 100755
index 0000000000000000000000000000000000000000..710030fb484e9284f4e046d7edfef49063c5d993
--- /dev/null
+++ b/lxsoftbin/sync-epel
@@ -0,0 +1,102 @@
+#!/bin/bash
+#
+# sync-epel
+# script to sync from tier1 mirrors
+# The script will only sync if:
+# a) the local mirror is out of date
+# b) the master mirror is not in a broken state
+# c) the tier1 mirror is not in a broken state
+# The script will retry additional tier1 mirrors if it's broken
+
+EMAIL=1
+EMAIL_LOG=`mktemp`
+EMAIL_TO="lxsoft-admins@cern.ch"
+TIER1_MIRRORS="archive.linux.duke.edu rsync.hrz.tu-chemnitz.de mirrors.kernel.org ftp-stud.hs-esslingen.de"
+RSYNC_BIN="/usr/bin/rsync"
+RSYNC_OPTS=" -H -rlptD --safe-links --delay-updates --delete-after --max-delete=5000 --timeout=3600 --exclude=.*rpm.?* --exclude=ppc/ --exclude=ppc64/ --exclude=s390x/ --exclude=ppc64le/ --exclude=.~tmp~ -H --numeric-ids --delete --exclude=5/ --exclude=5Server/ --exclude=5Client/"
+LOCAL_MIRROR="/mnt/data1/dist/epel"
+
+LOG_TIMESTAMP () {
+ echo "[`date +%H:%M:%S`]:"
+}
+
+GET_MD5SUM () {
+ URL=$1
+ if [[ $URL == *"dl.fedoraproject.org"* ]]; then
+ MD5SUM=`curl -s $URL | md5sum | awk '{print $1}'`
+ fi
+ if [[ $URL == *"mnt/data1"* ]]; then
+ MD5SUM=`md5sum $URL | awk '{print $1}'`
+ fi
+ if [[ $URL == *"rsync://"* ]]; then
+ rsync $URL /tmp/epeltempfile &> /dev/null
+ MD5SUM=`md5sum /tmp/epeltempfile | awk '{print $1}'`
+ rm -f /tmp/epeltempfile
+ fi
+ echo $MD5SUM
+}
+
+echo "$(LOG_TIMESTAMP) Starting EPEL sync script" | /usr/bin/tee -a $EMAIL_LOG
+echo "$(LOG_TIMESTAMP) Checking EPEL master repodata is sound ..." | /usr/bin/tee -a $EMAIL_LOG
+# check if master repodata is not actually broken
+repoquery --repofrompath=tmpepel7,https://dl.fedoraproject.org/pub/epel/7/x86_64/ --repoid=tmpepel7 --quiet -i fail2ban &> /dev/null
+if [ $? -ne 0 ]; then
+ echo "$(LOG_TIMESTAMP) EPEL master server metadata is broken for el7, not attempting sync" | /usr/bin/tee -a $EMAIL_LOG
+ break
+fi
+repoquery --repofrompath=tmpepel8,https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/ --repoid=tmpepel8 --quiet -i fail2ban &> /dev/null
+if [ $? -ne 0 ]; then
+ echo "$(LOG_TIMESTAMP) EPEL master server metadata is broken for el8, not attempting sync" | /usr/bin/tee -a $EMAIL_LOG
+ break
+fi
+
+# collect md5sums for comparison
+MASTER_FULLFILELIST_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/fullfilelist)
+MASTER_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/7/x86_64/repodata/repomd.xml)
+MASTER_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/repodata/repomd.xml)
+
+LOCAL_FULLFILELIST_MD5SUM=$(GET_MD5SUM $LOCAL_MIRROR/fullfilelist)
+LOCAL_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM $LOCAL_MIRROR/7/x86_64/repodata/repomd.xml)
+LOCAL_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM $LOCAL_MIRROR/8/Everything/x86_64/repodata/repomd.xml)
+
+# this number will change depending on how many things we check
+NEED_UPDATE=3
+if [ "$MASTER_FULLFILELIST_MD5SUM" == "$LOCAL_FULLFILELIST_MD5SUM" ]; then
+ NEED_UPDATE=$((NEED_UPDATE-1))
+fi
+
+if [ "$MASTER_EPEL7_REPODATA_MD5SUM" == "$LOCAL_EPEL7_REPODATA_MD5SUM" ]; then
+ NEED_UPDATE=$((NEED_UPDATE-1))
+fi
+if [ "$MASTER_EPEL8_REPODATA_MD5SUM" == "$LOCAL_EPEL8_REPODATA_MD5SUM" ]; then
+ NEED_UPDATE=$((NEED_UPDATE-1))
+fi
+if [ $NEED_UPDATE -ne 0 ]; then
+ echo "$(LOG_TIMESTAMP) Linuxsoft EPEL mirror is out of sync with master, running update"
+ for MIRROR in $TIER1_MIRRORS
+ do
+ MIRROR_FULLFILELIST_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel/fullfilelist)
+ MIRROR_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel/7/x86_64/repodata/repomd.xml)
+ MIRROR_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel//8/Everything/x86_64/repodata/repomd.xml)
+ if [ "$MASTER_FULLFILELIST_MD5SUM" == "$MIRROR_FULLFILELIST_MD5SUM" ] && [ "$MASTER_EPEL7_REPODATA_MD5SUM" == "$MIRROR_EPEL7_REPODATA_MD5SUM" ] && [ "$MASTER_EPEL8_REPODATA_MD5SUM" == "$MIRROR_EPEL8_REPODATA_MD5SUM" ]; then
+ echo "$(LOG_TIMESTAMP) Mirror $MIRROR is current, using it as a sync source" | /usr/bin/tee -a $EMAIL_LOG
+ MIRROR_SYNC_SOURCE=$MIRROR
+ REMOTE_MIRROR="rsync://$MIRROR/fedora-epel"
+ break
+ fi
+ done
+ if [ -z $MIRROR_SYNC_SOURCE ]; then
+ echo "$(LOG_TIMESTAMP) All tier1 mirrors are not currently in sync with the master, avoiding linuxsoft sync" | /usr/bin/tee -a $EMAIL_LOG
+ else
+ # finally run rsync :)
+ echo "$(LOG_TIMESTAMP) Running $RSYNC_BIN from $REMOTE_MIRROR to $LOCAL_MIRROR with $RSYNC_OPTS"
+ eval $RSYNC_BIN $REMOTE_MIRROR/ $LOCAL_MIRROR/ $RSYNC_OPTS
+ echo "$(LOG_TIMESTAMP) Finished $RSYNC_BIN with return code $?" | /usr/bin/tee -a $EMAIL_LOG
+ fi
+else
+ echo "$(LOG_TIMESTAMP) Linuxsoft epel7/8 repos are upto date, not running a sync at this time" | /usr/bin/tee -a $EMAIL_LOG
+fi
+if [ $EMAIL -eq 1 ]; then
+ cat $EMAIL_LOG | mail -s "EPEL sync status for `date +%Y%m%d`" -r Linux.Support@cern.ch $EMAIL_TO
+fi
+rm -f $EMAIL_LOG