Skip to content
Snippets Groups Projects

sync-epel-fedora: remove exclusions and sync epel as well

Merged sync-epel-fedora: remove exclusions and sync epel as well
los590 into master
Merged Ben Morrice requested to merge los590 into master
Compare
and
1 file
+ 39
− 23
Compare changes
  • Side-by-side
  • Inline
lxsoftbin/sync-epel → lxsoftbin/sync-epel-fedora
+ 39
− 23
#!/bin/bash
#
# sync-epel
# script to sync from tier1 mirrors
# sync-epel-fedora
# script to sync EPEL/Fedora from tier1 mirrors
# The script will only sync if:
# a) the local mirror is out of date
# b) the master mirror is not in a broken state
@@ -17,10 +17,11 @@ MASTER=lxsoftadm01.cern.ch
EMAIL=1
EMAIL_LOG=`mktemp`
EMAIL_TO="lxsoft-admins@cern.ch"
TIER1_MIRRORS="archive.linux.duke.edu rsync.hrz.tu-chemnitz.de mirrors.kernel.org ftp-stud.hs-esslingen.de"
TIER1_MIRRORS="archive.linux.duke.edu fedora-rsync.ftp.pub.2iij.net mirrors.kernel.org ftp-stud.hs-esslingen.de"
RSYNC_BIN="/usr/bin/rsync"
RSYNC_OPTS=" -v -H -rlptD --safe-links --delay-updates --delete-after --max-delete=5000 --timeout=3600 --exclude=.*rpm.?* --exclude=ppc/ --exclude=ppc64/ --exclude=s390x/ --exclude=ppc64le/ --exclude=.~tmp~ -H --numeric-ids --delete --exclude=5/ --exclude=5Server/ --exclude=5Client/"
LOCAL_MIRROR="/mnt/data1/dist/epel"
RSYNC_OPTS=" -v -H -rlptD --safe-links --delay-updates --delete-after --max-delete=5000 --timeout=3600 --exclude=.*rpm.?* --exclude=.~tmp~ --numeric-ids"
LOCAL_EPEL_MIRROR="/mnt/data1/dist/epel"
LOCAL_FEDORA_MIRROR="/mnt/data1/dist/fedora"
LOG_TIMESTAMP () {
echo "[`date +%H:%M:%S`]:"
@@ -35,14 +36,18 @@ GET_MD5SUM () {
MD5SUM=`md5sum $URL | awk '{print $1}'`
fi
if [[ $URL == *"rsync://"* ]]; then
rsync $URL /tmp/epeltempfile &> /dev/null
MD5SUM=`md5sum /tmp/epeltempfile | awk '{print $1}'`
rm -f /tmp/epeltempfile
rsync $URL /tmp/fedorasynctempfile &> /dev/null
if [ -f /tmp/fedorasynctempfile ]; then
MD5SUM=`md5sum /tmp/fedorasynctempfile | awk '{print $1}'`
rm -f /tmp/fedorasynctempfile
else
MD5SUM="failed"
fi
fi
echo $MD5SUM
}
echo "$(LOG_TIMESTAMP) Starting EPEL sync script" | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Starting EPEL/Fedora sync script" | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Checking EPEL master repodata is sound ..." | /usr/bin/tee -a $EMAIL_LOG
# check if master repodata is not actually broken
repoquery --repofrompath=tmpepel7,https://dl.fedoraproject.org/pub/epel/7/x86_64/ --repoid=tmpepel7 --quiet -i fail2ban &> /dev/null
@@ -57,17 +62,23 @@ if [ $? -ne 0 ]; then
fi
# collect md5sums for comparison
MASTER_FULLFILELIST_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/fullfilelist)
MASTER_EPEL_FULLFILELIST_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/fullfilelist)
MASTER_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/7/x86_64/repodata/repomd.xml)
MASTER_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/repodata/repomd.xml)
MASTER_FEDORA_FULLFILELIST_MD5SUM=$(GET_MD5SUM https://dl.fedoraproject.org/pub/fedora/fullfilelist)
LOCAL_FULLFILELIST_MD5SUM=$(GET_MD5SUM $LOCAL_MIRROR/fullfilelist)
LOCAL_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM $LOCAL_MIRROR/7/x86_64/repodata/repomd.xml)
LOCAL_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM $LOCAL_MIRROR/8/Everything/x86_64/repodata/repomd.xml)
LOCAL_EPEL_FULLFILELIST_MD5SUM=$(GET_MD5SUM $LOCAL_EPEL_MIRROR/fullfilelist)
LOCAL_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM $LOCAL_EPEL_MIRROR/7/x86_64/repodata/repomd.xml)
LOCAL_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM $LOCAL_EPEL_MIRROR/8/Everything/x86_64/repodata/repomd.xml)
LOCAL_FEDORA_FULLFILELIST_MD5SUM=$(GET_MD5SUM $LOCAL_FEDORA_MIRROR/fullfilelist)
# this number will change depending on how many things we check
NEED_UPDATE=3
if [ "$MASTER_FULLFILELIST_MD5SUM" == "$LOCAL_FULLFILELIST_MD5SUM" ]; then
NEED_UPDATE=4
if [ "$MASTER_EPEL_FULLFILELIST_MD5SUM" == "$LOCAL_EPEL_FULLFILELIST_MD5SUM" ]; then
NEED_UPDATE=$((NEED_UPDATE-1))
fi
if [ "$MASTER_FEDORA_FULLFILELIST_MD5SUM" == "$LOCAL_FEDORA_FULLFILELIST_MD5SUM" ]; then
NEED_UPDATE=$((NEED_UPDATE-1))
fi
@@ -78,16 +89,18 @@ if [ "$MASTER_EPEL8_REPODATA_MD5SUM" == "$LOCAL_EPEL8_REPODATA_MD5SUM" ]; then
NEED_UPDATE=$((NEED_UPDATE-1))
fi
if [ $NEED_UPDATE -ne 0 ]; then
echo "$(LOG_TIMESTAMP) Linuxsoft EPEL mirror is out of sync with master, running update" | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Linuxsoft EPEL/Fedora mirror is out of sync with master, running update" | /usr/bin/tee -a $EMAIL_LOG
for MIRROR in $TIER1_MIRRORS
do
MIRROR_FULLFILELIST_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel/fullfilelist)
MIRROR_EPEL_FULLFILELIST_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel/fullfilelist)
MIRROR_FEDORA_FULLFILELIST_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-enchilada/fullfilelist)
MIRROR_EPEL7_REPODATA_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel/7/x86_64/repodata/repomd.xml)
MIRROR_EPEL8_REPODATA_MD5SUM=$(GET_MD5SUM rsync://$MIRROR/fedora-epel//8/Everything/x86_64/repodata/repomd.xml)
if [ "$MASTER_FULLFILELIST_MD5SUM" == "$MIRROR_FULLFILELIST_MD5SUM" ] && [ "$MASTER_EPEL7_REPODATA_MD5SUM" == "$MIRROR_EPEL7_REPODATA_MD5SUM" ] && [ "$MASTER_EPEL8_REPODATA_MD5SUM" == "$MIRROR_EPEL8_REPODATA_MD5SUM" ]; then
if [ "$MASTER_EPEL_FULLFILELIST_MD5SUM" == "$MIRROR_EPEL_FULLFILELIST_MD5SUM" ] && [ "$MASTER_EPEL7_REPODATA_MD5SUM" == "$MIRROR_EPEL7_REPODATA_MD5SUM" ] && [ "$MASTER_EPEL8_REPODATA_MD5SUM" == "$MIRROR_EPEL8_REPODATA_MD5SUM" ] && [ "$MASTER_FEDORA_FULLFILELIST_MD5SUM" == "$MIRROR_FEDORA_FULLFILELIST_MD5SUM" ]; then
echo "$(LOG_TIMESTAMP) Mirror $MIRROR is current, using it as a sync source" | /usr/bin/tee -a $EMAIL_LOG
MIRROR_SYNC_SOURCE=$MIRROR
REMOTE_MIRROR="rsync://$MIRROR/fedora-epel"
REMOTE_EPEL_MIRROR="rsync://$MIRROR/fedora-epel"
REMOTE_FEDORA_MIRROR="rsync://$MIRROR/fedora-enchilada"
break
fi
done
@@ -95,14 +108,17 @@ if [ $NEED_UPDATE -ne 0 ]; then
echo "$(LOG_TIMESTAMP) All tier1 mirrors are not currently in sync with the master, avoiding linuxsoft sync" | /usr/bin/tee -a $EMAIL_LOG
else
# finally run rsync :)
echo "$(LOG_TIMESTAMP) Running $RSYNC_BIN from $REMOTE_MIRROR to $LOCAL_MIRROR with $RSYNC_OPTS" | /usr/bin/tee -a $EMAIL_LOG
eval $RSYNC_BIN $REMOTE_MIRROR/ $LOCAL_MIRROR/ $RSYNC_OPTS | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Running EPEL sync with $RSYNC_BIN from $REMOTE_EPEL_MIRROR to $LOCAL_EPEL_MIRROR with $RSYNC_OPTS" | /usr/bin/tee -a $EMAIL_LOG
eval $RSYNC_BIN $REMOTE_EPEL_MIRROR/ $LOCAL_EPEL_MIRROR/ $RSYNC_OPTS | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Finished $RSYNC_BIN with return code $?" | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Running Fedora sync with $RSYNC_BIN from $REMOTE_FEDORA_MIRROR to $LOCAL_FEDORA_MIRROR with $RSYNC_OPTS" | /usr/bin/tee -a $EMAIL_LOG
eval $RSYNC_BIN $REMOTE_FEDORA_MIRROR/ $LOCAL_FEDORA_MIRROR/ $RSYNC_OPTS | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Finished $RSYNC_BIN with return code $?" | /usr/bin/tee -a $EMAIL_LOG
fi
else
echo "$(LOG_TIMESTAMP) Linuxsoft epel7/8 repos are upto date, not running a sync at this time" | /usr/bin/tee -a $EMAIL_LOG
echo "$(LOG_TIMESTAMP) Linuxsoft EPEL/Fedora mirror is upto date, not running a sync at this time" | /usr/bin/tee -a $EMAIL_LOG
fi
if [ $EMAIL -eq 1 ]; then
cat $EMAIL_LOG | mail -s "EPEL sync status for `date +%Y%m%d`" -r Linux.Support@cern.ch $EMAIL_TO
cat $EMAIL_LOG | mail -s "EPEL/Fedora sync status for `date +%Y%m%d`" -r Linux.Support@cern.ch $EMAIL_TO
fi
rm -f $EMAIL_LOG