From e93ea0f1868f034c0569c60f7e7fa8329cef0828 Mon Sep 17 00:00:00 2001
From: Ben Morrice <ben.morrice@cern.ch>
Date: Wed, 17 Mar 2021 08:57:42 +0100
Subject: [PATCH] add dockerfile/entrypoint scripts
---
stream8_backups/Dockerfile | 11 +++++++++
stream8_backups/backup.sh | 30 +++++++++++++++++++++++
stream8_backups/common.sh | 18 ++++++++++++++
stream8_backups/email_report.tpl | 16 +++++++++++++
stream8_backups/get_credentials.sh | 14 +++++++++++
stream8_backups/interactive.sh | 38 ++++++++++++++++++++++++++++++
stream8_backups/ssmtp.conf | 4 ++++
7 files changed, 131 insertions(+)
create mode 100644 stream8_backups/Dockerfile
create mode 100755 stream8_backups/backup.sh
create mode 100644 stream8_backups/common.sh
create mode 100644 stream8_backups/email_report.tpl
create mode 100755 stream8_backups/get_credentials.sh
create mode 100755 stream8_backups/interactive.sh
create mode 100644 stream8_backups/ssmtp.conf
diff --git a/stream8_backups/Dockerfile b/stream8_backups/Dockerfile
new file mode 100644
index 0000000..9afd218
--- /dev/null
+++ b/stream8_backups/Dockerfile
@@ -0,0 +1,11 @@
+FROM gitlab-registry.cern.ch/linuxsupport/cc7-base:latest
+
+RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-7/copart-restic-epel-7.repo
+RUN yum install -y restic ssmtp gettext fuse
+COPY ssmtp.conf /etc/ssmtp/ssmtp.conf
+COPY *.sh *.tpl /root/
+
+WORKDIR /root
+
+ENTRYPOINT ["/root/backup.sh"]
+
diff --git a/stream8_backups/backup.sh b/stream8_backups/backup.sh
new file mode 100755
index 0000000..9296da7
--- /dev/null
+++ b/stream8_backups/backup.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source common.sh
+
+# we want to maintain the same $SOURCE path to simplify restores, but we
+# don't really want to read every snapshot on every run as this will take days
+# Let's just backup $TODAY, ignoring symlink snaps (zero updates) as well
+EXCLUDE_LIST=`mktemp`
+RESTIC_LOGFILE=`mktemp`
+find $SOURCE -mindepth 1 -maxdepth 1 \( -type l -o -type d \) ! -path "*$TODAY" > $EXCLUDE_LIST
+$RESTIC backup --tag $TODAY --exclude-file=$EXCLUDE_LIST $SOURCE &>> $RESTIC_LOGFILE
+rm -f $EXCLUDE_LIST
+
+# Check if there are any snapshots to forget/purge
+SNAPS_TO_REMOVE=`$RESTIC forget --dry-run --group-by paths --keep-within $PRUNE_SNAPSHOTS_OLDER_THAN | grep "remove .* snapshots" | awk '{print $2}'`
+if [ ! -z $SNAPS_TO_REMOVE ]; then
+ echo "Found $SNAPS_TO_REMOVE snapshots that are older than $PRUNE_SNAPSHOTS_OLDER_THAN. Purging from restic store ..." >> $RESTIC_LOGFILE
+ $RESTIC unlock &>> $RESTIC_LOGFILE
+ $RESTIC forget --group-by paths --keep-within $PRUNE_SNAPSHOTS_OLDER_THAN --prune &>> $RESTIC_LOGFILE
+fi
+
+echo "Sending email of $RESTIC_LOG to admins"
+export TODAY="$TODAY"
+export RESTIC_LOG="`cat $RESTIC_LOGFILE`"
+rm -f $RESTIC_LOGFILE
+
+envsubst < $TEMPLATE > email
+
+cat email | ssmtp -t -v
+rm -f email
diff --git a/stream8_backups/common.sh b/stream8_backups/common.sh
new file mode 100644
index 0000000..648a0ee
--- /dev/null
+++ b/stream8_backups/common.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+TODAY=`/bin/date +%Y%m%d`
+SOURCE=/data/$PATH_SNAPSHOTS/s8-snapshots
+if [ ! -d $SOURCE/.restic ]; then
+ mkdir $SOURCE/.restic
+fi
+if [ ! -d $SOURCE/.restic/tmpdir ]; then
+ mkdir $SOURCE/.restic/tmpdir
+fi
+if [ ! -d $SOURCE/.restic/cachedir ]; then
+ mkdir $SOURCE/.restic/cachedir
+fi
+export TMPDIR=$SOURCE/.restic/tmpdir
+TEMPLATE=email_report.tpl
+# s3.connections default is 5, let's increase it for better perf
+# and set the cachedir explicitly
+RESTIC="restic -o s3.connections=32 --cache-dir $SOURCE/.restic/cachedir"
diff --git a/stream8_backups/email_report.tpl b/stream8_backups/email_report.tpl
new file mode 100644
index 0000000..09c6851
--- /dev/null
+++ b/stream8_backups/email_report.tpl
@@ -0,0 +1,16 @@
+To: $EMAIL_ADMIN
+From: $EMAIL_FROM
+Reply-To: noreply.$EMAIL_FROM
+Return-Path: $EMAIL_ADMIN
+Subject: Stream8 - Backup for $TODAY
+
+Dear Linux admins,
+
+Today's backup ($TODAY) has completed with the following output:
+
+$RESTIC_LOG
+
+---
+Best regards,
+CERN Linux Droid
+(on behalf of the friendly humans of Linux Support)
diff --git a/stream8_backups/get_credentials.sh b/stream8_backups/get_credentials.sh
new file mode 100755
index 0000000..6451231
--- /dev/null
+++ b/stream8_backups/get_credentials.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+echo "This script defines variables required to access the S3 repo"
+echo "These variables can be found at https://gitlab.cern.ch/linuxsupport/cronjobs/centos8_backups/-/settings/ci_cd"
+echo -n "Please enter AWS_ACCESS_KEY_ID: "
+read AWS_ACCESS_KEY_ID
+echo -n "Please enter AWS_SECRET_ACCESS_KEY: "
+read AWS_SECRET_ACCESS_KEY
+echo -n "Please enter RESTIC_PASSWORD: "
+read RESTIC_PASSWORD
+
+echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> /tmp/credentials
+echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> /tmp/credentials
+echo "export RESTIC_PASSWORD=$RESTIC_PASSWORD" >> /tmp/credentials
+echo "Credentials saved to /tmp/credentials"
diff --git a/stream8_backups/interactive.sh b/stream8_backups/interactive.sh
new file mode 100755
index 0000000..8829e54
--- /dev/null
+++ b/stream8_backups/interactive.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+source common.sh
+FUSEMOUNT=fusemount
+
+if [ ! -f /tmp/credentials ]; then
+ source get_credentials.sh
+fi
+source /tmp/credentials
+
+if [ "$1" == "restore" ]; then
+ if [ -z $2 ]; then
+ echo "Error, need a snapshot [YYYYMMDD] to restore ..."
+ exit
+ fi
+ if [ -z $3 ]; then
+ echo "Error, need a target path to restore to ..."
+ exit
+ fi
+ $RESTIC restore latest --tag $2 -i $2 --target "$3"
+elif [ "$1" == "snapshots" ]; then
+ $RESTIC snapshots
+elif [ "$1" == "unlock" ]; then
+ $RESTIC unlock
+elif [ "$1" == "mount" ]; then
+ # TODO: check fusemount is not already mounted
+ if [ ! -d "$FUSEMOUNT" ]; then
+ mkdir $FUSEMOUNT
+ fi
+ echo "mounting \"$FUSEMOUNT\" (process backgrounded)"
+ echo "use: '$0 unmount' when finished"
+ $RESTIC mount $FUSEMOUNT &> /dev/null &
+elif [ "$1" == "unmount" ]; then
+ pkill -f $RESTIC &> /dev/null
+ fusermount -u $FUSEMOUNT &> /dev/null
+else
+ echo "usage: $0 <restore YYYYMMDD path|unlock|mount|unmount|snapshots>"
+fi
diff --git a/stream8_backups/ssmtp.conf b/stream8_backups/ssmtp.conf
new file mode 100644
index 0000000..518eea4
--- /dev/null
+++ b/stream8_backups/ssmtp.conf
@@ -0,0 +1,4 @@
+root=postmaster
+mailhub=cernmx.cern.ch
+TLS_CA_File=/etc/pki/tls/certs/ca-bundle.crt
+FromLineOverride=YES
--
GitLab