Merge branch 'testing' into 'master'

initial release

See merge request !1

.gitlab-ci.yml

+---
+include: 'https://gitlab.cern.ch/linuxsupport/cronjobs/base/raw/master/gitlab-ci.yml'
+
+
+build_stream8_backups:
+  stage: build
+  tags:
+    - docker-image-build
+  script: "echo" # unused but this line is required by GitLab CI
+  variables:
+    CONTEXT_DIR: stream8_backups
+    TO: $CI_REGISTRY_IMAGE/stream8_backups:$CI_COMMIT_REF_NAME
+
+deploy:
+  extends: .nomad
+  stage: deploy
+  script:
+    - for j in *.nomad; do echo -e "\n${j}:"; nomad job run <(envsubst < $j); done
+  only:
+    - tags
+    - master
+
+deploy_dev:
+  extends: .nomad
+  stage: deploy
+  script:
+    - for j in *.nomad; do echo -e "\n${j}:"; nomad job run <(envsubst < $j); done
+  except:
+    - tags
+    - master
+  when: manual

README.md

 # stream8_backups
 
+This is a nomad job to backup the daily snapshots created by stream8_snapshots
+
+The entrypoint `backup.sh` performs a backup of $TODAY using restic, storing the data on s3.cern.ch
+
+A helper script `interactive.sh` (included in the docker image) provides the ability to query the restic repository in an interactive fashion.
+
+To launch `interactive.sh` it is recommended to run a privileged docker instance as follows:
+
+```bash
+    $ export BRANCH=<branch> EMAIL=<you>@cern.ch S3REPO=s8-backups-testing PATH_SNAPSHOTS=TMP;
+    docker run -it --rm --entrypoint bash \
+    --privileged --cap-add SYS_ADMIN --cap-add MKNOD --device /dev/fuse
+    -v /mnt/data1/dist:/data \
+    -e PATH_SNAPSHOTS="$PATH_SNAPSHOTS" \
+    -e RESTIC_REPOSITORY="s3:s3.cern.ch/$S3REPO" \
+    -e EMAIL_FROM="linux.support@cern.ch" \
+    -e EMAIL_ADMIN="$EMAIL" \
+    -e NOMAD_ADDR="https://lxsoftadm.cern.ch:4646" \
+    gitlab-registry.cern.ch/linuxsupport/cronjobs/stream8_backups/stream8_backups:$BRANCH
+```
+
+(for the production data, use `S3REPO="s8-backups-production" PATH_SNAPSHOTS="cern/centos"`)
+
+`interactive.sh` provides the following abilities:
+
+1. list snapshots
+`./interactive.sh snapshots`
+2. restore a snapshot
+`./interactive.sh restore YYYYMMDD /path/to/restore/to`
+3. fuse mount the restic respository
+`./interactive.sh mount` (mounts to `/root/fusemount`)
+4. unmount the restic repository
+`./interactive.sh unmount`
+
+S3 credentials are required to access the restic repo, which are embedded into the nomad job for the daily backup use case.
+
+We are currently using credentials that map to the "Linuxsoft service" OpenStack tenant, which has a quota of 500GB
+
+When using `interactive.sh`, the script will ask you to enter S3 credentials. These credentials are stored within the container in `/tmp/credentials`. If you wish to use restic directly (outside of the `interactive.sh` script), you may `source /tmp/credentils` after first running `interactive.sh`.
+
+If you do not know where to retrieve the credentials, they can be found in the 'Variables' section of <https://gitlab.cern.ch/linuxsupport/cronjobs/stream8/-/settings/ci_cd> or alternatively via `tbag showkeys --hg lxsoft/adm`.
+
+```
+for i in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY RESTIC_PASSWORD; do tbag show --hg lxsoft/adm stream8_backups_$i; done
+```
+
+Note: The backup repository/bucket must be initialised once, which is a manual task.
+You can accomplish this by:
+```
+./get_credentials.sh
+source /tmp/credentials
+restic init
+```
+

dev.variables.sh

+SCHEDULE="15 9 * * 1"
+DATA="/mnt/data1/dist"
+PATH_SNAPSHOTS="tmp"
+S3_REPOSITORY="s8-backups-testing"
+PRUNE_SNAPSHOTS_OLDER_THAN="2d"
+EMAIL_FROM="Linux.Support@cern.ch"
+EMAIL_ADMIN="morrice@cern.ch"

prod.variables.sh

+SCHEDULE="45 9 * * *"
+DATA="/mnt/data1/dist"
+PATH_SNAPSHOTS="cern/centos"
+S3_REPOSITORY="s8-backups-production"
+PRUNE_SNAPSHOTS_OLDER_THAN="730d"
+EMAIL_FROM="Linux.Support@cern.ch"
+EMAIL_ADMIN="lxsoft-admins@cern.ch"

stream8_backups.nomad

+job "${PREFIX}_stream8_backups" {
+  datacenters = ["meyrin"]
+
+  type = "batch"
+
+  periodic {
+    cron = "${SCHEDULE}"
+    time_zone = "Europe/Zurich"
+    prohibit_overlap = false
+  }
+
+  reschedule {
+    attempts       = 276
+    interval       = "23h"
+    unlimited      = false
+    delay          = "5m"
+    delay_function = "constant"
+  }
+
+  task "${PREFIX}_stream8_backups" {
+    driver = "docker"
+
+    config {
+      image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/stream8_backups/stream8_backups:${CI_COMMIT_REF_NAME}"
+      force_pull = ${FORCE_PULL}
+      logging {
+        config {
+          tag = "${PREFIX}_stream8_backups"
+        }
+      }
+      volumes = [
+        "$DATA:/data",
+      ]
+    }
+
+    env {
+      RESTIC_PASSWORD = "$RESTIC_PASSWORD"
+      AWS_ACCESS_KEY_ID = "$AWS_ACCESS_KEY_ID"
+      AWS_SECRET_ACCESS_KEY = "$AWS_SECRET_ACCESS_KEY"
+      RESTIC_REPOSITORY = "s3:s3.cern.ch/$S3_REPOSITORY"
+      PRUNE_SNAPSHOTS_OLDER_THAN = "$PRUNE_SNAPSHOTS_OLDER_THAN"
+      PATH_SNAPSHOTS = "$PATH_SNAPSHOTS"
+      EMAIL_FROM = "$EMAIL_FROM"
+      EMAIL_ADMIN = "$EMAIL_ADMIN"
+      NOMAD_ADDR = "$NOMAD_ADDR"
+    }
+
+    resources {
+      cpu = 6000 # Mhz
+      memory = 1024 # MB
+
+      network {
+        mbits = 10
+      }
+    }
+
+  }
+}

stream8_backups/Dockerfile

+FROM gitlab-registry.cern.ch/linuxsupport/cc7-base:latest
+
+RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/copart/restic/repo/epel-7/copart-restic-epel-7.repo
+RUN yum install -y restic ssmtp gettext fuse
+COPY ssmtp.conf /etc/ssmtp/ssmtp.conf
+COPY *.sh *.tpl /root/
+
+WORKDIR /root
+
+ENTRYPOINT ["/root/backup.sh"]
+

stream8_backups/backup.sh

+#!/bin/bash
+
+source common.sh
+
+# we want to maintain the same $SOURCE path to simplify restores, but we
+# don't really want to read every snapshot on every run as this will take days
+# Let's just backup $TODAY, ignoring symlink snaps (zero updates) as well
+EXCLUDE_LIST=`mktemp`
+RESTIC_LOGFILE=`mktemp`
+find $SOURCE -mindepth 1 -maxdepth 1 \( -type l -o -type d \) ! -path "*$TODAY" > $EXCLUDE_LIST
+$RESTIC backup --tag $TODAY --exclude-file=$EXCLUDE_LIST $SOURCE &>> $RESTIC_LOGFILE
+rm -f $EXCLUDE_LIST
+
+# Check if there are any snapshots to forget/purge
+SNAPS_TO_REMOVE=`$RESTIC forget --dry-run --group-by paths --keep-within $PRUNE_SNAPSHOTS_OLDER_THAN | grep "remove .* snapshots" | awk '{print $2}'`
+if [ ! -z $SNAPS_TO_REMOVE ]; then
+  echo "Found $SNAPS_TO_REMOVE snapshots that are older than $PRUNE_SNAPSHOTS_OLDER_THAN. Purging from restic store ..." >> $RESTIC_LOGFILE
+  $RESTIC unlock &>> $RESTIC_LOGFILE
+  $RESTIC forget --group-by paths --keep-within $PRUNE_SNAPSHOTS_OLDER_THAN --prune &>> $RESTIC_LOGFILE
+fi
+
+echo "Sending email of $RESTIC_LOG to admins"
+export TODAY="$TODAY"
+export RESTIC_LOG="`cat $RESTIC_LOGFILE`"
+rm -f $RESTIC_LOGFILE
+
+envsubst < $TEMPLATE > email
+
+cat email | ssmtp -t -v
+rm -f email

stream8_backups/common.sh

+#!/bin/bash
+
+TODAY=`/bin/date +%Y%m%d`
+SOURCE=/data/$PATH_SNAPSHOTS/s8-snapshots
+if [ ! -d $SOURCE/.restic ]; then
+  mkdir $SOURCE/.restic
+fi
+if [ ! -d $SOURCE/.restic/tmpdir ]; then
+  mkdir $SOURCE/.restic/tmpdir
+fi
+if [ ! -d $SOURCE/.restic/cachedir ]; then
+  mkdir $SOURCE/.restic/cachedir
+fi
+export TMPDIR=$SOURCE/.restic/tmpdir
+TEMPLATE=email_report.tpl
+# s3.connections default is 5, let's increase it for better perf
+# and set the cachedir explicitly
+RESTIC="restic -o s3.connections=32 --cache-dir $SOURCE/.restic/cachedir"

stream8_backups/email_report.tpl

+To: $EMAIL_ADMIN
+From: $EMAIL_FROM
+Reply-To: noreply.$EMAIL_FROM
+Return-Path: $EMAIL_ADMIN
+Subject: Stream8 - Backup for $TODAY
+
+Dear Linux admins,
+
+Today's backup ($TODAY) has completed with the following output:
+
+$RESTIC_LOG
+
+---
+Best regards,
+CERN Linux Droid
+(on behalf of the friendly humans of Linux Support)

stream8_backups/get_credentials.sh

+#!/bin/bash
+echo "This script defines variables required to access the S3 repo"
+echo "These variables can be found at https://gitlab.cern.ch/linuxsupport/cronjobs/centos8_backups/-/settings/ci_cd"
+echo -n "Please enter AWS_ACCESS_KEY_ID: "
+read AWS_ACCESS_KEY_ID
+echo -n "Please enter AWS_SECRET_ACCESS_KEY: "
+read AWS_SECRET_ACCESS_KEY
+echo -n "Please enter RESTIC_PASSWORD: "
+read RESTIC_PASSWORD
+
+echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> /tmp/credentials
+echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> /tmp/credentials
+echo "export RESTIC_PASSWORD=$RESTIC_PASSWORD" >> /tmp/credentials
+echo "Credentials saved to /tmp/credentials"

stream8_backups/interactive.sh

+#!/bin/bash
+
+source common.sh
+FUSEMOUNT=fusemount
+
+if [ ! -f /tmp/credentials ]; then
+  source get_credentials.sh
+fi
+source /tmp/credentials
+
+if [ "$1" == "restore" ]; then
+  if [ -z $2 ]; then
+    echo "Error, need a snapshot [YYYYMMDD] to restore ..."
+    exit
+  fi
+  if [ -z $3 ]; then
+    echo "Error, need a target path to restore to ..."
+    exit
+  fi
+  $RESTIC restore latest --tag $2 -i $2 --target "$3"
+elif [ "$1" == "snapshots" ]; then
+  $RESTIC snapshots   
+elif [ "$1" == "unlock" ]; then
+  $RESTIC unlock   
+elif [ "$1" == "mount" ]; then
+  # TODO: check fusemount is not already mounted
+  if [ ! -d "$FUSEMOUNT" ]; then
+    mkdir $FUSEMOUNT
+  fi
+  echo "mounting \"$FUSEMOUNT\" (process backgrounded)"
+  echo "use: '$0 unmount' when finished"
+  $RESTIC mount $FUSEMOUNT &> /dev/null &
+elif [ "$1" == "unmount" ]; then
+  pkill -f $RESTIC &> /dev/null
+  fusermount -u $FUSEMOUNT &> /dev/null
+else
+  echo "usage: $0 <restore YYYYMMDD path|unlock|mount|unmount|snapshots>"
+fi

stream8_backups/ssmtp.conf

+root=postmaster
+mailhub=cernmx.cern.ch
+TLS_CA_File=/etc/pki/tls/certs/ca-bundle.crt
+FromLineOverride=YES