Skip to content
Snippets Groups Projects
Commit eda88a3a authored by Konstantinos Samaras-Tsakiris's avatar Konstantinos Samaras-Tsakiris
Browse files

Merge branch 'restore-script' into 'master'

Use oc rsync to restore files

See merge request !11
parents 23f1b083 af588e78
Branches
No related tags found
1 merge request!11Use oc rsync to restore files
Pipeline #2874478 passed
...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
kind: ClusterTask kind: ClusterTask
metadata: metadata:
name: clear-cache name: clear-cache
annotations: labels:
app: drupal app: drupal
annotations:
argocd.argoproj.io/compare-options: IgnoreExtraneous
spec: spec:
params: params:
- name: drupalSite - name: drupalSite
......
...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
kind: ClusterTask kind: ClusterTask
metadata: metadata:
name: database-restore name: database-restore
annotations: labels:
app: drupal app: drupal
annotations:
argocd.argoproj.io/compare-options: IgnoreExtraneous
spec: spec:
params: params:
- name: drupalSite - name: drupalSite
......
...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
kind: ClusterTask kind: ClusterTask
metadata: metadata:
name: drupalsite-backup name: drupalsite-backup
annotations: labels:
app: drupal app: drupal
annotations:
argocd.argoproj.io/compare-options: IgnoreExtraneous
spec: spec:
params: params:
- name: drupalSite - name: drupalSite
......
...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
kind: ClusterTask kind: ClusterTask
metadata: metadata:
name: drupalsite-restore name: drupalsite-restore
annotations: labels:
app: drupal app: drupal
annotations:
argocd.argoproj.io/compare-options: IgnoreExtraneous
spec: spec:
params: params:
- name: drupalSite - name: drupalSite
......
...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
kind: TaskRun kind: TaskRun
metadata: metadata:
generateName: clear-cache- generateName: clear-cache-
ownerReferences:
- apiVersion: tekton.dev/v1beta1
blockOwnerDeletion: true
controller: true
kind: Pipeline
name: build-test
uid: my-uid
spec: spec:
taskRef: taskRef:
name: clear-cache name: clear-cache
......
...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
kind: TaskRun kind: TaskRun
metadata: metadata:
generateName: database-restore- generateName: database-restore-
ownerReferences:
- apiVersion: tekton.dev/v1beta1
blockOwnerDeletion: true
controller: true
kind: Pipeline
name: build-test
uid: my-uid
spec: spec:
taskRef: taskRef:
name: database-restore name: database-restore
......
...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
kind: TaskRun kind: TaskRun
metadata: metadata:
generateName: drupalsite-backup- generateName: drupalsite-backup-
ownerReferences:
- apiVersion: tekton.dev/v1beta1
blockOwnerDeletion: true
controller: true
kind: ClusterTask
name: build-test
uid: my-uid
spec: spec:
taskRef: taskRef:
name: drupalsite-backup name: drupalsite-backup
......
...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1 ...@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
kind: TaskRun kind: TaskRun
metadata: metadata:
generateName: drupalsite-restore- generateName: drupalsite-restore-
ownerReferences:
- apiVersion: tekton.dev/v1beta1
blockOwnerDeletion: true
controller: true
kind: ClusterTask
name: build-test
uid: my-uid
spec: spec:
taskRef: taskRef:
name: drupalsite-restore name: drupalsite-restore
......
...@@ -15,10 +15,6 @@ spec: ...@@ -15,10 +15,6 @@ spec:
job-name: $TASK-$DRUPALSITE job-name: $TASK-$DRUPALSITE
spec: spec:
volumes: volumes:
- name: openstack-credentials
secret:
secretName: openstack-auth-secrets
defaultMode: 420
- name: velero-restic-credentials - name: velero-restic-credentials
secret: secret:
secretName: velero-restic-credentials secretName: velero-restic-credentials
...@@ -31,8 +27,10 @@ spec: ...@@ -31,8 +27,10 @@ spec:
command: command:
- /scripts/restore_pvs.sh - /scripts/restore_pvs.sh
env: env:
- name: OS_CLOUD - name: DRUPALSITE
value: openstack value: $DRUPALSITE
- name: NAMESPACE
value: $NAMESPACE
- name: VELERO_NAMESPACE - name: VELERO_NAMESPACE
value: $VELERO_NAMESPACE value: $VELERO_NAMESPACE
- name: VELERO_BACKUP_NAME - name: VELERO_BACKUP_NAME
...@@ -51,19 +49,9 @@ spec: ...@@ -51,19 +49,9 @@ spec:
key: cephfsBackupS3SecretKey key: cephfsBackupS3SecretKey
resources: {} resources: {}
volumeMounts: volumeMounts:
- name: openstack-credentials
readOnly: true
mountPath: /etc/openstack/
- name: velero-restic-credentials - name: velero-restic-credentials
readOnly: true readOnly: true
mountPath: /tmp mountPath: /tmp
lifecycle:
preStop:
exec:
command:
- /bin/sh
- '-c'
- umount /mnt; sleep 10
terminationMessagePath: /dev/termination-log terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File terminationMessagePolicy: File
imagePullPolicy: Always imagePullPolicy: Always
......
...@@ -13,16 +13,6 @@ validateVar(){ ...@@ -13,16 +13,6 @@ validateVar(){
set -e set -e
# Contact the OpenStack manila API to retrieve information about each of the manila shares
# We need this to be able to mount PVs for backup
# See https://clouddocs.web.cern.ch/file_shares/programmatic_access.html
MANILA_URL=$(openstack catalog show manilav2 -f json | jq -r '.endpoints[] | select(.interface == "public") | .url')
validateVar "$MANILA_URL" "MANILA_URL"
# OpenStack token issues will expire after 24h, so we can create several tokens per day
OPENSTACK_MANILA_SECRET=$(openstack token issue -f json | jq -r '.id')
validateVar "$OPENSTACK_MANILA_SECRET" "OPENSTACK_MANILA_SECRET"
# Use the velero backup name, to fetch the PodVolumeBackup resource and the ID of the restic snapshot, PV name accordingly # Use the velero backup name, to fetch the PodVolumeBackup resource and the ID of the restic snapshot, PV name accordingly
# List the PodVolumeBackup resources by label filtering # List the PodVolumeBackup resources by label filtering
POD_VOLUME_BACKUP=`oc get PodVolumeBackup -n "$VELERO_NAMESPACE" -l velero.io/backup-name="$VELERO_BACKUP_NAME" -o name` POD_VOLUME_BACKUP=`oc get PodVolumeBackup -n "$VELERO_NAMESPACE" -l velero.io/backup-name="$VELERO_BACKUP_NAME" -o name`
...@@ -42,60 +32,17 @@ RESTIC_REPO=$(echo "$POD_VOLUME_BACKUP_JSON" | jq -r '.spec.repoIdentifier') ...@@ -42,60 +32,17 @@ RESTIC_REPO=$(echo "$POD_VOLUME_BACKUP_JSON" | jq -r '.spec.repoIdentifier')
validateVar "$RESTIC_REPO" "RESTIC_REPO" validateVar "$RESTIC_REPO" "RESTIC_REPO"
echo "$RESTIC_REPO" echo "$RESTIC_REPO"
# Fetch the PV name
PV_NAME=pvc-$(echo "$POD_VOLUME_BACKUP_JSON" | jq -r '.spec.tags["pvc-uid"]')
validateVar "$PV_NAME" "PV_NAME"
echo "$PV_NAME"
PV_JSON=`oc get pv "$PV_NAME" -o json`
validateVar "$PV_JSON" "PV_JSON"
NAMESPACE_CSI_DRIVER=$(echo "$PV_JSON" | jq -r '.spec.csi.nodeStageSecretRef.namespace')
validateVar "$NAMESPACE_CSI_DRIVER" "NAMESPACE_CSI_DRIVER"
echo "$NAMESPACE_CSI_DRIVER"
# We need this information to access the manila API
MANILA_SHARE_ID=$(echo "$PV_JSON" | jq -r '.spec.csi.volumeAttributes.shareID')
validateVar "$MANILA_SHARE_ID" "MANILA_SHARE_ID"
echo "$MANILA_SHARE_ID"
MANILA_SHARE_ACCESS_ID=$(echo "$PV_JSON" | jq -r '.spec.csi.volumeAttributes.shareAccessID')
validateVar "$MANILA_SHARE_ACCESS_ID" "MANILA_SHARE_ACCESS_ID"
echo "$MANILA_SHARE_ACCESS_ID"
MANILA_EXPORT_LOCATIONS=$(curl -X GET -H "X-Auth-Token: $OPENSTACK_MANILA_SECRET" -H "X-Openstack-Manila-Api-Version: 2.51" "$MANILA_URL/shares/$MANILA_SHARE_ID/export_locations")
validateVar "$MANILA_EXPORT_LOCATIONS" "MANILA_EXPORT_LOCATIONS"
echo "$MANILA_EXPORT_LOCATIONS"
# Stores monitors and path of the PV, similar to
# 137.138.121.135:6789,188.184.85.133:6789,188.184.91.157:6789:/volumes/_nogroup/337f5361-bee2-415b-af8e-53eaec1add43
CEPHFS_PATH_PV=$(echo "$MANILA_EXPORT_LOCATIONS" | jq -r '.export_locations[]?.path')
validateVar "$CEPHFS_PATH_PV" "CEPHFS_PATH_PV"
echo "$CEPHFS_PATH_PV"
# Stores the userKey credentials needed to manually mount CephFS PVs
MANILA_ACCESS_RULES=$(curl -X GET -H "X-Auth-Token: $OPENSTACK_MANILA_SECRET" -H "X-Openstack-Manila-Api-Version: 2.51" "$MANILA_URL/share-access-rules/$MANILA_SHARE_ACCESS_ID")
validateVar "$MANILA_ACCESS_RULES" "MANILA_ACCESS_RULES"
echo "$MANILA_ACCESS_RULES"
CEPHFS_USERKEY=$(echo "$MANILA_ACCESS_RULES" | jq -r '.access.access_key')
validateVar "$CEPHFS_USERKEY" "CEPHFS_USERKEY"
echo "$CEPHFS_USERKEY"
echo "mounting $PV_NAME in /mnt JOB_UID: $JOB_UID ..."
mount -t ceph "$CEPHFS_PATH_PV" -o name="$PV_NAME",noatime,secret="$CEPHFS_USERKEY" /mnt
# The target directory for restic restore needs to have the same permissions as '/drupal-data' for rsync later # The target directory for restic restore needs to have the same permissions as '/drupal-data' for rsync later
mkdir -p /restore mkdir -p /restore
chmod 777 /restore chmod 777 /restore
restic -p /tmp/repository-password -r "$RESTIC_REPO" restore "$RESTIC_SNAPSHOT_ID" --target /restore restic -p /tmp/repository-password -r "$RESTIC_REPO" restore "$RESTIC_SNAPSHOT_ID" --target /restore
rsync -avz /restore/ /mnt/ --delete # Fetch the pod name from the drupalSite env var
POD_NAME=`oc get pods -l app=drupal,drupalSite="$DRUPALSITE" -o name -n "$NAMESPACE" | sed "s/pod\///g" | head -n 1`
validateVar "$POD_NAME" "POD_NAME"
# Unmount pv from /mnt earlier mounted oc rsync /restore/ "$POD_NAME":/drupal-data --delete -n "$NAMESPACE" -c php-fpm
echo "unmounting $PV_NAME from /mnt JOB_UID: $JOB_UID ..."
umount /mnt
# We remove /root/.cache/ in each iteration to prevent restic backups to run out of memory and fail the cronjobs we run, # We remove /root/.cache/ in each iteration to prevent restic backups to run out of memory and fail the cronjobs we run,
# as we detected this malfunction in our infra. # as we detected this malfunction in our infra.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment