Merge branch 'restore-script' into 'master'

Use oc rsync to restore files

See merge request !11

chart/drupal-operations/templates/clear-cache.yaml

@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
 kind: ClusterTask
 metadata:
   name: clear-cache
-  annotations:
+  labels:
     app: drupal
+  annotations:
+    argocd.argoproj.io/compare-options: IgnoreExtraneous
 spec:
   params:
     - name: drupalSite

chart/drupal-operations/templates/database-restore.yaml

@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
 kind: ClusterTask
 metadata:
   name: database-restore
-  annotations:
+  labels:
     app: drupal
+  annotations:
+    argocd.argoproj.io/compare-options: IgnoreExtraneous
 spec:
   params:
     - name: drupalSite

chart/drupal-operations/templates/drupalsite-backup.yaml

@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
 kind: ClusterTask
 metadata:
   name: drupalsite-backup
-  annotations:
+  labels:
     app: drupal
+  annotations:
+    argocd.argoproj.io/compare-options: IgnoreExtraneous
 spec:
   params:
     - name: drupalSite

chart/drupal-operations/templates/drupalsite-restore.yaml

@@ -2,8 +2,10 @@ apiVersion: tekton.dev/v1beta1
 kind: ClusterTask
 metadata:
   name: drupalsite-restore
-  annotations:
+  labels:
     app: drupal
+  annotations:
+    argocd.argoproj.io/compare-options: IgnoreExtraneous
 spec:
   params:
     - name: drupalSite

examples/clear-cache-taskrun.yaml

@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
 kind: TaskRun
 metadata:
   generateName: clear-cache-
-  ownerReferences:
-    - apiVersion: tekton.dev/v1beta1
-      blockOwnerDeletion: true
-      controller: true
-      kind: Pipeline
-      name: build-test
-      uid: my-uid
 spec:
   taskRef:
     name: clear-cache

examples/database-restore-taskrun.yaml

@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
 kind: TaskRun
 metadata:
   generateName: database-restore-
-  ownerReferences:
-    - apiVersion: tekton.dev/v1beta1
-      blockOwnerDeletion: true
-      controller: true
-      kind: Pipeline
-      name: build-test
-      uid: my-uid
 spec:
   taskRef:
     name: database-restore

examples/drupalsite-backup-taskrun.yaml

@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
 kind: TaskRun
 metadata:
   generateName: drupalsite-backup-
-  ownerReferences:
-    - apiVersion: tekton.dev/v1beta1
-      blockOwnerDeletion: true
-      controller: true
-      kind: ClusterTask
-      name: build-test
-      uid: my-uid
 spec:
   taskRef:
     name: drupalsite-backup

examples/drupalsite-restore-taskrun.yaml

@@ -2,13 +2,6 @@ apiVersion: tekton.dev/v1beta1
 kind: TaskRun
 metadata:
   generateName: drupalsite-restore-
-  ownerReferences:
-    - apiVersion: tekton.dev/v1beta1
-      blockOwnerDeletion: true
-      controller: true
-      kind: ClusterTask
-      name: build-test
-      uid: my-uid
 spec:
   taskRef:
     name: drupalsite-restore

tekton-task-templates/restore_pv_job.yaml

@@ -15,10 +15,6 @@ spec:
         job-name: $TASK-$DRUPALSITE
     spec:
       volumes:
-        - name: openstack-credentials
-          secret:
-            secretName: openstack-auth-secrets
-            defaultMode: 420
         - name: velero-restic-credentials
           secret:
             secretName: velero-restic-credentials
@@ -31,8 +27,10 @@ spec:
           command:
             - /scripts/restore_pvs.sh
           env:
-            - name: OS_CLOUD
-              value: openstack
+            - name: DRUPALSITE
+              value: $DRUPALSITE
+            - name: NAMESPACE
+              value: $NAMESPACE
             - name: VELERO_NAMESPACE
               value: $VELERO_NAMESPACE
             - name: VELERO_BACKUP_NAME
@@ -51,19 +49,9 @@ spec:
                   key: cephfsBackupS3SecretKey
           resources: {}
           volumeMounts:
-            - name: openstack-credentials
-              readOnly: true
-              mountPath: /etc/openstack/
             - name: velero-restic-credentials
               readOnly: true
               mountPath: /tmp
-          lifecycle:
-            preStop:
-              exec:
-                command:
-                  - /bin/sh
-                  - '-c'
-                  - umount /mnt; sleep 10
           terminationMessagePath: /dev/termination-log
           terminationMessagePolicy: File
           imagePullPolicy: Always

velero-restic-restore/restore_pvs.sh

@@ -13,16 +13,6 @@ validateVar(){
 set -e
 
 
-# Contact the OpenStack manila API to retrieve information about each of the manila shares
-# We need this to be able to mount PVs for backup
-# See https://clouddocs.web.cern.ch/file_shares/programmatic_access.html
-MANILA_URL=$(openstack catalog show manilav2 -f json | jq -r '.endpoints[] | select(.interface == "public") | .url')
-validateVar "$MANILA_URL" "MANILA_URL"
-
-# OpenStack token issues will expire after 24h, so we can create several tokens per day
-OPENSTACK_MANILA_SECRET=$(openstack token issue -f json | jq -r '.id')
-validateVar "$OPENSTACK_MANILA_SECRET" "OPENSTACK_MANILA_SECRET"
-
 # Use the velero backup name, to fetch the PodVolumeBackup resource and the ID of the restic snapshot, PV name accordingly
 # List the PodVolumeBackup resources by label filtering
 POD_VOLUME_BACKUP=`oc get PodVolumeBackup -n "$VELERO_NAMESPACE" -l velero.io/backup-name="$VELERO_BACKUP_NAME" -o name`
@@ -42,60 +32,17 @@ RESTIC_REPO=$(echo "$POD_VOLUME_BACKUP_JSON" | jq -r '.spec.repoIdentifier')
 validateVar "$RESTIC_REPO" "RESTIC_REPO"
 echo "$RESTIC_REPO"
 
-# Fetch the PV name
-PV_NAME=pvc-$(echo "$POD_VOLUME_BACKUP_JSON" | jq -r '.spec.tags["pvc-uid"]')
-validateVar "$PV_NAME" "PV_NAME"
-echo "$PV_NAME"
-
-PV_JSON=`oc get pv "$PV_NAME" -o json`
-validateVar "$PV_JSON" "PV_JSON"
-
-NAMESPACE_CSI_DRIVER=$(echo "$PV_JSON" | jq -r '.spec.csi.nodeStageSecretRef.namespace')
-validateVar "$NAMESPACE_CSI_DRIVER" "NAMESPACE_CSI_DRIVER"
-echo "$NAMESPACE_CSI_DRIVER"
-
-# We need this information to access the manila API
-MANILA_SHARE_ID=$(echo "$PV_JSON" | jq -r '.spec.csi.volumeAttributes.shareID')
-validateVar "$MANILA_SHARE_ID" "MANILA_SHARE_ID"
-echo "$MANILA_SHARE_ID"
-
-MANILA_SHARE_ACCESS_ID=$(echo "$PV_JSON" | jq -r '.spec.csi.volumeAttributes.shareAccessID')
-validateVar "$MANILA_SHARE_ACCESS_ID" "MANILA_SHARE_ACCESS_ID"
-echo "$MANILA_SHARE_ACCESS_ID"
-
-MANILA_EXPORT_LOCATIONS=$(curl -X GET -H "X-Auth-Token: $OPENSTACK_MANILA_SECRET" -H "X-Openstack-Manila-Api-Version: 2.51" "$MANILA_URL/shares/$MANILA_SHARE_ID/export_locations")
-validateVar "$MANILA_EXPORT_LOCATIONS" "MANILA_EXPORT_LOCATIONS"
-echo "$MANILA_EXPORT_LOCATIONS"
-
-# Stores monitors and path of the PV, similar to
-# 137.138.121.135:6789,188.184.85.133:6789,188.184.91.157:6789:/volumes/_nogroup/337f5361-bee2-415b-af8e-53eaec1add43
-CEPHFS_PATH_PV=$(echo "$MANILA_EXPORT_LOCATIONS" | jq -r '.export_locations[]?.path')
-validateVar "$CEPHFS_PATH_PV" "CEPHFS_PATH_PV"
-echo "$CEPHFS_PATH_PV"
-
-# Stores the userKey credentials needed to manually mount CephFS PVs
-MANILA_ACCESS_RULES=$(curl -X GET -H "X-Auth-Token: $OPENSTACK_MANILA_SECRET" -H "X-Openstack-Manila-Api-Version: 2.51" "$MANILA_URL/share-access-rules/$MANILA_SHARE_ACCESS_ID")
-validateVar "$MANILA_ACCESS_RULES" "MANILA_ACCESS_RULES"
-echo "$MANILA_ACCESS_RULES"
-
-CEPHFS_USERKEY=$(echo "$MANILA_ACCESS_RULES" | jq -r '.access.access_key')
-validateVar "$CEPHFS_USERKEY" "CEPHFS_USERKEY"
-echo "$CEPHFS_USERKEY"
-
-echo "mounting $PV_NAME in /mnt JOB_UID: $JOB_UID ..."
-mount -t ceph "$CEPHFS_PATH_PV" -o name="$PV_NAME",noatime,secret="$CEPHFS_USERKEY" /mnt
-
 # The target directory for restic restore needs to have the same permissions as '/drupal-data' for rsync later
 mkdir -p /restore
 chmod 777 /restore
 
 restic -p /tmp/repository-password -r "$RESTIC_REPO" restore "$RESTIC_SNAPSHOT_ID" --target /restore
 
-rsync -avz /restore/ /mnt/ --delete
+# Fetch the pod name from the drupalSite env var
+POD_NAME=`oc get pods -l app=drupal,drupalSite="$DRUPALSITE" -o name -n "$NAMESPACE" | sed "s/pod\///g" | head -n 1`
+validateVar "$POD_NAME" "POD_NAME"
 
-# Unmount pv from /mnt earlier mounted
-echo "unmounting $PV_NAME from /mnt JOB_UID: $JOB_UID ..."
-umount /mnt
+oc rsync /restore/ "$POD_NAME":/drupal-data  --delete -n "$NAMESPACE" -c php-fpm
 
 # We remove /root/.cache/ in each iteration to prevent restic backups to run out of memory and fail the cronjobs we run,
 # as we detected this malfunction in our infra.