From 3e271378420a5b065a6b5a7cb81c235064985781 Mon Sep 17 00:00:00 2001 From: Dat WMDE Date: Tue, 3 Sep 2024 13:58:25 +0200 Subject: [PATCH] Backup job script for single wiki (#1766) Bug: T365443 Co-authored-by: Deniz Erdogan Co-authored-by: Thomas Arrow --- k8s/jobs/README.md | 18 ++++++++++++- k8s/jobs/singleWikiBackup.sh | 16 +++++++++++ k8s/jobs/singleWikiBackup.yaml | 38 ++++++++++++++++++++++++++ k8s/jobs/singleWikiBackupPvc.yaml | 12 +++++++++ k8s/jobs/singleWikiRestore.sh | 12 +++++++++ k8s/jobs/singleWikiRestore.yaml | 44 +++++++++++++++++++++++++++++++ 6 files changed, 139 insertions(+), 1 deletion(-) create mode 100755 k8s/jobs/singleWikiBackup.sh create mode 100644 k8s/jobs/singleWikiBackup.yaml create mode 100644 k8s/jobs/singleWikiBackupPvc.yaml create mode 100755 k8s/jobs/singleWikiRestore.sh create mode 100644 k8s/jobs/singleWikiRestore.yaml diff --git a/k8s/jobs/README.md b/k8s/jobs/README.md index 7a33e6e32..7b3275965 100644 --- a/k8s/jobs/README.md +++ b/k8s/jobs/README.md @@ -59,4 +59,20 @@ Should be submitted to the correct cluster using `kubectl create -f changeReplic Should be run immediately after running resetOtherSqlSecretsJob.yaml This updates the replication password by logging into the secondary pod (not just the service which may already have become unavailable due to replication lag) using the root password. -This job will need updating if there is more than one replica server to add each additional replica server. \ No newline at end of file +This job will need updating if there is more than one replica server to add each additional replica server. + +## singleWikiBackup.sh +Uses the ENV `DATABASE_NAME` + +This creates a PVC to store the backups of single wiki databases. +It then creates a job which uses mysqldump to dump the db specified by `DATABASE_NAME` to .sql files which it stores +in the above PVC. +It takes the dump from the primary sql replica to try and ensure consistency. +Running this job repeatedly will overwrite the previous backup. +This PVC is not automatically deleted so care should be taken to remove it after use + +## singleWikiRestore.sh +Uses the ENV `DATABASE_NAME` + +This works in conjunction with `singleWikiBackup.sh` to restore a database from this temporary backup. +It uses the primary replica to write to. diff --git a/k8s/jobs/singleWikiBackup.sh b/k8s/jobs/singleWikiBackup.sh new file mode 100755 index 000000000..d35586bb6 --- /dev/null +++ b/k8s/jobs/singleWikiBackup.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# DATABASE_NAME should be set to the database to backup + +if [[ -z "$DATABASE_NAME" ]]; then + echo "DATABASE_NAME not set" + exit 1 +fi + +# This creates the PVC (if it doesn't exist) for backups to be saved to +# It will not be automatically deleted so you may need to clean this up when you are done +kubectl apply -f singleWikiBackupPvc.yaml + +kubectl create -f singleWikiBackup.yaml -o=json --dry-run=client |\ +jq ".spec.template.spec.containers[0].env += [{\"name\": \"DATABASE_NAME\", \"value\": \"${DATABASE_NAME}\"}]" |\ +kubectl create -f - diff --git a/k8s/jobs/singleWikiBackup.yaml b/k8s/jobs/singleWikiBackup.yaml new file mode 100644 index 000000000..c9957ec38 --- /dev/null +++ b/k8s/jobs/singleWikiBackup.yaml @@ -0,0 +1,38 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: backup-single-wiki- + namespace: adhoc-jobs +spec: + ttlSecondsAfterFinished: 604800 #7days + template: + metadata: + name: backup-single-wiki + spec: + volumes: + - name: temporary-backup-pvc + persistentVolumeClaim: + claimName: temporary-backup-pvc + containers: + - name: backup-single-wiki + env: + - name: MYSQL_PWD + valueFrom: + secretKeyRef: + name: sql-secrets-passwords + key: mariadb-root-password + image: mariadb:10.5 + + command: + - 'bash' + - '-c' + - | + mysqldump --verbose \ + --host sql-mariadb-primary.default.svc.cluster.local \ + ${DATABASE_NAME} \ + > /backup/${DATABASE_NAME}.sql + volumeMounts: + - name: temporary-backup-pvc + mountPath: "/backup/" + + restartPolicy: OnFailure diff --git a/k8s/jobs/singleWikiBackupPvc.yaml b/k8s/jobs/singleWikiBackupPvc.yaml new file mode 100644 index 000000000..769a51181 --- /dev/null +++ b/k8s/jobs/singleWikiBackupPvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: temporary-backup-pvc + namespace: adhoc-jobs +spec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 3Gi diff --git a/k8s/jobs/singleWikiRestore.sh b/k8s/jobs/singleWikiRestore.sh new file mode 100755 index 000000000..7b0092a30 --- /dev/null +++ b/k8s/jobs/singleWikiRestore.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# DATABASE_NAME should be set to the database to restore + +if [[ -z "$DATABASE_NAME" ]]; then + echo "DATABASE_NAME not set" + exit 1 +fi + +kubectl create -f singleWikiRestore.yaml -o=json --dry-run=client |\ +jq ".spec.template.spec.containers[0].env += [{\"name\": \"DATABASE_NAME\", \"value\": \"${DATABASE_NAME}\"}]" |\ +kubectl create -f - diff --git a/k8s/jobs/singleWikiRestore.yaml b/k8s/jobs/singleWikiRestore.yaml new file mode 100644 index 000000000..cb443a4cc --- /dev/null +++ b/k8s/jobs/singleWikiRestore.yaml @@ -0,0 +1,44 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: restore-single-wiki- + namespace: adhoc-jobs +spec: + ttlSecondsAfterFinished: 604800 #7days + template: + metadata: + name: restore-single-wiki + spec: + volumes: + - name: temporary-backup-pvc + persistentVolumeClaim: + claimName: temporary-backup-pvc + containers: + - name: restore-single-wiki + env: + - name: MYSQL_PWD + valueFrom: + secretKeyRef: + name: sql-secrets-passwords + key: mariadb-root-password + image: mariadb:10.5 + + command: + - 'bash' + - '-c' + - | + mysql --show-warnings \ + --host sql-mariadb-primary.default.svc.cluster.local \ + ${DATABASE_NAME} \ + < /backup/${DATABASE_NAME}.sql + + if [[ $? == 0 ]]; then + echo "SQL import successful" + else + echo "Error while importing SQL" + fi + volumeMounts: + - name: temporary-backup-pvc + mountPath: "/backup/" + + restartPolicy: OnFailure