Add uploading OVN backups to Ceph Swift API gateway

docs/infrastructure-ovn-setup.md

@@ -98,3 +98,11 @@ kubectl apply -k /opt/genestack/kustomize/ovn
 
 After running the setup, nodes will have the label `ovn.openstack.org/configured` with a date stamp when it was configured.
 If there's ever a need to reconfigure a node, simply remove the label and the DaemonSet will take care of it automatically.
+
+!!! note
+
+    To upload backups to a Ceph Swift API gateway, edit ovn-backup.config to set
+    `SWIFT_UPLOAD' "true"`, edit the other related options appropriately (i.e.,
+    set the SWIFT_BASE_URL and CONTAINER) and put the username and secret key of
+    the account to use in `swift-account.env` before running `kubectl apply` an
+    indicated above.

kustomize/ovn/kustomization.yaml

@@ -1,3 +1,17 @@
+secretGenerator:
+  - name: ovn-backup-swift-account
+    namespace: kube-system
+    envs:
+      - swift-account.env
+configMapGenerator:
+  - name: ovn-backup-script
+    namespace: kube-system
+    files:
+    - ovn-backup.sh
+  - name: ovn-backup-config
+    namespace: kube-system
+    envs:
+    - ovn-backup.config
 resources:
   - ovn-setup.yaml
   - ovn-backup.yaml

kustomize/ovn/ovn-backup.config

@@ -0,0 +1,16 @@
+RETENTION_DAYS=30
+BACKUP_DIR=/backup
+# You probably want to place this on the PersistentVolume so that it doesn't get
+# deleted with the pod by cron, or use /dev/null if you don't care. You can use
+# `kubectl logs` since log messages also go to STDOUT.
+LOG_FILE=/backup/upload.log
+LOG_LEVEL=INFO
+
+# From here forward, variables for uploading to a Ceph Swift interface.
+SWIFT_UPLOAD=false
+SWIFT_BASE_URL=http://FIX_ME:8081
+
+# Nothing after this line makes any difference unless you used
+# SWIFT_UPLOAD: "true"
+# above.
+CONTAINER=test-ovn-backup

kustomize/ovn/ovn-backup.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+if [[ "$LOG_LEVEL" == "DEBUG" ]]
+then
+    set -x
+fi
+
+SWIFT_CONTAINER_BASE_URL="$SWIFT_BASE_URL/swift/v1"
+export SWIFT_CONTAINER_BASE_URL
+
+log_level() {
+    local LEVEL="$1"
+    case "$LEVEL" in
+      DEBUG)
+        echo 5
+        ;;
+      INFO)
+        echo 4
+        ;;
+      WARNING)
+        echo 3
+        ;;
+      ERROR)
+        echo 2
+        ;;
+      CRITICAL)
+        echo 1
+        ;;
+      *)
+        exit 3
+        ;;
+    esac
+}
+export -f log_level
+
+log_line() {
+    local LEVEL
+    LEVEL="$(log_level "$1")"
+    if [[ "$LEVEL" -ge "$LOG_LEVEL" ]]
+    then
+        local line
+        line=$(date +"%b %d %H:%M:%S $*")
+        echo "$line" | tee -a "$LOG_FILE"
+    fi
+}
+export -f log_line # exported for upload_file
+
+# Delete old backup files on volume.
+cd "$BACKUP_DIR" || exit 2
+find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete;
+
+# Make a backup in YYYY/MM/DD directory in $BACKUP_DIR
+YMD="$(date +"%Y/%m/%d")"
+mkdir -p "$YMD" && cd "$YMD" || exit 2 # kubectl-ko creates backups in $PWD, so we cd first.
+/kube-ovn/kubectl-ko nb backup || log_line ERROR "nb backup failed"
+/kube-ovn/kubectl-ko sb backup || log_line ERROR "sb backup failed"
+
+if [[ "$SWIFT_UPLOAD" != "true" ]]
+then
+    exit 0
+fi
+
+# Everything from here forward deals with uploading to Rackspace OSPCv1 Cloud
+# Files.
+
+cd "$BACKUP_DIR" || exit 2
+CURL="$(which curl)"
+export CONTAINER CURL # these need to reach the subshell below used with `find`
+HEADER_TEMP_FILE=$(mktemp /tmp/headers.XXXXXXXX)
+$CURL -sS -D "$HEADER_TEMP_FILE" -H "X-Auth-User: $USERNAME" -H "X-Auth-Key: $SECRET_KEY" "$SWIFT_BASE_URL/auth/v1.1"
+sed -i -e 's/\r//g' "$HEADER_TEMP_FILE" # strip carriage returns
+token=$(awk '/X-Auth-Token/ { print $2 }' "$HEADER_TEMP_FILE")
+rm "$HEADER_TEMP_FILE"
+export token
+
+# wrap curl with some things we will always use
+curl_wrap() {
+    $CURL -sS -H "X-Auth-Token: $token" "$@"
+}
+export -f curl_wrap
+
+# Create the container if it doesn't exist
+# TODO fixme
+check_container=$(curl_wrap -o /dev/null -w "%{http_code}" "$SWIFT_CONTAINER_BASE_URL/$CONTAINER")
+if ! [[ "$check_container" =~ 20[0-9] ]]
+then
+  curl_wrap -X PUT "$SWIFT_CONTAINER_BASE_URL/$CONTAINER"
+fi
+
+# upload_file uploads $1 to the CF container
+upload_file() {
+    FILE="$1"
+    local curl_return
+    curl_return=$(curl_wrap -w "%{http_code}" \
+      -X PUT "${SWIFT_CONTAINER_BASE_URL}/${CONTAINER}/$FILE" -T "$FILE")
+    if [[ "$curl_return" == "201" ]]
+    then
+      log_line INFO "SUCCESSFUL UPLOAD $FILE"
+    else
+      log_line ERROR "FAILURE API returned $curl_return uploading $FILE (expected 201)"
+    fi
+}
+export -f upload_file
+
+# find created backups and upload them
+cd "$BACKUP_DIR" || exit 2
+# unusual find syntax to use an exported function from the shell
+find "$YMD" -type f -exec bash -c 'upload_file "$0"' {} \;

kustomize/ovn/ovn-backup.yaml

@@ -1,9 +1,11 @@
-# This writes OVN NB and SB snapshots to a persistent volume, assuming you
-# installed OVN with kubespray, since it assumes resources exist as seen in the
+# The resources contained in this file (ovn-backup.yaml) write OVN NB and SB
+# snapshots to a persistent volume, assuming you installed OVN with kubespray,
+# since it assumes resources exist as seen in the
 # genestack/submodules/kubespray/roles/network_plugin/kube-ovn/templates
 # directory, assuming you have checked out the genestack submodules.
 # (For instance, it uses the `ovn` service account as seen in
 # genestack/submodules/kubespray/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
+---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -38,20 +40,22 @@ spec:
           - name: backup
             persistentVolumeClaim:
               claimName: ovndb-backup
+          - name: backup-script
+            configMap:
+              name: ovn-backup-script
+              defaultMode: 0744
           containers:
           - name: ovn-central-backup
-            env:
-            - name: RETENTION_DAYS
-              value: "30"
-            command: ["/bin/sh", "-c"]
-            args:
-            - >
-              find /backup -ctime +$RETENTION_DAYS -delete;
-              /kube-ovn/kubectl-ko nb backup;
-              /kube-ovn/kubectl-ko sb backup;
-              mv /kube-ovn/ovn*db*.backup /backup;
+            envFrom:
+              - configMapRef:
+                  name: ovn-backup-config
+              - secretRef:
+                  name: ovn-backup-swift-account
+            command: ["/backup-script/ovn-backup.sh"]
             image: docker.io/kubeovn/kube-ovn:v1.11.5
             imagePullPolicy: IfNotPresent
             volumeMounts:
             - name: backup
               mountPath: "/backup"
+            - name: backup-script
+              mountPath: /backup-script

kustomize/ovn/swift-account.env

@@ -0,0 +1,2 @@
+USERNAME=username
+SECRET_KEY=secret_key