Add uploading OVN backups to Ceph Swift API gateway

docs/infrastructure-ovn-setup.md

@@ -98,3 +98,11 @@ kubectl apply -k /opt/genestack/kustomize/ovn
 
 After running the setup, nodes will have the label `ovn.openstack.org/configured` with a date stamp when it was configured.
 If there's ever a need to reconfigure a node, simply remove the label and the DaemonSet will take care of it automatically.
+
+!!! note
+
+    To upload backups to OSPCv1 Cloud Files, edit ovn-backup-config-patch.yaml
+    to set `UPLOAD_OSPC_V1: "true"`, edit the other related options
+    appropriately (for instance, select a Rackspace OSPCv1 Cloud Files region)
+    and put the username and API key of the account to use in
+    `ospcv1-account.env` before running `kubectl apply` an indicated above.

kustomize/ovn/kustomization.yaml

@@ -1,3 +1,10 @@
+secretGenerator:
+  - name: ovn-backup-ospcv1-account
+    namespace: kube-system
+    envs:
+      - ospcv1-account.env
 resources:
   - ovn-setup.yaml
   - ovn-backup.yaml
+patches:
+  - path: ovn-backup-config-patch.yaml

kustomize/ovn/ospcv1-account.env

@@ -0,0 +1,2 @@
+USERNAME=ospcv1_username
+API_KEY=ospcv1_api_key

kustomize/ovn/ovn-backup-config-patch.yaml

@@ -0,0 +1,25 @@
+# This ConfigMap contains variables controlling the run of the OVN DB backups.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ovn-backup-config
+  namespace: kube-system
+data:
+  RETENTION_DAYS: "30"
+  BACKUP_DIR: "/backup"
+  # You probably want to place this on the PersistentVolume so that it doesn't
+  # get deleted with the pod by cron, or use /dev/null if you don't care.
+  # You can use `kubectl logs` since log messages also go to STDOUT.
+  LOG_FILE: "/backup/upload.log"
+
+  # From here forward, variables for uploading to Rackspace OSPCv1 Cloud Files
+  UPLOAD_OSPC_V1: "false"
+  # Nothing after this line makes any difference unless you used
+  # UPLOAD_OSPC_V1: "true"
+  # above.
+  CF_REGION: "dfw" # OSPCv1 Cloud Files region to use
+  # You should probably include the name of the environment in CONTAINER
+  CONTAINER: "test-ovn-backup"
+  # Standard log levels DEBUG, INFO, WARNING, ERROR, CRITICAL
+  # DEBUG also does `set -x` toward the beginning of the backup script.
+  LOG_LEVEL: INFO

kustomize/ovn/ovn-backup.yaml

@@ -1,9 +1,11 @@
-# This writes OVN NB and SB snapshots to a persistent volume, assuming you
-# installed OVN with kubespray, since it assumes resources exist as seen in the
+# The resources contained in this file (ovn-backup.yaml) write OVN NB and SB
+# snapshots to a persistent volume, assuming you installed OVN with kubespray,
+# since it assumes resources exist as seen in the
 # genestack/submodules/kubespray/roles/network_plugin/kube-ovn/templates
 # directory, assuming you have checked out the genestack submodules.
 # (For instance, it uses the `ovn` service account as seen in
 # genestack/submodules/kubespray/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
+---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -17,6 +19,154 @@ spec:
       storage: 1Gi
   storageClassName: general
 ---
+# The script that runs the backup and uploads the file.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ovn-backup-script
+  namespace: kube-system
+data:
+  ovn-backup.sh: |
+    #!/bin/bash
+
+    if [[ "$LOG_LEVEL" == "DEBUG" ]]
+    then
+        set -x
+    fi
+
+    log_level() {
+        local LEVEL="$1"
+        case "$LEVEL" in
+          DEBUG)
+            echo 5
+            ;;
+          INFO)
+            echo 4
+            ;;
+          WARNING)
+            echo 3
+            ;;
+          ERROR)
+            echo 2
+            ;;
+          CRITICAL)
+            echo 1
+            ;;
+          *)
+            exit 3
+            ;;
+        esac
+    }
+    export -f log_level
+
+    log_line() {
+        local LEVEL
+        LEVEL="$(log_level "$1")"
+        if [[ "$LEVEL" -ge "$LOG_LEVEL" ]]
+        then
+            local line
+            line=$(date +"%b %d %H:%M:%S $*")
+            echo "$line" | tee -a "$LOG_FILE"
+        fi
+    }
+    export -f log_line # exported for upload_file
+
+    # Delete old backup files on volume.
+    cd "$BACKUP_DIR" || exit 2
+    find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete;
+
+    # Make a backup in YYYY/MM/DD directory in $BACKUP_DIR
+    YMD="$(date +"%Y/%m/%d")"
+    mkdir -p "$YMD" && cd "$YMD" || exit 2 # kubectl-ko creates backups in $PWD, so we cd first.
+    /kube-ovn/kubectl-ko nb backup
+    /kube-ovn/kubectl-ko sb backup
+
+    if [[ "$UPLOAD_OSPC_V1" != "true" ]]
+    then
+        exit 0
+    fi
+
+    # Everything from here forward deals with uploading to Rackspace OSPCv1 Cloud
+    # Files.
+
+    cd "$BACKUP_DIR" || exit 2
+    CURL="$(which curl)"
+    export CONTAINER CURL # these need to reach the subshell below used with `find`
+    CF_URL_GREP_EXPR="https:\\\/\\\/storage101\.${CF_REGION}[0-9]\.clouddrive\.com\\\/v1\\\/MossoCloudFS_[[:digit:]]+"
+    CATALOG_FILE="$(mktemp /tmp/XXXXXXXX)"
+    $CURL -sS -X "POST" \
+             "https://identity.api.rackspacecloud.com/v2.0/tokens" \
+             -H 'Content-Type: application/json' \
+             -H 'Accept: application/json' \
+             -d "{
+          \"auth\": {
+            \"RAX-KSKEY:apiKeyCredentials\": {
+              \"username\": \"$USERNAME\",
+              \"apiKey\": \"$API_KEY\"
+            }
+          }
+        }" > "$CATALOG_FILE"
+    token=$(grep -oE '"id":"AA[^\"]+"' "$CATALOG_FILE" | cut -d \" -f 4)
+    CF_URL=$(grep -oE "$CF_URL_GREP_EXPR" "$CATALOG_FILE" | sed -e 's#\\/#/#g')
+    rm "$CATALOG_FILE"
+    export token CF_URL
+
+    # wrap curl with some things we will always use
+    curl_wrap() {
+        $CURL -sS -H "X-Auth-Token: $token" "$@"
+    }
+    export -f curl_wrap
+
+    # Create the container if it doesn't exist
+    check_container=$(curl_wrap -o /dev/null -w "%{http_code}" "${CF_URL}/$CONTAINER")
+    if ! [[ "$check_container" =~ 20[0-9] ]]
+    then
+      curl_wrap -X PUT "${CF_URL}/$CONTAINER"
+    fi
+
+    # upload_file uploads $1 to the CF container
+    upload_file() {
+        FILE="$1"
+        local curl_return
+        curl_return=$(curl_wrap -w "%{http_code}" \
+          -X PUT "${CF_URL}/${CONTAINER}/$FILE" -T "$FILE")
+        if [[ "$curl_return" == "201" ]]
+        then
+          log_line INFO "SUCCESSFUL UPLOAD $FILE"
+        else
+          log_line ERROR "FAILURE Cloud Files returned $curl_return uploading $FILE (expected 201)"
+        fi
+    }
+    export -f upload_file
+
+    # find created backups and upload them
+    cd "$BACKUP_DIR" || exit 2
+    # unusual find syntax to use an exported function from the shell
+    find "$YMD" -type f -exec bash -c 'upload_file "$0"' {} \;
+---
+# This ConfigMap contains variables controlling the run of the OVN DB backups.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ovn-backup-config
+  namespace: kube-system
+data:
+  RETENTION_DAYS: "30"
+  BACKUP_DIR: "/backup"
+  # You probably want to place this on the PersistentVolume so that it doesn't
+  # get deleted with the pod by cron, or use /dev/null if you don't care.
+  # You can use `kubectl logs` since log messages also go to STDOUT.
+  LOG_FILE: "/backup/upload.log"
+
+  # From here forward, variables for uploading to Rackspace OSPCv1 Cloud Files
+  UPLOAD_OSPC_V1: "false"
+  CF_REGION: "dfw" # OSPCv1 Cloud Files region to use
+  # You should probably include the name of the environment in CONTAINER
+  CONTAINER: "test-ovn-backup"
+  # Standard log levels DEBUG, INFO, WARNING, ERROR, CRITICAL
+  # DEBUG also does `set -x` toward the beginning of the backup script.
+  LOG_LEVEL: INFO
+---
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -38,20 +188,22 @@ spec:
           - name: backup
             persistentVolumeClaim:
               claimName: ovndb-backup
+          - name: backup-script
+            configMap:
+              name: ovn-backup-script
+              defaultMode: 0744
           containers:
           - name: ovn-central-backup
-            env:
-            - name: RETENTION_DAYS
-              value: "30"
-            command: ["/bin/sh", "-c"]
-            args:
-            - >
-              find /backup -ctime +$RETENTION_DAYS -delete;
-              /kube-ovn/kubectl-ko nb backup;
-              /kube-ovn/kubectl-ko sb backup;
-              mv /kube-ovn/ovn*db*.backup /backup;
+            envFrom:
+              - configMapRef:
+                  name: ovn-backup-config
+              - secretRef:
+                  name: ovn-backup-ospcv1-account
+            command: ["/backup-script/ovn-backup.sh"]
             image: docker.io/kubeovn/kube-ovn:v1.11.5
             imagePullPolicy: IfNotPresent
             volumeMounts:
             - name: backup
               mountPath: "/backup"
+            - name: backup-script
+              mountPath: /backup-script