diff --git a/docs/infrastructure-ovn-setup.md b/docs/infrastructure-ovn-setup.md index 282adfc6..af28d6fc 100644 --- a/docs/infrastructure-ovn-setup.md +++ b/docs/infrastructure-ovn-setup.md @@ -98,3 +98,11 @@ kubectl apply -k /opt/genestack/kustomize/ovn After running the setup, nodes will have the label `ovn.openstack.org/configured` with a date stamp when it was configured. If there's ever a need to reconfigure a node, simply remove the label and the DaemonSet will take care of it automatically. + +!!! note + + To upload backups to OSPCv1 Cloud Files, edit ovn-backup-config-patch.yaml + to set `UPLOAD_OSPC_V1: "true"`, edit the other related options + appropriately (for instance, select a Rackspace OSPCv1 Cloud Files region) + and put the username and API key of the account to use in + `ospcv1-account.env` before running `kubectl apply` an indicated above. diff --git a/kustomize/ovn/kustomization.yaml b/kustomize/ovn/kustomization.yaml index 54636ccd..5c8d3fc3 100644 --- a/kustomize/ovn/kustomization.yaml +++ b/kustomize/ovn/kustomization.yaml @@ -1,3 +1,10 @@ +secretGenerator: + - name: ovn-backup-ospcv1-account + namespace: kube-system + envs: + - ospcv1-account.env resources: - ovn-setup.yaml - ovn-backup.yaml +patches: + - path: ovn-backup-config-patch.yaml diff --git a/kustomize/ovn/ospcv1-account.env b/kustomize/ovn/ospcv1-account.env new file mode 100644 index 00000000..cbdfeaa4 --- /dev/null +++ b/kustomize/ovn/ospcv1-account.env @@ -0,0 +1,2 @@ +USERNAME=ospcv1_username +API_KEY=ospcv1_api_key diff --git a/kustomize/ovn/ovn-backup-config-patch.yaml b/kustomize/ovn/ovn-backup-config-patch.yaml new file mode 100644 index 00000000..0580b1ea --- /dev/null +++ b/kustomize/ovn/ovn-backup-config-patch.yaml @@ -0,0 +1,25 @@ +# This ConfigMap contains variables controlling the run of the OVN DB backups. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ovn-backup-config + namespace: kube-system +data: + RETENTION_DAYS: "30" + BACKUP_DIR: "/backup" + # You probably want to place this on the PersistentVolume so that it doesn't + # get deleted with the pod by cron, or use /dev/null if you don't care. + # You can use `kubectl logs` since log messages also go to STDOUT. + LOG_FILE: "/backup/upload.log" + + # From here forward, variables for uploading to Rackspace OSPCv1 Cloud Files + UPLOAD_OSPC_V1: "false" + # Nothing after this line makes any difference unless you used + # UPLOAD_OSPC_V1: "true" + # above. + CF_REGION: "dfw" # OSPCv1 Cloud Files region to use + # You should probably include the name of the environment in CONTAINER + CONTAINER: "test-ovn-backup" + # Standard log levels DEBUG, INFO, WARNING, ERROR, CRITICAL + # DEBUG also does `set -x` toward the beginning of the backup script. + LOG_LEVEL: INFO diff --git a/kustomize/ovn/ovn-backup.yaml b/kustomize/ovn/ovn-backup.yaml index cbfc9ea0..397dbb40 100644 --- a/kustomize/ovn/ovn-backup.yaml +++ b/kustomize/ovn/ovn-backup.yaml @@ -1,9 +1,11 @@ -# This writes OVN NB and SB snapshots to a persistent volume, assuming you -# installed OVN with kubespray, since it assumes resources exist as seen in the +# The resources contained in this file (ovn-backup.yaml) write OVN NB and SB +# snapshots to a persistent volume, assuming you installed OVN with kubespray, +# since it assumes resources exist as seen in the # genestack/submodules/kubespray/roles/network_plugin/kube-ovn/templates # directory, assuming you have checked out the genestack submodules. # (For instance, it uses the `ovn` service account as seen in # genestack/submodules/kubespray/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +--- apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -17,6 +19,154 @@ spec: storage: 1Gi storageClassName: general --- +# The script that runs the backup and uploads the file. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ovn-backup-script + namespace: kube-system +data: + ovn-backup.sh: | + #!/bin/bash + + if [[ "$LOG_LEVEL" == "DEBUG" ]] + then + set -x + fi + + log_level() { + local LEVEL="$1" + case "$LEVEL" in + DEBUG) + echo 5 + ;; + INFO) + echo 4 + ;; + WARNING) + echo 3 + ;; + ERROR) + echo 2 + ;; + CRITICAL) + echo 1 + ;; + *) + exit 3 + ;; + esac + } + export -f log_level + + log_line() { + local LEVEL + LEVEL="$(log_level "$1")" + if [[ "$LEVEL" -ge "$LOG_LEVEL" ]] + then + local line + line=$(date +"%b %d %H:%M:%S $*") + echo "$line" | tee -a "$LOG_FILE" + fi + } + export -f log_line # exported for upload_file + + # Delete old backup files on volume. + cd "$BACKUP_DIR" || exit 2 + find "$BACKUP_DIR" -ctime +"$RETENTION_DAYS" -delete; + + # Make a backup in YYYY/MM/DD directory in $BACKUP_DIR + YMD="$(date +"%Y/%m/%d")" + mkdir -p "$YMD" && cd "$YMD" || exit 2 # kubectl-ko creates backups in $PWD, so we cd first. + /kube-ovn/kubectl-ko nb backup + /kube-ovn/kubectl-ko sb backup + + if [[ "$UPLOAD_OSPC_V1" != "true" ]] + then + exit 0 + fi + + # Everything from here forward deals with uploading to Rackspace OSPCv1 Cloud + # Files. + + cd "$BACKUP_DIR" || exit 2 + CURL="$(which curl)" + export CONTAINER CURL # these need to reach the subshell below used with `find` + CF_URL_GREP_EXPR="https:\\\/\\\/storage101\.${CF_REGION}[0-9]\.clouddrive\.com\\\/v1\\\/MossoCloudFS_[[:digit:]]+" + CATALOG_FILE="$(mktemp /tmp/XXXXXXXX)" + $CURL -sS -X "POST" \ + "https://identity.api.rackspacecloud.com/v2.0/tokens" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json' \ + -d "{ + \"auth\": { + \"RAX-KSKEY:apiKeyCredentials\": { + \"username\": \"$USERNAME\", + \"apiKey\": \"$API_KEY\" + } + } + }" > "$CATALOG_FILE" + token=$(grep -oE '"id":"AA[^\"]+"' "$CATALOG_FILE" | cut -d \" -f 4) + CF_URL=$(grep -oE "$CF_URL_GREP_EXPR" "$CATALOG_FILE" | sed -e 's#\\/#/#g') + rm "$CATALOG_FILE" + export token CF_URL + + # wrap curl with some things we will always use + curl_wrap() { + $CURL -sS -H "X-Auth-Token: $token" "$@" + } + export -f curl_wrap + + # Create the container if it doesn't exist + check_container=$(curl_wrap -o /dev/null -w "%{http_code}" "${CF_URL}/$CONTAINER") + if ! [[ "$check_container" =~ 20[0-9] ]] + then + curl_wrap -X PUT "${CF_URL}/$CONTAINER" + fi + + # upload_file uploads $1 to the CF container + upload_file() { + FILE="$1" + local curl_return + curl_return=$(curl_wrap -w "%{http_code}" \ + -X PUT "${CF_URL}/${CONTAINER}/$FILE" -T "$FILE") + if [[ "$curl_return" == "201" ]] + then + log_line INFO "SUCCESSFUL UPLOAD $FILE" + else + log_line ERROR "FAILURE Cloud Files returned $curl_return uploading $FILE (expected 201)" + fi + } + export -f upload_file + + # find created backups and upload them + cd "$BACKUP_DIR" || exit 2 + # unusual find syntax to use an exported function from the shell + find "$YMD" -type f -exec bash -c 'upload_file "$0"' {} \; +--- +# This ConfigMap contains variables controlling the run of the OVN DB backups. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ovn-backup-config + namespace: kube-system +data: + RETENTION_DAYS: "30" + BACKUP_DIR: "/backup" + # You probably want to place this on the PersistentVolume so that it doesn't + # get deleted with the pod by cron, or use /dev/null if you don't care. + # You can use `kubectl logs` since log messages also go to STDOUT. + LOG_FILE: "/backup/upload.log" + + # From here forward, variables for uploading to Rackspace OSPCv1 Cloud Files + UPLOAD_OSPC_V1: "false" + CF_REGION: "dfw" # OSPCv1 Cloud Files region to use + # You should probably include the name of the environment in CONTAINER + CONTAINER: "test-ovn-backup" + # Standard log levels DEBUG, INFO, WARNING, ERROR, CRITICAL + # DEBUG also does `set -x` toward the beginning of the backup script. + LOG_LEVEL: INFO +--- apiVersion: batch/v1 kind: CronJob metadata: @@ -38,20 +188,22 @@ spec: - name: backup persistentVolumeClaim: claimName: ovndb-backup + - name: backup-script + configMap: + name: ovn-backup-script + defaultMode: 0744 containers: - name: ovn-central-backup - env: - - name: RETENTION_DAYS - value: "30" - command: ["/bin/sh", "-c"] - args: - - > - find /backup -ctime +$RETENTION_DAYS -delete; - /kube-ovn/kubectl-ko nb backup; - /kube-ovn/kubectl-ko sb backup; - mv /kube-ovn/ovn*db*.backup /backup; + envFrom: + - configMapRef: + name: ovn-backup-config + - secretRef: + name: ovn-backup-ospcv1-account + command: ["/backup-script/ovn-backup.sh"] image: docker.io/kubeovn/kube-ovn:v1.11.5 imagePullPolicy: IfNotPresent volumeMounts: - name: backup mountPath: "/backup" + - name: backup-script + mountPath: /backup-script