Adds etcd backup job

This patch adds etcd backup cronjob with frequency of every 3
hours.

.github/workflows/kustomize-etcd-backups.yaml

@@ -0,0 +1,33 @@
+name: Kustomize GitHub Actions for Backups directory
+
+on:
+  pull_request:
+    paths:
+      - kustomize/backups/etcd/**
+      - .github/workflows/kustomize-etcd-backups.yaml
+jobs:
+  kustomize:
+    name: Kustomize
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: azure/setup-helm@v3
+        with:
+          version: latest
+          token: "${{ secrets.GITHUB_TOKEN }}"
+        id: helm
+      - name: Kustomize Install
+        working-directory: /usr/local/bin/
+        run: |
+          if [ ! -f /usr/local/bin/kustomize ]; then
+            curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | sudo bash
+          fi
+      - name: Run Kustomize Build
+        run: |
+          kustomize build kustomize/backups/etcd/  > /tmp/rendered.yaml
+      - name: Return Kustomize Build
+        uses: actions/upload-artifact@v2
+        with:
+          name: kustomize-etcd-backup-artifact
+          path: /tmp/rendered.yaml

Containerfiles/etcd-backup/Containerfile

@@ -0,0 +1,24 @@
+FROM python:3.9-alpine3.17
+
+ARG ETCD_VERSION=v3.4.13
+
+ENV ETCDCTL_ENDPOINTS "https://127.0.0.1:2379"
+ENV ETCDCTL_CACERT "/etc/ssl/etcd/ssl/ca.crt"
+ENV ETCDCTL_KEY "/etc/ssl/etcd/ssl/healthcheck-client.key"
+ENV ETCDCTL_CERT "/etc/ssl/etcd/ssl/healthcheck-client.crt"
+ENV S3_ENDPOINT "10.74.8.135"
+ENV S3_PORT "8081"
+ENV S3_ACCESS_KEY "abcd"
+ENV S3_SECRET_KEY "abcd"
+
+RUN apk add --update --no-cache bash ca-certificates tzdata openssl
+RUN pip install boto
+
+RUN wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
+ && tar xzf etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
+ && mv etcd-${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin/etcdctl \
+ && rm -rf etcd-${ETCD_VERSION}-linux-amd64*
+
+COPY ./backup.py /backup.py
+
+ENTRYPOINT ["/bin/bash"]

Containerfiles/etcd-backup/backup.py

@@ -0,0 +1,74 @@
+import os
+import sys
+import boto
+import boto.s3.connection
+
+
+def get_env_variables():
+    """Get environment variables."""
+    access_key = os.getenv("ACCESS_KEY")
+    secret_key = os.getenv("SECRET_KEY")
+    host = os.getenv("S3_HOST")
+
+    port_str = os.getenv("S3_PORT", "8081")
+    try:
+        port = int(port_str)
+    except ValueError:
+        raise ValueError(
+            f"Environment variable 'S3_PORT' has an invalid value: {port_str}"
+        )
+
+    # Properly convert the 'S3_HOST_SSL' environment variable to a boolean
+    secure_str = os.getenv("S3_HOST_SSL", "false").lower()
+    secure = secure_str in ["true", "1", "t", "y", "yes"]
+
+    return access_key, secret_key, host, port, secure
+
+
+def create_s3_connection(access_key, secret_key, host, port, secure):
+    """Create S3 connection."""
+    conn = boto.connect_s3(
+        aws_access_key_id=access_key,
+        aws_secret_access_key=secret_key,
+        host=host,
+        port=port,
+        is_secure=secure,
+        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
+    )
+    return conn
+
+
+def upload_file_to_bucket(conn, file_to_upload):
+    """Upload a file to a specific S3 bucket."""
+    bucket_name = "etcd-backup-bucket"
+    bucket = conn.get_bucket(bucket_name)
+    key = bucket.new_key(os.path.basename(file_to_upload))
+    key.set_contents_from_filename(file_to_upload)
+
+
+def list_all_buckets(conn):
+    """List all buckets."""
+    for bucket in conn.get_all_buckets():
+        print(
+            "{name}\t{created}".format(
+                name=bucket.name,
+                created=bucket.creation_date,
+            )
+        )
+
+
+def main():
+    """Main function."""
+    if len(sys.argv) != 2:
+        print("Usage: python your_script.py <file_to_upload>")
+        sys.exit(1)
+
+    file_to_upload = sys.argv[1]
+    access_key, secret_key, host, port, secure = get_env_variables()
+    conn = create_s3_connection(access_key, secret_key, host, port, secure)
+    upload_file_to_bucket(conn, file_to_upload)
+    list_all_buckets(conn)
+
+
+if __name__ == "__main__":
+    main()

docs/etcd-backup.md

@@ -0,0 +1,47 @@
+# Etcd Backup
+
+In order to backup etcd we create a backup CronJob resource. This constitues of 3 things:
+
+1. etcd-bakcup container image with the etcdctl binary and the python script that uploads
+the backup to Ceph S3 endpoint or any S3 compatible endpoint.
+
+2. The CronJob deployment resource. This job will only be done on the box with label set
+matching is-etcd-backup-enabled.
+
+3. Secrets required for the backup to function. These include the location of the
+S3 endpoint, access keys, and etcd certs to access etcd endpoints.
+
+
+Label one or more box in the cluster to run the job:
+
+```
+kubectl label node etcd01.sjc.ohthree.com is-etcd-backup-node=true
+```
+
+Create the secret:
+
+```
+kubectl --namespace openstack \
+        create secret generic etcd-backup-secrets \
+        --type Opaque \
+        --from-literal=ACCESS_KEY="sadbq4bcva2392dasflkdsp" \
+        --from-literal=SECRET_KEY="aldskflkjpoq32ibdsfko23bnalkfdao2" \
+        --from-literal=S3_HOST="127.0.0.1" \
+        --from-literal=S3_PORT="8081" \
+        --from-literal=ETCDCTL_API="3" \
+        --from-literal=ETCDCTL_ENDPOINTS="https://127.0.0.1:2379" \
+        --from-literal=ETCDCTL_CACERT="/etc/ssl/etcd/ssl/ca.pem" \
+        --from-literal=ETCDCTL_CERT="/etc/ssl/etcd/ssl/member-etcd01.sjc.ohthree.com.pem" \
+        --from-literal=ETCDCTL_KEY="/etc/ssl/etcd/ssl/member-etcd01.sjc.ohthree.com-key.pem"
+```
+
+!!! note
+
+    Make sure to use the correct values for your region.
+
+
+Next, Deploy the backup job:
+
+```
+kubectl apply -k /opt/genestack/kustomize/backups/etcd/etcd-backup.yaml --namespace openstack
+```

kustomize/backups/etcd/etcd-backup.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-backup
+spec:
+  schedule: "0 */3 * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  concurrencyPolicy: Allow
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          initContainers:
+          - name: etcd-backup
+            image: csengteam/etcd-backup:v0.0.4
+            env:
+            - name: ETCDCTL_API
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: ETCDCTL_API
+            - name: ETCDCTL_ENDPOINTS
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: ETCDCTL_ENDPOINTS
+            - name: ETCDCTL_CACERT
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: ETCDCTL_CACERT
+            - name: ETCDCTL_CERT
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: ETCDCTL_CERT
+            - name: ETCDCTL_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: ETCDCTL_KEY
+            command: ["/bin/bash", "-c"]
+            args: ["etcdctl snapshot save /data/etcd-backup/etcd-snapshot-$(date +%Y-%m-%d).db"]
+            volumeMounts:
+            - mountPath: /etc/ssl/etcd/ssl
+              name: etcd-certs
+              readOnly: true
+            - mountPath: /data/etcd-backup
+              name: etcd-backup
+          restartPolicy: OnFailure
+          containers:
+          - name: backup-to-s3
+            image: csengteam/etcd-backup:v0.0.4
+            env:
+            - name: ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: ACCESS_KEY
+            - name: SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: SECRET_KEY
+            - name: S3_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: S3_HOST
+            - name: S3_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: etcd-backup-secrets
+                  key: S3_PORT
+            command: ["/bin/bash", "-c"]
+            args: ["python /backup.py /data/etcd-backup/etcd-snapshot-$(date +%Y-%m-%d).db"]
+            volumeMounts:
+            - mountPath: /data/etcd-backup
+              name: etcd-backup
+          hostNetwork: true
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                    - key: is-etcd-backup-node
+                      operator: Exists
+          tolerations:
+            - key: node.kubernetes.io/memory-pressure
+              effect: NoSchedule
+              operator: Exists
+          volumes:
+          - name: etcd-certs
+            hostPath:
+              path: /etc/ssl/etcd/ssl
+              type: Directory
+          - name: etcd-backup
+            hostPath:
+              path: /data/etcd-backup
+              type: DirectoryOrCreate

kustomize/backups/etcd/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - etcd-backup.yaml

mkdocs.yml

@@ -203,6 +203,7 @@ nav:
       - Running Genestack Upgrade: genestack-upgrade.md
       - Running Kubespray Upgrade: k8s-kubespray-upgrade.md
       - Infrastructure:
+          - Etcd Backup: etcd-backup.md
           - OVN Database Backup: infrastructure-ovn-db-backup.md
           - MariaDB Operations: infrastructure-mariadb-ops.md
       - OpenStack: