diff --git a/.github/workflows/kustomize-etcd-backups.yaml b/.github/workflows/kustomize-etcd-backups.yaml new file mode 100644 index 00000000..ef658177 --- /dev/null +++ b/.github/workflows/kustomize-etcd-backups.yaml @@ -0,0 +1,33 @@ +name: Kustomize GitHub Actions for Backups directory + +on: + pull_request: + paths: + - kustomize/backups/etcd/** + - .github/workflows/kustomize-etcd-backups.yaml +jobs: + kustomize: + name: Kustomize + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: azure/setup-helm@v3 + with: + version: latest + token: "${{ secrets.GITHUB_TOKEN }}" + id: helm + - name: Kustomize Install + working-directory: /usr/local/bin/ + run: | + if [ ! -f /usr/local/bin/kustomize ]; then + curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | sudo bash + fi + - name: Run Kustomize Build + run: | + kustomize build kustomize/backups/etcd/ > /tmp/rendered.yaml + - name: Return Kustomize Build + uses: actions/upload-artifact@v2 + with: + name: kustomize-etcd-backup-artifact + path: /tmp/rendered.yaml diff --git a/Containerfiles/etcd-backup/Containerfile b/Containerfiles/etcd-backup/Containerfile new file mode 100644 index 00000000..899b0086 --- /dev/null +++ b/Containerfiles/etcd-backup/Containerfile @@ -0,0 +1,24 @@ +FROM python:3.9-alpine3.17 + +ARG ETCD_VERSION=v3.4.13 + +ENV ETCDCTL_ENDPOINTS "https://127.0.0.1:2379" +ENV ETCDCTL_CACERT "/etc/ssl/etcd/ssl/ca.crt" +ENV ETCDCTL_KEY "/etc/ssl/etcd/ssl/healthcheck-client.key" +ENV ETCDCTL_CERT "/etc/ssl/etcd/ssl/healthcheck-client.crt" +ENV S3_ENDPOINT "10.74.8.135" +ENV S3_PORT "8081" +ENV S3_ACCESS_KEY "abcd" +ENV S3_SECRET_KEY "abcd" + +RUN apk add --update --no-cache bash ca-certificates tzdata openssl +RUN pip install boto + +RUN wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \ + && tar xzf etcd-${ETCD_VERSION}-linux-amd64.tar.gz \ + && mv etcd-${ETCD_VERSION}-linux-amd64/etcdctl /usr/local/bin/etcdctl \ + && rm -rf etcd-${ETCD_VERSION}-linux-amd64* + +COPY ./backup.py /backup.py + +ENTRYPOINT ["/bin/bash"] diff --git a/Containerfiles/etcd-backup/backup.py b/Containerfiles/etcd-backup/backup.py new file mode 100644 index 00000000..8324c4f3 --- /dev/null +++ b/Containerfiles/etcd-backup/backup.py @@ -0,0 +1,74 @@ +import os +import sys +import boto +import boto.s3.connection + + +def get_env_variables(): + """Get environment variables.""" + access_key = os.getenv("ACCESS_KEY") + secret_key = os.getenv("SECRET_KEY") + host = os.getenv("S3_HOST") + + port_str = os.getenv("S3_PORT", "8081") + try: + port = int(port_str) + except ValueError: + raise ValueError( + f"Environment variable 'S3_PORT' has an invalid value: {port_str}" + ) + + # Properly convert the 'S3_HOST_SSL' environment variable to a boolean + secure_str = os.getenv("S3_HOST_SSL", "false").lower() + secure = secure_str in ["true", "1", "t", "y", "yes"] + + return access_key, secret_key, host, port, secure + + +def create_s3_connection(access_key, secret_key, host, port, secure): + """Create S3 connection.""" + conn = boto.connect_s3( + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + host=host, + port=port, + is_secure=secure, + calling_format=boto.s3.connection.OrdinaryCallingFormat(), + ) + return conn + + +def upload_file_to_bucket(conn, file_to_upload): + """Upload a file to a specific S3 bucket.""" + bucket_name = "etcd-backup-bucket" + bucket = conn.get_bucket(bucket_name) + key = bucket.new_key(os.path.basename(file_to_upload)) + key.set_contents_from_filename(file_to_upload) + + +def list_all_buckets(conn): + """List all buckets.""" + for bucket in conn.get_all_buckets(): + print( + "{name}\t{created}".format( + name=bucket.name, + created=bucket.creation_date, + ) + ) + + +def main(): + """Main function.""" + if len(sys.argv) != 2: + print("Usage: python your_script.py ") + sys.exit(1) + + file_to_upload = sys.argv[1] + access_key, secret_key, host, port, secure = get_env_variables() + conn = create_s3_connection(access_key, secret_key, host, port, secure) + upload_file_to_bucket(conn, file_to_upload) + list_all_buckets(conn) + + +if __name__ == "__main__": + main() diff --git a/docs/etcd-backup.md b/docs/etcd-backup.md new file mode 100644 index 00000000..11f9b61c --- /dev/null +++ b/docs/etcd-backup.md @@ -0,0 +1,47 @@ +# Etcd Backup + +In order to backup etcd we create a backup CronJob resource. This constitues of 3 things: + +1. etcd-bakcup container image with the etcdctl binary and the python script that uploads +the backup to Ceph S3 endpoint or any S3 compatible endpoint. + +2. The CronJob deployment resource. This job will only be done on the box with label set +matching is-etcd-backup-enabled. + +3. Secrets required for the backup to function. These include the location of the +S3 endpoint, access keys, and etcd certs to access etcd endpoints. + + +Label one or more box in the cluster to run the job: + +``` +kubectl label node etcd01.sjc.ohthree.com is-etcd-backup-node=true +``` + +Create the secret: + +``` +kubectl --namespace openstack \ + create secret generic etcd-backup-secrets \ + --type Opaque \ + --from-literal=ACCESS_KEY="sadbq4bcva2392dasflkdsp" \ + --from-literal=SECRET_KEY="aldskflkjpoq32ibdsfko23bnalkfdao2" \ + --from-literal=S3_HOST="127.0.0.1" \ + --from-literal=S3_PORT="8081" \ + --from-literal=ETCDCTL_API="3" \ + --from-literal=ETCDCTL_ENDPOINTS="https://127.0.0.1:2379" \ + --from-literal=ETCDCTL_CACERT="/etc/ssl/etcd/ssl/ca.pem" \ + --from-literal=ETCDCTL_CERT="/etc/ssl/etcd/ssl/member-etcd01.sjc.ohthree.com.pem" \ + --from-literal=ETCDCTL_KEY="/etc/ssl/etcd/ssl/member-etcd01.sjc.ohthree.com-key.pem" +``` + +!!! note + + Make sure to use the correct values for your region. + + +Next, Deploy the backup job: + +``` +kubectl apply -k /opt/genestack/kustomize/backups/etcd/etcd-backup.yaml --namespace openstack +``` diff --git a/kustomize/backups/etcd/etcd-backup.yaml b/kustomize/backups/etcd/etcd-backup.yaml new file mode 100644 index 00000000..5a1936d4 --- /dev/null +++ b/kustomize/backups/etcd/etcd-backup.yaml @@ -0,0 +1,101 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: etcd-backup +spec: + schedule: "0 */3 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + concurrencyPolicy: Allow + jobTemplate: + spec: + template: + spec: + initContainers: + - name: etcd-backup + image: csengteam/etcd-backup:v0.0.4 + env: + - name: ETCDCTL_API + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: ETCDCTL_API + - name: ETCDCTL_ENDPOINTS + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: ETCDCTL_ENDPOINTS + - name: ETCDCTL_CACERT + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: ETCDCTL_CACERT + - name: ETCDCTL_CERT + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: ETCDCTL_CERT + - name: ETCDCTL_KEY + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: ETCDCTL_KEY + command: ["/bin/bash", "-c"] + args: ["etcdctl snapshot save /data/etcd-backup/etcd-snapshot-$(date +%Y-%m-%d).db"] + volumeMounts: + - mountPath: /etc/ssl/etcd/ssl + name: etcd-certs + readOnly: true + - mountPath: /data/etcd-backup + name: etcd-backup + restartPolicy: OnFailure + containers: + - name: backup-to-s3 + image: csengteam/etcd-backup:v0.0.4 + env: + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: ACCESS_KEY + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: SECRET_KEY + - name: S3_HOST + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: S3_HOST + - name: S3_PORT + valueFrom: + secretKeyRef: + name: etcd-backup-secrets + key: S3_PORT + command: ["/bin/bash", "-c"] + args: ["python /backup.py /data/etcd-backup/etcd-snapshot-$(date +%Y-%m-%d).db"] + volumeMounts: + - mountPath: /data/etcd-backup + name: etcd-backup + hostNetwork: true + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: is-etcd-backup-node + operator: Exists + tolerations: + - key: node.kubernetes.io/memory-pressure + effect: NoSchedule + operator: Exists + volumes: + - name: etcd-certs + hostPath: + path: /etc/ssl/etcd/ssl + type: Directory + - name: etcd-backup + hostPath: + path: /data/etcd-backup + type: DirectoryOrCreate diff --git a/kustomize/backups/etcd/kustomization.yaml b/kustomize/backups/etcd/kustomization.yaml new file mode 100644 index 00000000..824ee0a7 --- /dev/null +++ b/kustomize/backups/etcd/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - etcd-backup.yaml diff --git a/mkdocs.yml b/mkdocs.yml index 0dfa0999..453eaa49 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -203,6 +203,7 @@ nav: - Running Genestack Upgrade: genestack-upgrade.md - Running Kubespray Upgrade: k8s-kubespray-upgrade.md - Infrastructure: + - Etcd Backup: etcd-backup.md - OVN Database Backup: infrastructure-ovn-db-backup.md - MariaDB Operations: infrastructure-mariadb-ops.md - OpenStack: