feat(backup): add backup tool

.github/workflows/docker.yml

@@ -38,6 +38,9 @@ jobs:
           - name: redis
             file: docker/redis.dockerfile
             runs-on: ubuntu-latest
+          - name: backup
+            file: docker/backup.dockerfile
+            runs-on: ubuntu-latest
 
     runs-on: ${{ matrix.runs-on }}
 

docker/backup.Dockerfile

@@ -0,0 +1,24 @@
+FROM postgres:alpine AS postgresql
+
+FROM redis:alpine AS redis
+
+FROM clickhouse/clickhouse-server:head-alpine AS clickhouse
+
+FROM alpine:latest
+
+RUN apk add --no-cache \
+    bash \
+    curl \
+    jq \
+    mongodb-tools \
+    libpq \
+    lz4-libs \
+    aws-cli
+
+COPY --from=postgresql /usr/local/bin/pg_dumpall /usr/local/bin/pg_dumpall
+COPY --from=postgresql /usr/local/bin/pg_dump /usr/local/bin/pg_dump
+COPY --from=redis /usr/local/bin/redis-cli /usr/local/bin/redis-cli
+
+COPY docker/backup /scripts
+
+ENTRYPOINT ["/scripts/backup.sh"]

docker/backup/backup.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cd "$(dirname "$(realpath "$0")")"
+
+source ./utils.sh
+
+# Default values
+VERBOSE=${VERBOSE:-0}
+DEBUG=${DEBUG:-0}
+TMP_DIR=${TMP_DIR:-}
+S3_BUCKET=${S3_BUCKET:-}
+S3_ENDPOINT=${S3_ENDPOINT:-}
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+DB_HOST=${DB_HOST:-}
+DB_PORT=${DB_PORT:-}
+DB_USER=${DB_USER:-}
+DB_PASSWORD=${DB_PASSWORD:-}
+EXTRA_ARGS=${EXTRA_ARGS:-}
+COMMAND=${COMMAND:-}
+OUTPUT_PREFIX=${OUTPUT_PREFIX:-}
+OUTPUT_DATE=${OUTPUT_DATE:-}
+OUTPUT_FILE=${OUTPUT_FILE:-}
+
+parse_args "$@"
+
+or_default TMP_DIR "/tmp"
+or_default S3_ENDPOINT "https://${AWS_REGION:-us-east-1}.amazonaws.com"
+or_default OUTPUT_DATE "$(date +%Y%m%d%H%M%S)"
+or_default OUTPUT_PREFIX "${COMMAND}_"
+or_default OUTPUT_FILE "${OUTPUT_PREFIX}${OUTPUT_DATE}"
+or_default DB_HOST "localhost"
+
+
+# if debug print out the variables
+if [ "$DEBUG" -eq 1 ]; then
+    args=(
+        --verbose
+        --debug
+        --tmp-dir=\"$TMP_DIR\"
+        --s3-bucket=\"$S3_BUCKET\"
+        --s3-endpoint=\"$S3_ENDPOINT\"
+        --aws-access-key-id=\"$AWS_ACCESS_KEY_ID\"
+        --aws-secret-access-key=\"$AWS_SECRET_ACCESS_KEY\"
+        --output-prefix=\"$OUTPUT_PREFIX\"
+        --db-host=\"$DB_HOST\"
+        --db-port=\"$DB_PORT\"
+        --db-user=\"$DB_USER\"
+        --db-password=\"$DB_PASSWORD\"
+        --extra-args=\"$EXTRA_ARGS\"
+        $COMMAND
+    )
+
+    log "DEBUG" "${args[*]}"
+fi
+
+
+case "$COMMAND" in
+    postgresql)
+        . ./postgresql.sh
+        ;;
+    redis)
+        . ./redis.sh
+        ;;
+    mongodb)
+        . ./mongodb.sh
+        ;;
+    clickhouse)
+        . ./clickhouse.sh
+        ;;
+esac

docker/backup/clickhouse.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+clickhouse() {
+
+    local backup_query="BACKUP ALL TO S3('${S3_ENDPOINT}/${S3_BUCKET}/${OUTPUT_FILE}.tar.gz', '${AWS_ACCESS_KEY_ID}', '${AWS_SECRET_ACCESS_KEY}');"
+
+    if [ -n "$EXTRA_ARGS" ]; then
+        backup_query="${backup_query} ${EXTRA_ARGS}"
+    fi
+
+    # if user and password user:password otherwise either one
+    local user_info=""
+    if [ -n "$DB_USER" ] && [ -n "$DB_PASSWORD" ]; then
+        user_info="${DB_USER}:${DB_PASSWORD}@"
+    elif [ -n "$DB_USER" ]; then
+        user_info="${DB_USER}@"
+    elif [ -n "$DB_PASSWORD" ]; then
+        user_info="${DB_PASSWORD}@"
+    fi
+
+    or_default "DB_PORT" "8123"
+
+    log "INFO" "Starting ClickHouse backup"
+
+    invoke bash -c "echo \"$backup_query\" | curl \"http://${user_info}${DB_HOST}:${DB_PORT}/?\" --data-binary @-"
+}
+
+clickhouse "$@"

docker/backup/mongodb.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+mongodb() {
+    local output_file="${TMP_DIR}/${OUTPUT_FILE}.tar.gz"
+
+    or_default "DB_PORT" "27017"
+
+    local args=(
+        "--host=$DB_HOST"
+        "--port=$DB_PORT"
+        "--archive=$output_file"
+        "--gzip"
+    )
+
+    if [ -n "$DB_USER" ] && [ -n "$DB_PASSWORD" ]; then
+        args+=(
+            "--username=$DB_USER"
+            "--password=$DB_PASSWORD"
+        )
+    fi
+
+    if [ -n "$EXTRA_ARGS" ]; then
+        args+=("$EXTRA_ARGS")
+    fi
+
+    log "INFO" "Starting MongoDB backup"
+
+    invoke mongodump "${args[@]}"
+
+    s3_upload "$output_file"
+}
+
+mongodb "$@"

docker/backup/postgresql.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+postgresql() {
+    local output_file="${TMP_DIR}/${OUTPUT_FILE}.sql"
+
+    or_default "DB_PORT" "5432"
+    or_default "DB_USER" "postgres"
+
+    local args=(
+        --host="$DB_HOST"
+        --port="$DB_PORT"
+        --file="$output_file"
+        --username="$DB_USER"
+        --no-password
+    )
+
+    if [ -n "$EXTRA_ARGS" ]; then
+        args+=($EXTRA_ARGS)
+    fi
+
+    log "INFO" "Starting PostgreSQL backup"
+
+    PGPASSWORD=$DB_PASSWORD invoke pg_dumpall ${args[@]}
+
+    # GZIP the output file
+    gzip "$output_file"
+
+    s3_upload "$output_file.gz"
+}
+
+postgresql "$@"

docker/backup/redis.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+redis() {
+    local output_file="${TMP_DIR}/${OUTPUT_FILE}.rdb"
+
+    or_default "DB_PORT" "6379"
+
+    # Create a Redis backup using `redis-cli`
+    local args=(
+        -h "$DB_HOST"
+        -p "$DB_PORT"
+        --rdb "$output_file"
+    )
+
+    if [ -n "$DB_PASSWORD" ]; then
+        args+=(-a "$DB_PASSWORD")
+    fi
+
+    args+=("$EXTRA_ARGS")
+
+    log "INFO" "Starting Redis backup"
+
+    invoke redis-cli "${args[@]}"
+    invoke gzip "$output_file"
+
+    s3_upload "$output_file.gz"
+}
+
+redis "$@"