From 819316a353b56133c483e215a25f3374a04f23b4 Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Tue, 10 Dec 2024 16:12:06 -0500 Subject: [PATCH] docs: Rework some blueprints to separate DB tools image and kanister-tools image Separation of images allows us to build less images in kanister release process and allows the users of those blueprints to always use up to data databse images. Signed-off-by: Daniil Fedotov --- .../mongodb/blueprint-v2/mongo-blueprint.yaml | 55 ++++++++++++++---- .../mssql/blueprint-v2/mssql-blueprint.yaml | 54 ++++++++++++++---- .../mysql/blueprint-v2/mysql-blueprint.yaml | 55 ++++++++++++++---- .../blueprint-v2/postgres-blueprint.yaml | 56 +++++++++++++++---- 4 files changed, 177 insertions(+), 43 deletions(-) diff --git a/examples/mongodb/blueprint-v2/mongo-blueprint.yaml b/examples/mongodb/blueprint-v2/mongo-blueprint.yaml index 12ce0ccbea..9a77cf489b 100644 --- a/examples/mongodb/blueprint-v2/mongo-blueprint.yaml +++ b/examples/mongodb/blueprint-v2/mongo-blueprint.yaml @@ -11,7 +11,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.takeConsistentBackup.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: takeConsistentBackup objects: mongosecret: @@ -20,8 +20,13 @@ actions: namespace: "{{ .StatefulSet.Namespace }}" args: namespace: "{{ .StatefulSet.Namespace }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: bitnami/mongodb:7.0-debian-12 + backgroundCommand: - bash - -o - errexit @@ -32,15 +37,27 @@ actions: host='{{ .StatefulSet.Name }}-0.{{ .StatefulSet.Name }}-headless.{{ .StatefulSet.Namespace }}.svc.cluster.local' dbPassword='{{ index .Phases.takeConsistentBackup.Secrets.mongosecret.Data "mongodb-root-password" | toString }}' dump_cmd="mongodump --oplog --gzip --archive --host ${host} -u root -p ${dbPassword}" + ${dump_cmd} > /tmp/data + + outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | backup_file_path='rs_backup.gz' - ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here # Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot` - mongoBackup phases: - - func: KubeTask + - func: MultiContainerRun name: pullFromStore objects: mongosecret: @@ -49,8 +66,26 @@ actions: namespace: "{{ .StatefulSet.Namespace }}" args: namespace: "{{ .StatefulSet.Namespace }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + backgroundCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path='rs_backup.gz' + kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: bitnami/mongodb:7.0-debian-12 + outputCommand: - bash - -o - errexit @@ -61,9 +96,7 @@ actions: host='{{ .StatefulSet.Name }}-0.{{ .StatefulSet.Name }}-headless.{{ .StatefulSet.Namespace }}.svc.cluster.local' dbPassword='{{ index .Phases.pullFromStore.Secrets.mongosecret.Data "mongodb-root-password" | toString }}' restore_cmd="mongorestore --gzip --archive --oplogReplay --drop --host ${host} -u root -p ${dbPassword}" - backup_file_path='rs_backup.gz' - kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}' - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd} + cat /tmp/data | ${restore_cmd} delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -74,7 +107,7 @@ actions: name: deleteFromStore args: namespace: "{{ .Namespace.Name }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 + image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' command: - bash - -o diff --git a/examples/mssql/blueprint-v2/mssql-blueprint.yaml b/examples/mssql/blueprint-v2/mssql-blueprint.yaml index 547b084e24..5118ed3c95 100644 --- a/examples/mssql/blueprint-v2/mssql-blueprint.yaml +++ b/examples/mssql/blueprint-v2/mssql-blueprint.yaml @@ -8,7 +8,7 @@ actions: mssqlCloudDump: kopiaSnapshot: "{{ .Phases.dumpToObjectStore.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: dumpToObjectStore objects: mssql: @@ -16,8 +16,13 @@ actions: name: '{{ index .Object.metadata.labels "app" }}' namespace: '{{ .Deployment.Namespace }}' args: - image: ghcr.io/kanisterio/mssql-tools:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: ghcr.io/kanisterio/mssql-tools:0.112.0 + backgroundCommand: - bash - -o - errexit @@ -25,7 +30,6 @@ actions: - pipefail - -c - | - kopiaPath="backup.tar.gz" root_password="{{ index .Phases.dumpToObjectStore.Secrets.mssql.Data "SA_PASSWORD" | toString }}" root_uname="sa" mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}" @@ -33,13 +37,26 @@ actions: databases=$(/opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "SET NOCOUNT ON; SELECT name FROM sys.databases WHERE name NOT IN ('master','model','msdb','tempdb')" -b -s "," -h -1) for database in $databases; do /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "backup database $database to disk = '/tmp/backup/$database.bak' with format;"; done kubectl cp ${mssql_pod}:/tmp/backup /tmp/backup - tar zcvf - -C /tmp/ backup | kando location push --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --output-name "kopiaOutput" - + tar zcvf - -C /tmp/ backup > /tmp/data kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup + + outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + kopiaPath="backup.tar.gz" + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --output-name "kopiaOutput" - + restore: inputArtifactNames: - mssqlCloudDump phases: - - func: KubeTask + - func: MultiContainerRun name: restoreFromObjectStore objects: mssql: @@ -47,8 +64,13 @@ actions: name: '{{ index .Object.metadata.labels "app" }}' namespace: '{{ .Deployment.Namespace }}' args: - image: ghcr.io/kanisterio/mssql-tools:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + backgroundCommand: - bash - -o - errexit @@ -58,15 +80,27 @@ actions: - | kopiaPath="backup.tar.gz" kopia_snap='{{ .ArtifactsIn.mssqlCloudDump.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --kopia-snapshot ${kopia_snap} - > /tmp/data + + outputImage: ghcr.io/kanisterio/mssql-tools:0.112.0 + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | root_password="{{ index .Phases.restoreFromObjectStore.Secrets.mssql.Data "SA_PASSWORD" | toString }}" root_uname="sa" mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}" server_name="{{ index .Deployment.Name }}.{{ index .Deployment.Namespace }}.svc.cluster.local" - kando location pull --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --kopia-snapshot ${kopia_snap} - | tar zxvf - -C /tmp/ + cat /tmp/data | tar zxvf - -C /tmp/ kubectl cp /tmp/backup ${mssql_pod}:/tmp/backup backup_files=$(ls /tmp/backup) for script in $backup_files; do database="$(cut -d'.' -f1 <<<"$script")"; /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "restore database $database from disk = '/tmp/backup/$script' with replace"; done kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup + delete: inputArtifactNames: - mssqlCloudDump @@ -74,7 +108,7 @@ actions: - func: KubeTask name: deleteFromBlobStore args: - image: ghcr.io/kanisterio/mssql-tools:0.112.0 + image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' command: - bash - -o diff --git a/examples/mysql/blueprint-v2/mysql-blueprint.yaml b/examples/mysql/blueprint-v2/mysql-blueprint.yaml index 3816a949e1..1081f1b89f 100644 --- a/examples/mysql/blueprint-v2/mysql-blueprint.yaml +++ b/examples/mysql/blueprint-v2/mysql-blueprint.yaml @@ -11,7 +11,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.dumpToStore.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: dumpToStore objects: mysqlSecret: @@ -19,9 +19,14 @@ actions: name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}' namespace: '{{ .StatefulSet.Namespace }}' args: - image: ghcr.io/kanisterio/mysql-sidecar:0.112.0 namespace: "{{ .StatefulSet.Namespace }}" - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: mysql:8 + backgroundCommand: - bash - -o - errexit @@ -29,17 +34,29 @@ actions: - pipefail - -c - | - backup_file_path="dump.sql" root_password="{{ index .Phases.dumpToStore.Secrets.mysqlSecret.Data "mysql-root-password" | toString }}" dump_cmd="mysqldump --column-statistics=0 -u root --password=${root_password} -h {{ index .Object.metadata.labels "app.kubernetes.io/instance" }} --single-transaction --all-databases" - ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + ${dump_cmd} > /tmp/data + + outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path="dump.sql" + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here # Use the `--kopia-snapshot` flag in kando to pass in `mysqlBackup.KopiaSnapshot` - mysqlBackup phases: - - func: KubeTask + - func: MultiContainerRun name: restoreFromStore objects: mysqlSecret: @@ -47,9 +64,14 @@ actions: name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}' namespace: '{{ .StatefulSet.Namespace }}' args: - image: ghcr.io/kanisterio/mysql-sidecar:0.112.0 namespace: "{{ .StatefulSet.Namespace }}" - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + backgroundCommand: - bash - -o - errexit @@ -59,9 +81,20 @@ actions: - | backup_file_path="dump.sql" kopia_snap='{{ .ArtifactsIn.mysqlBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: mysql:8 + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | root_password="{{ index .Phases.restoreFromStore.Secrets.mysqlSecret.Data "mysql-root-password" | toString }}" - restore_cmd="mysql -u root --password=${root_password} -h {{ index .Object.metadata.labels "app.kubernetes.io/instance" }}" - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd} + cat /tmp/data | mysql -u root --password=${root_password} -h {{ index .Object.metadata.labels "app.kubernetes.io/instance" }} + delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -71,7 +104,7 @@ actions: - func: KubeTask name: deleteFromStore args: - image: ghcr.io/kanisterio/mysql-sidecar:0.112.0 + image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' namespace: "{{ .Namespace.Name }}" command: - bash diff --git a/examples/postgresql/blueprint-v2/postgres-blueprint.yaml b/examples/postgresql/blueprint-v2/postgres-blueprint.yaml index 1be3126ee7..09bd924c47 100644 --- a/examples/postgresql/blueprint-v2/postgres-blueprint.yaml +++ b/examples/postgresql/blueprint-v2/postgres-blueprint.yaml @@ -12,7 +12,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.pgDump.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: pgDump objects: pgSecret: @@ -20,9 +20,14 @@ actions: name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql' namespace: '{{ .StatefulSet.Namespace }}' args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 namespace: '{{ .StatefulSet.Namespace }}' - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: postgres:17-bullseye + backgroundCommand: - bash - -o - errexit @@ -33,8 +38,20 @@ actions: export PGHOST='{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql.{{ .StatefulSet.Namespace }}.svc.cluster.local' export PGUSER='postgres' export PGPASSWORD='{{ index .Phases.pgDump.Secrets.pgSecret.Data "postgres-password" | toString }}' + pg_dumpall --clean -U $PGUSER > /tmp/data + + outputImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | backup_file_path="backup.sql" - pg_dumpall --clean -U $PGUSER | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: kind: StatefulSet inputArtifactNames: @@ -42,7 +59,7 @@ actions: # Use the `--kopia-snapshot` flag in kando to pass in `pgBackup.KopiaSnapshot` - pgBackup phases: - - func: KubeTask + - func: MultiContainerRun name: pgRestore objects: pgSecret: @@ -50,9 +67,27 @@ actions: name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql' namespace: '{{ .StatefulSet.Namespace }}' args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 namespace: '{{ .StatefulSet.Namespace }}' - command: + sharedVolumeMedium: Memory + + initImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data; chmod 777 /tmp/data"] + + backgroundImage: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' + backgroundCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path="backup.sql" + kopia_snap='{{ .ArtifactsIn.pgBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: postgres:17-bullseye + outputCommand: - bash - -o - errexit @@ -63,9 +98,8 @@ actions: export PGHOST='{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql.{{ .StatefulSet.Namespace }}.svc.cluster.local' export PGUSER='postgres' export PGPASSWORD='{{ index .Phases.pgRestore.Secrets.pgSecret.Data "postgres-password" | toString }}' - backup_file_path="backup.sql" - kopia_snap='{{ .ArtifactsIn.pgBackup.KopiaSnapshot }}' - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | psql -q -U "${PGUSER}" + cat /tmp/data | psql -q -U "${PGUSER}" + delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -75,7 +109,7 @@ actions: - func: KubeTask name: deleteDump args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 + image: '{{if index .Options "kanisterImage" }} {{- .Options.kanisterImage -}} {{else -}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{- end}}' namespace: "{{ .Namespace.Name }}" command: - bash