From 22ecadd785f8212587242380f1b7de4d0aefa70b Mon Sep 17 00:00:00 2001 From: Daniil Fedotov Date: Fri, 29 Nov 2024 16:42:24 -0500 Subject: [PATCH] docs: Rework blueprints to separate DB tools image and kanister-tools image Separation of images allows us to build less images in kanister release process and allows the users of those blueprints to always use up to data databse images. Signed-off-by: Daniil Fedotov --- .../blueprint-v2/elasticsearch-blueprint.yaml | 58 +++++++++++++++---- .../k8s/etcd-incluster-blueprint.yaml | 31 +++++++--- .../etcd-incluster-ocp-blueprint.yaml | 32 +++++++--- .../mongo-dep-config-blueprint.yaml | 54 +++++++++++++---- .../mongodb/blueprint-v2/mongo-blueprint.yaml | 55 ++++++++++++++---- .../mssql/blueprint-v2/mssql-blueprint.yaml | 54 +++++++++++++---- .../mysql-dep-config-blueprint.yaml | 57 ++++++++++++++---- .../postgres-dep-config-blueprint.yaml | 58 +++++++++++++++---- .../hook-blueprint/postgres-ha-hook.yaml | 2 +- .../blueprint-v2/postgres-blueprint.yaml | 56 ++++++++++++++---- 10 files changed, 365 insertions(+), 92 deletions(-) diff --git a/examples/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml b/examples/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml index 59334d3dd9..e0a52f3e03 100644 --- a/examples/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml +++ b/examples/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml @@ -11,17 +11,23 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.backupToStore.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: backupToStore objects: esMasterCredSecret: kind: Secret name: "{{ index .Object.metadata.labels.app }}-credentials" namespace: "{{ .StatefulSet.Namespace }}" + args: namespace: "{{ .StatefulSet.Namespace }}" - image: "ghcr.io/kanisterio/es-sidecar:0.112.0" - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: elasticdump/elasticsearch-dump:latest + backgroundCommand: - bash - -o - errexit @@ -33,25 +39,41 @@ actions: backup_file_path='backup.gz' master_username="{{ index .Phases.backupToStore.Secrets.esMasterCredSecret.Data "username" | toString }}" master_password="{{ index .Phases.backupToStore.Secrets.esMasterCredSecret.Data "password" | toString }}" - NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=https://${master_username}:${master_password}@${host_name}:9200 --output=/backup - gzip -c /backup | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=https://${master_username}:${master_password}@${host_name}:9200 --output=$ > /tmp/data + + ouptutImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + cat /tmp/data | gzip -c | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + + restore: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here # Use the `--kopia-snapshot` flag in kando to pass in `esBackup.KopiaSnapshot` - esBackup phases: - - func: KubeTask + - func: MultiContainerRun name: restoreFromObjectStore objects: esMasterCredSecret: kind: Secret name: "{{ index .Object.metadata.labels.app }}-credentials" - namespace: "{{ .StatefulSet.Namespace }}" + namespace: "{{ .StatefulSet.Namespace }}" args: namespace: "{{ .StatefulSet.Namespace }}" - image: "ghcr.io/kanisterio/es-sidecar:0.112.0" - command: + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: - bash - -o - errexit @@ -59,12 +81,24 @@ actions: - pipefail - -c - | - host_name="{{ .Object.spec.serviceName }}.{{ .StatefulSet.Namespace }}.svc.cluster.local" backup_file_path='backup.gz' kopia_snap='{{ .ArtifactsIn.esBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | gunzip -c > /tmp/data + + ouptutImage: elasticdump/elasticsearch-dump:latest + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + host_name="{{ .Object.spec.serviceName }}.{{ .StatefulSet.Namespace }}.svc.cluster.local" master_username="{{ index .Phases.restoreFromObjectStore.Secrets.esMasterCredSecret.Data "username" | toString }}" master_password="{{ index .Phases.restoreFromObjectStore.Secrets.esMasterCredSecret.Data "password" | toString }}" - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | gunzip -c | NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=$ --output=https://${master_username}:${master_password}@${host_name}:9200 + cat /tmp/data | NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=$ --output=https://${master_username}:${master_password}@${host_name}:9200 + delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -75,7 +109,7 @@ actions: name: deleteFromStore args: namespace: "{{ .Namespace.Name }}" - image: "ghcr.io/kanisterio/es-sidecar:0.112.0" + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} command: - bash - -o diff --git a/examples/etcd/etcd-in-cluster/k8s/etcd-incluster-blueprint.yaml b/examples/etcd/etcd-in-cluster/k8s/etcd-incluster-blueprint.yaml index e57dc70164..0f6d49d7ce 100644 --- a/examples/etcd/etcd-in-cluster/k8s/etcd-incluster-blueprint.yaml +++ b/examples/etcd/etcd-in-cluster/k8s/etcd-incluster-blueprint.yaml @@ -34,11 +34,16 @@ actions: kando output etcdPod $ETCD_POD kando output etcdNS $ETCDNS - - func: KubeTask + - func: MultiContainerRun name: uploadSnapshot args: - image: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0 + backgroundCommand: - sh - -o - errexit @@ -46,10 +51,22 @@ actions: - pipefail - -c - | - BACKUP_LOCATION=etcd_backups/{{ .Object.metadata.namespace }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02T15:04:05Z07:00" }}/etcd-backup.db.gz kubectl cp {{ .Phases.takeSnapshot.Output.etcdNS }}/{{ .Phases.takeSnapshot.Output.etcdPod }}:/tmp/etcd-backup.db /tmp/etcd-backup.db - gzip /tmp/etcd-backup.db - kando location push --profile '{{ toJson .Profile }}' /tmp/etcd-backup.db.gz --path $BACKUP_LOCATION + gzip -c /tmp/etcd-backup.db > /tmp/data + + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - sh + - -o + - errexit + - -o + - pipefail + - -c + - | + BACKUP_LOCATION=etcd_backups/{{ .Object.metadata.namespace }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02T15:04:05Z07:00" }}/etcd-backup.db.gz + + cat /tmp/data > kando location push --profile '{{ toJson .Profile }}' --path $BACKUP_LOCATION - kando output backupLocation $BACKUP_LOCATION - func: KubeTask @@ -74,7 +91,7 @@ actions: name: deleteFromObjectStore args: namespace: "{{ .Namespace.Name }}" - image: "ghcr.io/kanisterio/kanister-tools:0.112.0" + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} command: - bash - -o diff --git a/examples/etcd/etcd-in-cluster/ocp/blueprint-v2/etcd-incluster-ocp-blueprint.yaml b/examples/etcd/etcd-in-cluster/ocp/blueprint-v2/etcd-incluster-ocp-blueprint.yaml index 21da6e476c..0694e8cece 100644 --- a/examples/etcd/etcd-in-cluster/ocp/blueprint-v2/etcd-incluster-ocp-blueprint.yaml +++ b/examples/etcd/etcd-in-cluster/ocp/blueprint-v2/etcd-incluster-ocp-blueprint.yaml @@ -31,11 +31,16 @@ actions: kando output etcdPod $ETCD_POD kando output etcdNS $etcdns - - func: KubeTask + - func: MultiContainerRun name: uploadSnapshot args: - image: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0 + backgroundCommand: - sh - -o - errexit @@ -43,9 +48,22 @@ actions: - pipefail - -c - | - BACKUP_LOCATION='etcd-backup.db.gz' kubectl cp -c etcd {{ .Phases.takeSnapshot.Output.etcdNS }}/{{ .Phases.takeSnapshot.Output.etcdPod }}:/tmp/etcd-backup.db /tmp/etcd-backup.db - gzip -c /tmp/etcd-backup.db | kando location push --profile '{{ toJson .Profile }}' --path "${BACKUP_LOCATION}" --output-name "kopiaOutput" - + gzip -c /tmp/etcd-backup.db > /tmp/data + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - sh + - -o + - errexit + - -o + - pipefail + - -c + - | + BACKUP_LOCATION='etcd-backup.db.gz' + + cat /tmp/data > | kando location push --profile '{{ toJson .Profile }}' --path "${BACKUP_LOCATION}" --output-name "kopiaOutput" - + kando output backupLocation $BACKUP_LOCATION - func: KubeTask name: removeSnapshot @@ -73,7 +91,7 @@ actions: - func: PrepareData name: copyFromObjectStore args: - image: "ghcr.io/kanisterio/kanister-tools:0.112.0" + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} namespace: "{{ .Object.metadata.namespace }}" podOverride: nodeSelector: @@ -108,7 +126,7 @@ actions: name: deleteFromObjectStore args: namespace: "{{ .Namespace.Name }}" - image: "ghcr.io/kanisterio/kanister-tools:0.112.0" + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} command: - bash - -o diff --git a/examples/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml b/examples/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml index 3a99be783d..ac98fc9755 100644 --- a/examples/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml +++ b/examples/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml @@ -11,7 +11,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.mongoDump.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: mongoDump objects: mongosecret: @@ -20,8 +20,13 @@ actions: namespace: "{{ .DeploymentConfig.Namespace }}" args: namespace: "{{ .DeploymentConfig.Namespace }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: bitnami/mongodb:7.0-debian-12 + backgroundCommand: - bash - -o - errexit @@ -33,15 +38,26 @@ actions: dbPassword='{{ index .Phases.mongoDump.Secrets.mongosecret.Data "database-admin-password" | toString }}' dump_cmd="mongodump --gzip --archive --host ${host} -u admin -p ${dbPassword}" echo $dump_cmd + ${dump_cmd} > /tmp/data + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | backup_file_path='rs_backup.gz' - ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - restore: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here # Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot` - mongoBackup phases: - - func: KubeTask + - func: MultiContainerRun name: pullFromStore objects: mongosecret: @@ -50,8 +66,26 @@ actions: namespace: "{{ .DeploymentConfig.Namespace }}" args: namespace: "{{ .DeploymentConfig.Namespace }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path='rs_backup.gz' + kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: bitnami/mongodb:7.0-debian-12 + outputCommand: - bash - -o - errexit @@ -62,9 +96,7 @@ actions: host="{{ .DeploymentConfig.Name }}.{{ .DeploymentConfig.Namespace }}.svc.cluster.local" dbPassword='{{ index .Phases.pullFromStore.Secrets.mongosecret.Data "database-admin-password" | toString }}' restore_cmd="mongorestore --gzip --archive --drop --host ${host} -u admin -p ${dbPassword}" - backup_file_path='rs_backup.gz' - kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}' - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd} + cat /tmp/data | ${restore_cmd} delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -75,7 +107,7 @@ actions: name: deleteFromStore args: namespace: "{{ .Namespace.Name }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} command: - bash - -o diff --git a/examples/mongodb/blueprint-v2/mongo-blueprint.yaml b/examples/mongodb/blueprint-v2/mongo-blueprint.yaml index 12ce0ccbea..40e0a6a8dd 100644 --- a/examples/mongodb/blueprint-v2/mongo-blueprint.yaml +++ b/examples/mongodb/blueprint-v2/mongo-blueprint.yaml @@ -11,7 +11,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.takeConsistentBackup.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: takeConsistentBackup objects: mongosecret: @@ -20,8 +20,13 @@ actions: namespace: "{{ .StatefulSet.Namespace }}" args: namespace: "{{ .StatefulSet.Namespace }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: bitnami/mongodb:7.0-debian-12 + backgroundCommand: - bash - -o - errexit @@ -32,15 +37,27 @@ actions: host='{{ .StatefulSet.Name }}-0.{{ .StatefulSet.Name }}-headless.{{ .StatefulSet.Namespace }}.svc.cluster.local' dbPassword='{{ index .Phases.takeConsistentBackup.Secrets.mongosecret.Data "mongodb-root-password" | toString }}' dump_cmd="mongodump --oplog --gzip --archive --host ${host} -u root -p ${dbPassword}" + ${dump_cmd} > /tmp/data + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | backup_file_path='rs_backup.gz' - ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here # Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot` - mongoBackup phases: - - func: KubeTask + - func: MultiContainerRun name: pullFromStore objects: mongosecret: @@ -49,8 +66,26 @@ actions: namespace: "{{ .StatefulSet.Namespace }}" args: namespace: "{{ .StatefulSet.Namespace }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path='rs_backup.gz' + kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: bitnami/mongodb:7.0-debian-12 + outputCommand: - bash - -o - errexit @@ -61,9 +96,7 @@ actions: host='{{ .StatefulSet.Name }}-0.{{ .StatefulSet.Name }}-headless.{{ .StatefulSet.Namespace }}.svc.cluster.local' dbPassword='{{ index .Phases.pullFromStore.Secrets.mongosecret.Data "mongodb-root-password" | toString }}' restore_cmd="mongorestore --gzip --archive --oplogReplay --drop --host ${host} -u root -p ${dbPassword}" - backup_file_path='rs_backup.gz' - kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}' - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd} + cat /tmp/data | ${restore_cmd} delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -74,7 +107,7 @@ actions: name: deleteFromStore args: namespace: "{{ .Namespace.Name }}" - image: ghcr.io/kanisterio/mongodb:0.112.0 + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} command: - bash - -o diff --git a/examples/mssql/blueprint-v2/mssql-blueprint.yaml b/examples/mssql/blueprint-v2/mssql-blueprint.yaml index 547b084e24..bf37d136cb 100644 --- a/examples/mssql/blueprint-v2/mssql-blueprint.yaml +++ b/examples/mssql/blueprint-v2/mssql-blueprint.yaml @@ -8,7 +8,7 @@ actions: mssqlCloudDump: kopiaSnapshot: "{{ .Phases.dumpToObjectStore.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: dumpToObjectStore objects: mssql: @@ -16,8 +16,13 @@ actions: name: '{{ index .Object.metadata.labels "app" }}' namespace: '{{ .Deployment.Namespace }}' args: - image: ghcr.io/kanisterio/mssql-tools:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: ghcr.io/kanisterio/mssql-tools:0.112.0 + backgroundCommand: - bash - -o - errexit @@ -25,7 +30,6 @@ actions: - pipefail - -c - | - kopiaPath="backup.tar.gz" root_password="{{ index .Phases.dumpToObjectStore.Secrets.mssql.Data "SA_PASSWORD" | toString }}" root_uname="sa" mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}" @@ -33,13 +37,26 @@ actions: databases=$(/opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "SET NOCOUNT ON; SELECT name FROM sys.databases WHERE name NOT IN ('master','model','msdb','tempdb')" -b -s "," -h -1) for database in $databases; do /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "backup database $database to disk = '/tmp/backup/$database.bak' with format;"; done kubectl cp ${mssql_pod}:/tmp/backup /tmp/backup - tar zcvf - -C /tmp/ backup | kando location push --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --output-name "kopiaOutput" - + tar zcvf - -C /tmp/ backup > /tmp/data kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + kopiaPath="backup.tar.gz" + cat /tmp/data > kando location push --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --output-name "kopiaOutput" - + restore: inputArtifactNames: - mssqlCloudDump phases: - - func: KubeTask + - func: MultiContainerRun name: restoreFromObjectStore objects: mssql: @@ -47,8 +64,13 @@ actions: name: '{{ index .Object.metadata.labels "app" }}' namespace: '{{ .Deployment.Namespace }}' args: - image: ghcr.io/kanisterio/mssql-tools:0.112.0 - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: - bash - -o - errexit @@ -58,15 +80,27 @@ actions: - | kopiaPath="backup.tar.gz" kopia_snap='{{ .ArtifactsIn.mssqlCloudDump.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --kopia-snapshot ${kopia_snap} - > /tmp/data + + outputImage: ghcr.io/kanisterio/mssql-tools:0.112.0 + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | root_password="{{ index .Phases.restoreFromObjectStore.Secrets.mssql.Data "SA_PASSWORD" | toString }}" root_uname="sa" mssql_pod="{{ index .Deployment.Namespace }}/{{ index .Deployment.Pods 0 }}" server_name="{{ index .Deployment.Name }}.{{ index .Deployment.Namespace }}.svc.cluster.local" - kando location pull --profile '{{ toJson .Profile }}' --path "${kopiaPath}" --kopia-snapshot ${kopia_snap} - | tar zxvf - -C /tmp/ + cat /tmp/data > tar zxvf - -C /tmp/ kubectl cp /tmp/backup ${mssql_pod}:/tmp/backup backup_files=$(ls /tmp/backup) for script in $backup_files; do database="$(cut -d'.' -f1 <<<"$script")"; /opt/mssql-tools/bin/sqlcmd -S ${server_name} -U ${root_uname} -P ${root_password} -Q "restore database $database from disk = '/tmp/backup/$script' with replace"; done kubectl exec -it {{ index .Deployment.Pods 0 }} -n {{ index .Deployment.Namespace }} -- rm -r /tmp/backup + delete: inputArtifactNames: - mssqlCloudDump @@ -74,7 +108,7 @@ actions: - func: KubeTask name: deleteFromBlobStore args: - image: ghcr.io/kanisterio/mssql-tools:0.112.0 + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} command: - bash - -o diff --git a/examples/mysql-deploymentconfig/blueprint-v2/mysql-dep-config-blueprint.yaml b/examples/mysql-deploymentconfig/blueprint-v2/mysql-dep-config-blueprint.yaml index f88d45cfe2..4eca3e3a90 100644 --- a/examples/mysql-deploymentconfig/blueprint-v2/mysql-dep-config-blueprint.yaml +++ b/examples/mysql-deploymentconfig/blueprint-v2/mysql-dep-config-blueprint.yaml @@ -11,7 +11,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.dumpToStore.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: dumpToStore objects: mysqlsecret: @@ -19,9 +19,15 @@ actions: name: "{{ .DeploymentConfig.Name }}" namespace: "{{ .DeploymentConfig.Namespace }}" args: - image: ghcr.io/kanisterio/mysql-sidecar:0.112.0 namespace: "{{ .DeploymentConfig.Namespace }}" - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + ## FIXME: do we want to use mysql 9? + backgroundImage: mysql:8 + backgroundCommand: - bash - -o - errexit @@ -29,17 +35,29 @@ actions: - pipefail - -c - | - backup_file_path="dump.sql" root_password="{{ index .Phases.dumpToStore.Secrets.mysqlsecret.Data "database-root-password" | toString }}" dump_cmd="mysqldump --column-statistics=0 -u root --password=${root_password} -h {{ .DeploymentConfig.Name }} --single-transaction --all-databases" - ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + ${dump_cmd} > /tmp/data + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path="dump.sql" + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here # Use the `--kopia-snapshot` flag in kando to pass in `mysqlBackup.KopiaSnapshot` - mysqlBackup phases: - - func: KubeTask + - func: MultiContainerRun name: restoreFromStore objects: mysqlsecret: @@ -47,9 +65,15 @@ actions: name: "{{ .DeploymentConfig.Name }}" namespace: "{{ .DeploymentConfig.Namespace }}" args: - image: ghcr.io/kanisterio/mysql-sidecar:0.112.0 namespace: "{{ .DeploymentConfig.Namespace }}" - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: - bash - -o - errexit @@ -59,9 +83,22 @@ actions: - | backup_file_path="dump.sql" kopia_snap='{{ .ArtifactsIn.mysqlBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + ## FIXME: do we want to use mysql 9? + outputImage: mysql:8 + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | root_password="{{ index .Phases.restoreFromStore.Secrets.mysqlsecret.Data "database-root-password" | toString }}" restore_cmd="mysql -u root --password=${root_password} -h {{ .DeploymentConfig.Name }}" - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd} + cat /tmp/data | ${restore_cmd} + delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -71,7 +108,7 @@ actions: - func: KubeTask name: deleteFromStore args: - image: ghcr.io/kanisterio/mysql-sidecar:0.112.0 + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} namespace: "{{ .Namespace.Name }}" command: - bash diff --git a/examples/postgresql-deploymentconfig/blueprint-v2/postgres-dep-config-blueprint.yaml b/examples/postgresql-deploymentconfig/blueprint-v2/postgres-dep-config-blueprint.yaml index 30aec9ea86..16dec48098 100644 --- a/examples/postgresql-deploymentconfig/blueprint-v2/postgres-dep-config-blueprint.yaml +++ b/examples/postgresql-deploymentconfig/blueprint-v2/postgres-dep-config-blueprint.yaml @@ -12,7 +12,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.pgDump.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: pgDump objects: pgSecret: @@ -20,9 +20,14 @@ actions: name: '{{ .DeploymentConfig.Name }}-{{ .DeploymentConfig.Namespace }}' namespace: '{{ .DeploymentConfig.Namespace }}' args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 namespace: '{{ .DeploymentConfig.Namespace }}' - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: postgres:17-bullseye + backgroundCommand: - bash - -o - errexit @@ -33,8 +38,20 @@ actions: export PGHOST='{{ .DeploymentConfig.Name }}.{{ .DeploymentConfig.Namespace }}.svc.cluster.local' export PGUSER='postgres' export PGPASSWORD='{{ index .Phases.pgDump.Secrets.pgSecret.Data "postgresql_admin_password" | toString }}' - backup_file_path="backup.sql" - pg_dumpall --clean -U $PGUSER | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + pg_dumpall --clean -U $PGUSER > /tmp/data + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path="backup.sql" + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: kind: DeploymentConfig inputArtifactNames: @@ -42,7 +59,7 @@ actions: # Use the `--kopia-snapshot` flag in kando to pass in `pgBackup.KopiaSnapshot` - pgBackup phases: - - func: KubeTask + - func: MultiContainerRun name: pgRestore objects: pgSecret: @@ -50,9 +67,27 @@ actions: name: '{{ .DeploymentConfig.Name }}-{{ .DeploymentConfig.Namespace }}' namespace: '{{ .DeploymentConfig.Namespace }}' args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 namespace: '{{ .DeploymentConfig.Namespace }}' - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path="backup.sql" + kopia_snap='{{ .ArtifactsIn.pgBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: postgres:17-bullseye + outputCommand: - bash - -o - errexit @@ -63,9 +98,8 @@ actions: export PGHOST='{{ .DeploymentConfig.Name }}.{{ .DeploymentConfig.Namespace }}.svc.cluster.local' export PGUSER='postgres' export PGPASSWORD='{{ index .Phases.pgRestore.Secrets.pgSecret.Data "postgresql_admin_password" | toString }}' - backup_file_path="backup.sql" - kopia_snap='{{ .ArtifactsIn.pgBackup.KopiaSnapshot }}' - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | psql -q -U "${PGUSER}" + cat /tmp/data | psql -q -U "${PGUSER}" + delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -75,7 +109,7 @@ actions: - func: KubeTask name: deleteDump args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} namespace: "{{ .Namespace.Name }}" command: - bash diff --git a/examples/postgresql-ha/hook-blueprint/postgres-ha-hook.yaml b/examples/postgresql-ha/hook-blueprint/postgres-ha-hook.yaml index 3dcf4b9324..6e153c05d8 100644 --- a/examples/postgresql-ha/hook-blueprint/postgres-ha-hook.yaml +++ b/examples/postgresql-ha/hook-blueprint/postgres-ha-hook.yaml @@ -26,7 +26,7 @@ actions: namespace: '{{ .StatefulSet.Namespace }}' args: namespace: '{{ .StatefulSet.Namespace }}' - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 + image: {{if .Options.psqlImage }} {{.Options.psqlImage}} {{else}} postgres:17-bullseye {{end}} command: - bash - -o diff --git a/examples/postgresql/blueprint-v2/postgres-blueprint.yaml b/examples/postgresql/blueprint-v2/postgres-blueprint.yaml index 1be3126ee7..be3df10784 100644 --- a/examples/postgresql/blueprint-v2/postgres-blueprint.yaml +++ b/examples/postgresql/blueprint-v2/postgres-blueprint.yaml @@ -12,7 +12,7 @@ actions: # `kopiaOutput` is the name provided to kando using `--output-name` flag kopiaSnapshot: "{{ .Phases.pgDump.Output.kopiaOutput }}" phases: - - func: KubeTask + - func: MultiContainerRun name: pgDump objects: pgSecret: @@ -20,9 +20,14 @@ actions: name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql' namespace: '{{ .StatefulSet.Namespace }}' args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 namespace: '{{ .StatefulSet.Namespace }}' - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: postgres:17-bullseye + backgroundCommand: - bash - -o - errexit @@ -33,8 +38,20 @@ actions: export PGHOST='{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql.{{ .StatefulSet.Namespace }}.svc.cluster.local' export PGUSER='postgres' export PGPASSWORD='{{ index .Phases.pgDump.Secrets.pgSecret.Data "postgres-password" | toString }}' + pg_dumpall --clean -U $PGUSER > /tmp/data + + outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + outputCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | backup_file_path="backup.sql" - pg_dumpall --clean -U $PGUSER | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" - + restore: kind: StatefulSet inputArtifactNames: @@ -42,7 +59,7 @@ actions: # Use the `--kopia-snapshot` flag in kando to pass in `pgBackup.KopiaSnapshot` - pgBackup phases: - - func: KubeTask + - func: MultiContainerRun name: pgRestore objects: pgSecret: @@ -50,9 +67,27 @@ actions: name: '{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql' namespace: '{{ .StatefulSet.Namespace }}' args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 namespace: '{{ .StatefulSet.Namespace }}' - command: + sharedVolumeMedium: Memory + + initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"] + + backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} + backgroundCommand: + - bash + - -o + - errexit + - -o + - pipefail + - -c + - | + backup_file_path="backup.sql" + kopia_snap='{{ .ArtifactsIn.pgBackup.KopiaSnapshot }}' + kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data + + outputImage: postgres:17-bullseye + outputCommand: - bash - -o - errexit @@ -63,9 +98,8 @@ actions: export PGHOST='{{ index .Object.metadata.labels "app.kubernetes.io/instance" }}-postgresql.{{ .StatefulSet.Namespace }}.svc.cluster.local' export PGUSER='postgres' export PGPASSWORD='{{ index .Phases.pgRestore.Secrets.pgSecret.Data "postgres-password" | toString }}' - backup_file_path="backup.sql" - kopia_snap='{{ .ArtifactsIn.pgBackup.KopiaSnapshot }}' - kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | psql -q -U "${PGUSER}" + cat /tmp/data > psql -q -U "${PGUSER}" + delete: inputArtifactNames: # The kopia snapshot info created in backup phase can be used here @@ -75,7 +109,7 @@ actions: - func: KubeTask name: deleteDump args: - image: ghcr.io/kanisterio/postgres-kanister-tools:0.112.0 + image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}} namespace: "{{ .Namespace.Name }}" command: - bash