docs: Rework blueprints to separate DB tools image and kanister-tools…

… image Separation of images allows us to build less images in kanister release process and allows the users of those blueprints to always use up to data databse images. Signed-off-by: Daniil Fedotov <[email protected]>
kanisterio · Nov 29, 2024 · 22ecadd · 22ecadd
1 parent f3f3360
commit 22ecadd
Show file tree

Hide file tree

Showing 10 changed files with 365 additions and 92 deletions.
diff --git a/examples/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml b/examples/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml
@@ -11,17 +11,23 @@ actions:
         # `kopiaOutput` is the name provided to kando using `--output-name` flag
         kopiaSnapshot: "{{ .Phases.backupToStore.Output.kopiaOutput }}"
     phases:
-    - func: KubeTask
+    - func: MultiContainerRun
       name: backupToStore
       objects:
         esMasterCredSecret:
           kind: Secret
           name: "{{ index .Object.metadata.labels.app }}-credentials"
           namespace: "{{ .StatefulSet.Namespace }}"
+
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "ghcr.io/kanisterio/es-sidecar:0.112.0"
-        command:
+        sharedVolumeMedium: Memory
+
+        initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"]
+
+        backgroundImage: elasticdump/elasticsearch-dump:latest
+        backgroundCommand:
         - bash
         - -o
         - errexit
@@ -33,38 +39,66 @@ actions:
           backup_file_path='backup.gz'
           master_username="{{ index .Phases.backupToStore.Secrets.esMasterCredSecret.Data "username" | toString }}"
           master_password="{{ index .Phases.backupToStore.Secrets.esMasterCredSecret.Data "password" | toString }}"
-          NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=https://${master_username}:${master_password}@${host_name}:9200 --output=/backup
-          gzip -c /backup | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
+          NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=https://${master_username}:${master_password}@${host_name}:9200 --output=$ > /tmp/data
+
+        ouptutImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        outputCommand:
+        - bash
+        - -o
+        - errexit
+        - -o
+        - pipefail
+        - -c
+        - |
+          cat /tmp/data | gzip -c | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
+
+
   restore:
     inputArtifactNames:
     # The kopia snapshot info created in backup phase can be used here
     # Use the `--kopia-snapshot` flag in kando to pass in `esBackup.KopiaSnapshot`
     - esBackup
     phases:
-    - func: KubeTask
+    - func: MultiContainerRun
       name: restoreFromObjectStore
       objects:
         esMasterCredSecret:
           kind: Secret
           name: "{{ index .Object.metadata.labels.app }}-credentials"
-          namespace: "{{ .StatefulSet.Namespace }}"      
+          namespace: "{{ .StatefulSet.Namespace }}"
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "ghcr.io/kanisterio/es-sidecar:0.112.0"
-        command:
+
+        initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"]
+
+        backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        backgroundCommand:
         - bash
         - -o
         - errexit
         - -o
         - pipefail
         - -c
         - |
-          host_name="{{ .Object.spec.serviceName }}.{{ .StatefulSet.Namespace }}.svc.cluster.local"
           backup_file_path='backup.gz'
           kopia_snap='{{ .ArtifactsIn.esBackup.KopiaSnapshot }}'
+          kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | gunzip -c > /tmp/data
+
+        ouptutImage: elasticdump/elasticsearch-dump:latest
+        outputCommand:
+        - bash
+        - -o
+        - errexit
+        - -o
+        - pipefail
+        - -c
+        - |
+          host_name="{{ .Object.spec.serviceName }}.{{ .StatefulSet.Namespace }}.svc.cluster.local"
           master_username="{{ index .Phases.restoreFromObjectStore.Secrets.esMasterCredSecret.Data "username" | toString }}"
           master_password="{{ index .Phases.restoreFromObjectStore.Secrets.esMasterCredSecret.Data "password" | toString }}"
-          kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | gunzip -c | NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=$ --output=https://${master_username}:${master_password}@${host_name}:9200
+          cat /tmp/data | NODE_TLS_REJECT_UNAUTHORIZED=0 elasticdump --bulk=true --input=$ --output=https://${master_username}:${master_password}@${host_name}:9200
+
   delete:
     inputArtifactNames:
     # The kopia snapshot info created in backup phase can be used here
@@ -75,7 +109,7 @@ actions:
       name: deleteFromStore
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: "ghcr.io/kanisterio/es-sidecar:0.112.0"
+        image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
         command:
         - bash
         - -o

diff --git a/examples/etcd/etcd-in-cluster/k8s/etcd-incluster-blueprint.yaml b/examples/etcd/etcd-in-cluster/k8s/etcd-incluster-blueprint.yaml
@@ -34,22 +34,39 @@ actions:
             kando output etcdPod $ETCD_POD
             kando output etcdNS $ETCDNS
 
-    - func: KubeTask
+    - func: MultiContainerRun
       name: uploadSnapshot
       args:
-        image: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0
-        command:
+        sharedVolumeMedium: Memory
+
+        initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"]
+
+        backgroundImage: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0
+        backgroundCommand:
           - sh
           - -o
           - errexit
           - -o
           - pipefail
           - -c
           - |
-            BACKUP_LOCATION=etcd_backups/{{ .Object.metadata.namespace }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02T15:04:05Z07:00" }}/etcd-backup.db.gz
             kubectl cp {{ .Phases.takeSnapshot.Output.etcdNS }}/{{ .Phases.takeSnapshot.Output.etcdPod }}:/tmp/etcd-backup.db /tmp/etcd-backup.db
-            gzip /tmp/etcd-backup.db
-            kando location push --profile '{{ toJson .Profile }}'  /tmp/etcd-backup.db.gz --path $BACKUP_LOCATION
+            gzip -c /tmp/etcd-backup.db > /tmp/data
+
+
+        outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        outputCommand:
+          - sh
+          - -o
+          - errexit
+          - -o
+          - pipefail
+          - -c
+          - |
+            BACKUP_LOCATION=etcd_backups/{{ .Object.metadata.namespace }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02T15:04:05Z07:00" }}/etcd-backup.db.gz
+
+            cat /tmp/data > kando location push --profile '{{ toJson .Profile }}' --path $BACKUP_LOCATION -
             kando output backupLocation $BACKUP_LOCATION
 
     - func: KubeTask
@@ -74,7 +91,7 @@ actions:
       name: deleteFromObjectStore
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: "ghcr.io/kanisterio/kanister-tools:0.112.0"
+        image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
         command:
         - bash
         - -o

diff --git a/examples/etcd/etcd-in-cluster/ocp/blueprint-v2/etcd-incluster-ocp-blueprint.yaml b/examples/etcd/etcd-in-cluster/ocp/blueprint-v2/etcd-incluster-ocp-blueprint.yaml
@@ -31,21 +31,39 @@ actions:
             kando output etcdPod $ETCD_POD
             kando output etcdNS $etcdns
 
-    - func: KubeTask
+    - func: MultiContainerRun
       name: uploadSnapshot
       args:
-        image: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0
-        command:
+        sharedVolumeMedium: Memory
+
+        initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"]
+
+        backgroundImage: ghcr.io/kanisterio/kanister-kubectl-1.18:0.112.0
+        backgroundCommand:
           - sh
           - -o
           - errexit
           - -o
           - pipefail
           - -c
           - |
-            BACKUP_LOCATION='etcd-backup.db.gz'
             kubectl cp -c etcd {{ .Phases.takeSnapshot.Output.etcdNS }}/{{ .Phases.takeSnapshot.Output.etcdPod }}:/tmp/etcd-backup.db /tmp/etcd-backup.db
-            gzip -c /tmp/etcd-backup.db  | kando location push --profile '{{ toJson .Profile }}' --path "${BACKUP_LOCATION}" --output-name "kopiaOutput" -
+            gzip -c /tmp/etcd-backup.db  > /tmp/data
+
+        outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        outputCommand:
+          - sh
+          - -o
+          - errexit
+          - -o
+          - pipefail
+          - -c
+          - |
+            BACKUP_LOCATION='etcd-backup.db.gz'
+
+            cat /tmp/data > | kando location push --profile '{{ toJson .Profile }}' --path "${BACKUP_LOCATION}" --output-name "kopiaOutput" -
+            kando output backupLocation $BACKUP_LOCATION
 
     - func: KubeTask
       name: removeSnapshot
@@ -73,7 +91,7 @@ actions:
     - func: PrepareData
       name: copyFromObjectStore
       args:
-        image: "ghcr.io/kanisterio/kanister-tools:0.112.0"
+        image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
         namespace: "{{ .Object.metadata.namespace }}"
         podOverride:
           nodeSelector:
@@ -108,7 +126,7 @@ actions:
       name: deleteFromObjectStore
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: "ghcr.io/kanisterio/kanister-tools:0.112.0"
+        image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
         command:
         - bash
         - -o

diff --git a/examples/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml b/examples/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml
@@ -11,7 +11,7 @@ actions:
         # `kopiaOutput` is the name provided to kando using `--output-name` flag
         kopiaSnapshot: "{{ .Phases.mongoDump.Output.kopiaOutput }}"
     phases:
-    - func: KubeTask
+    - func: MultiContainerRun
       name: mongoDump
       objects:
         mongosecret:
@@ -20,8 +20,13 @@ actions:
           namespace: "{{ .DeploymentConfig.Namespace }}"
       args:
         namespace: "{{ .DeploymentConfig.Namespace }}"
-        image: ghcr.io/kanisterio/mongodb:0.112.0
-        command:
+        sharedVolumeMedium: Memory
+
+        initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"]
+
+        backgroundImage: bitnami/mongodb:7.0-debian-12
+        backgroundCommand:
           - bash
           - -o
           - errexit
@@ -33,15 +38,26 @@ actions:
             dbPassword='{{ index .Phases.mongoDump.Secrets.mongosecret.Data "database-admin-password" | toString }}'
             dump_cmd="mongodump --gzip --archive --host ${host} -u admin -p ${dbPassword}"
             echo $dump_cmd
+            ${dump_cmd} > /tmp/data
+
+        outputImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        outputCommand:
+          - bash
+          - -o
+          - errexit
+          - -o
+          - pipefail
+          - -c
+          - |
             backup_file_path='rs_backup.gz'
-            ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
+            cat /tmp/data | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
   restore:
     inputArtifactNames:
     # The kopia snapshot info created in backup phase can be used here
     # Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot`
       - mongoBackup
     phases:
-    - func: KubeTask
+    - func: MultiContainerRun
       name: pullFromStore
       objects:
         mongosecret:
@@ -50,8 +66,26 @@ actions:
           namespace: "{{ .DeploymentConfig.Namespace }}"
       args:
         namespace: "{{ .DeploymentConfig.Namespace }}"
-        image: ghcr.io/kanisterio/mongodb:0.112.0
-        command:
+        sharedVolumeMedium: Memory
+
+        initImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        initCommand: ["bash", "-o", "errexit", "-o", "pipefail", "-c", "mkfifo /tmp/data"]
+
+        backgroundImage: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
+        backgroundCommand:
+          - bash
+          - -o
+          - errexit
+          - -o
+          - pipefail
+          - -c
+          - |
+            backup_file_path='rs_backup.gz'
+            kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}'
+            kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - > /tmp/data
+
+        outputImage: bitnami/mongodb:7.0-debian-12
+        outputCommand:
           - bash
           - -o
           - errexit
@@ -62,9 +96,7 @@ actions:
             host="{{ .DeploymentConfig.Name }}.{{ .DeploymentConfig.Namespace }}.svc.cluster.local"
             dbPassword='{{ index .Phases.pullFromStore.Secrets.mongosecret.Data "database-admin-password" | toString }}'
             restore_cmd="mongorestore --gzip --archive --drop --host ${host} -u admin -p ${dbPassword}"
-            backup_file_path='rs_backup.gz'
-            kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}'
-            kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd}
+            cat /tmp/data | ${restore_cmd}
   delete:
     inputArtifactNames:
     # The kopia snapshot info created in backup phase can be used here
@@ -75,7 +107,7 @@ actions:
       name: deleteFromStore
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: ghcr.io/kanisterio/mongodb:0.112.0
+        image: {{if .Options.kanisterImage }} {{.Options.kanisterImage}} {{else}} ghcr.io/kanisterio/kanister-tools:0.112.0 {{end}}
         command:
           - bash
           - -o