Merge branch 'master' into kopiaKanisterIntegrationPhase2

.github/workflows/atlas-image-build.yaml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           fetch-depth: 0
-      - uses: tj-actions/changed-files@716b1e13042866565e00e85fd4ec490e186c4a2f # v41.0.1
+      - uses: tj-actions/changed-files@90a06d6ba9543371ab4df8eeca0be07ca6054959 # v42.0.2
         name: Get changed files
         id: changed-files
         with:
@@ -39,7 +39,7 @@ jobs:
       uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
     - name: Image metadata
       id: meta
-      uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v5.5.0
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
       with:
         images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
         tags: |

.github/workflows/dependendy-review.yml

@@ -17,4 +17,4 @@ jobs:
       - name: 'Checkout Repository'
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4

.github/workflows/grype-vulnerability-scanner.yaml

@@ -1,5 +1,13 @@
 name: container vulnerability scanning
-on: [workflow_dispatch]
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["Build and test"]
+    types:
+      - completed
+    branches:
+      - master
+
 jobs:
   vulnerability-scanner:
     runs-on: ubuntu-20.04

.github/workflows/kanister-image-build.yaml

@@ -44,7 +44,7 @@ jobs:
       uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
     - name: Image metadata
       id: meta
-      uses: docker/metadata-action@dbef88086f6cef02e264edb7dbf63250c17cef6c # v5.5.0
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
       with:
         images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
         tags: |

.github/workflows/ossf-scorecard.yml

@@ -39,12 +39,12 @@ jobs:
       -
         # Upload the results to GitHub's code scanning dashboard.
         name: "Upload to results to dashboard"
-        uses: github/codeql-action/upload-sarif@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0
+        uses: github/codeql-action/upload-sarif@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2
         with:
           sarif_file: results.sarif
       -
         name: "Upload analysis results as 'Job Artifact'"
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
         with:
           name: SARIF file
           path: results.sarif

.goreleaser.yml

@@ -64,8 +64,8 @@ dockers:
   dockerfile: 'docker/tools/Dockerfile'
   build_flag_templates:
   - "--build-arg=kan_tools_version={{ .Tag }}"
-# Refers to https://github.com/kopia/kopia/commit/fc640a98e4914e1da9fff6be1931ebfd767d3ee3
-  - "--build-arg=kopia_build_commit=fc640a9"
+# Refers to https://github.com/kopia/kopia/commit/1d6f852cd6534f4bea978cbdc85c583803d79f77
+  - "--build-arg=kopia_build_commit=1d6f852"
   - "--build-arg=kopia_repo_org=kopia"
   extra_files:
   - 'LICENSE'

BUILD.md

@@ -52,6 +52,14 @@ make test
 make build-controller
 ```
 
+To build kanister tools (kanctl and kando), use the following conmmand:
+```sh
+make build GOBORING=true BIN=<kanctl|kando> ARCH=<arm64|amd64>
+```
+
+This will build a selected binary `BIN` for a selected architecture `ARCH`.
+
+
 To build the controller OCI image:
 ```sh
 make release-controller \
@@ -100,6 +108,22 @@ helm upgrade kanister ./helm/kanister-operator \
 Most of the Makefile targets can work in a non-Docker development setup, by
 setting the `DOCKER_BUILD` variable to `false`.
 
+## Testing
+
+Kanister is using `check` library to extend go testing capabilities: https://github.com/kastenhq/check
+It's recommended to write new tests using this library for consistency.
+
+`make test` runs all tests in the repository.
+To run tests for specific package you can run `go test` in that package directory.
+It's recommended to do that in build image shell, you can run it with `make shell`.
+
+The `check` library handles arguments differently from standard `go test`
+- to run specific test, you can use `-check.f <test regex>` to filter test (or suite) names
+- to increase verbosity, you can use `-check.v` or `-check.vv`
+- to controll how many suites from the package run in parallel, you can use `-check.suitep <number>`
+
+See https://github.com/kastenhq/check and https://github.com/kastenhq/check/blob/v1/run.go#L30 for more information
+
 ## Documentation
 
 The source of the documentation is found in the `docs` folder. They are written

Dockerfile.in

@@ -1,4 +1,4 @@
-ARG base_image=registry.access.redhat.com/ubi9/ubi-minimal:9.3-1475
+ARG base_image=registry.access.redhat.com/ubi9/ubi-minimal:9.3-1552
 FROM ${base_image}
 ARG kanister_version
 

docker/controller/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3-1475
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3-1552
 
 LABEL maintainer="Tom Manville<[email protected]>"
 

docker/kanister-mongodb-replicaset/image/Dockerfile

@@ -7,6 +7,6 @@ ADD . /kanister
 
 RUN /kanister/install.sh && rm -rf /kanister && rm -rf /tmp && mkdir /tmp
 
-COPY --from=restic/restic:0.11.0 /usr/bin/restic /usr/local/bin/restic
+COPY --from=restic/restic:0.16.2 /usr/bin/restic /usr/local/bin/restic
 
 CMD ["tail", "-f", "/dev/null"]

docker/kanister-mysql/image/Dockerfile

@@ -1,12 +1,12 @@
 ARG TOOLS_IMAGE
-FROM registry.access.redhat.com/ubi9/ubi:9.1.0-1646.1669627755 as builder
+FROM registry.access.redhat.com/ubi9/ubi:9.3-1476 as builder
 
 RUN dnf clean all && rm -rf /var/cache/dnf
 RUN dnf -y upgrade
 # Download the RPM file to avoid timeouts during install
-RUN curl -LO https://dev.mysql.com/get/mysql80-community-release-el9-1.noarch.rpm
+RUN curl -LO https://dev.mysql.com/get/mysql80-community-release-el9-5.noarch.rpm
 # Install from the local file
-RUN dnf install -y mysql80-community-release-el9-1.noarch.rpm
+RUN dnf install -y mysql80-community-release-el9-5.noarch.rpm
 
 RUN dnf install -y mysql-community-client
 

docker/mongodb-atlas/Dockerfile

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3-1475
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3-1552
 
 RUN cat >/etc/yum.repos.d/mongodb.repo <<EOL
 [mongodb-org-6.0]

docker/tools/Dockerfile

@@ -74,7 +74,7 @@ RUN apt-get update && apt-get -y install ca-certificates && \
 USER kopia:kopia
 
 # Build tools image
-FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3-1475
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.3-1552
 ARG kan_tools_version="test-version"
 LABEL name="kanister-tools" \
     vendor="Kanister" \

docs/functions.rst

@@ -306,6 +306,7 @@ Arguments:
    `includePath`, Yes, `string`, path of the data to be backed up
    `backupArtifactPrefix`, Yes, `string`, path to store the backup on the object store
    `encryptionKey`, No, `string`, encryption key to be used for backups
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
 
 Outputs:
 
@@ -368,6 +369,7 @@ Arguments:
    `includePath`, Yes, `string`, path of the data to be backed up
    `backupArtifactPrefix`, Yes, `string`, path to store the backup on the object store appended by pod name later
    `encryptionKey`, No, `string`, encryption key to be used for backups
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
 
 Outputs:
 
@@ -430,6 +432,7 @@ and restores data to the specified path.
    `pod`, No, `string`, pod to which the volumes are attached
    `volumes`, No, `map[string]string`, Mapping of `pvcName` to `mountPath` under which the volume will be available
    `encryptionKey`, No, `string`, encryption key to be used during backups
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
    `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with
 
 .. note::
@@ -505,6 +508,7 @@ respective PVCs and restores data to the specified path.
    `pods`, No, `string`, pods to which the volumes are attached
    `encryptionKey`, No, `string`, encryption key to be used during backups
    `backupInfo`, Yes, `string`, snapshot info generated as output in BackupDataAll function
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
    `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with
 
 .. note::
@@ -575,6 +579,7 @@ Arguments:
    `volume`, Yes, `string`, name of the source PVC
    `dataArtifactPrefix`, Yes, `string`, path on the object store to store the data in
    `encryptionKey`, No, `string`, encryption key to be used during backups
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
    `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with
 
 Outputs:
@@ -620,6 +625,7 @@ This function deletes the snapshot data backed up by the :ref:`backupdata` funct
    `backupID`, No, `string`, (required if backupTag not provided) unique snapshot id generated during backup
    `backupTag`, No, `string`, (required if backupID not provided) unique tag added during the backup
    `encryptionKey`, No, `string`, encryption key to be used during backups
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
    `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with
 
 Example:
@@ -657,6 +663,7 @@ BackupDataAll function.
    `backupInfo`, Yes, `string`, snapshot info generated as output in BackupDataAll function
    `encryptionKey`, No, `string`, encryption key to be used during backups
    `reclaimSpace`, No, `bool`, provides a way to specify if space should be reclaimed
+   `insecureTLS`, No, `bool`, enables insecure connection for data mover
    `podOverride`, No, `map[string]interface{}`, specs to override default pod specs with
 
 Example:

examples/README.md

@@ -5,8 +5,7 @@ blueprints, to make experimenting with Kanister easier.
 
 If you are new to Kanister, start with the `time-log` blueprint. After that,
 feel free to try out some of the more advanced examples like the blueprints for
-MySQL, PgSQL, MongoDB and ElasticSearch, following the instructions in the
-[Kanister documentation](https://docs.kanister.io/helm.html#kanister-enabled-applications).
+MySQL, PgSQL, MongoDB and ElasticSearch, following the instructions in the corresponding folder.
 
 Every example has its own README.md with information on how to set things up and
 commands to be run.

examples/aws-rds/postgresql/README.md

@@ -9,7 +9,7 @@ This example is to demonstrate how Kanister can be integrated with AWS RDS insta
 ## Prerequisites
 
 - Kubernetes 1.10+
-- Kanister controller version 0.104.0 installed in your cluster
+- Kanister controller version 0.105.0 installed in your cluster
 - Kanctl CLI installed (https://docs.kanister.io/tooling.html#kanctl)
 
 ## Create RDS instance on AWS

examples/aws-rds/postgresql/rds-postgres-blueprint.yaml

@@ -14,7 +14,7 @@ actions:
     - func: KubeTask
       name: backupSnapshots
       args:
-        image: "ghcr.io/kanisterio/postgres-kanister-tools:0.104.0"
+        image: "ghcr.io/kanisterio/postgres-kanister-tools:0.105.0"
         namespace: "{{ .Object.metadata.namespace }}"
         command:
           - bash
@@ -53,7 +53,7 @@ actions:
     - func: KubeTask
       name: restoreSnapshots
       args:
-        image: "ghcr.io/kanisterio/postgres-kanister-tools:0.104.0"
+        image: "ghcr.io/kanisterio/postgres-kanister-tools:0.105.0"
         namespace: "{{ .Object.metadata.namespace }}"
         command:
           - bash
@@ -90,7 +90,7 @@ actions:
     - func: KubeTask
       name: restoreSnapshots
       args:
-        image: "ghcr.io/kanisterio/postgres-kanister-tools:0.104.0"
+        image: "ghcr.io/kanisterio/postgres-kanister-tools:0.105.0"
         namespace: "{{ .Namespace.Name }}"
         command:
           - bash

examples/aws-rds/postgresql/rds-postgres-dump-blueprint.yaml

@@ -66,7 +66,7 @@ actions:
       name: deleteBackup
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: ghcr.io/kanisterio/kanister-tools:0.104.0
+        image: ghcr.io/kanisterio/kanister-tools:0.105.0
         command:
           - bash
           - -o

examples/cassandra/README.md

@@ -7,7 +7,7 @@ As the official documentation of [Cassandra](http://cassandra.apache.org/) says,
 * Kubernetes 1.9+
 * Kubernetes beta APIs enabled only if `podDisruptionBudget` is enabled
 * PV support on the underlying infrastructure
-* Kanister controller version 0.104.0 installed in your cluster, let's say in namespace `<kanister-operator-namespace>`
+* Kanister controller version 0.105.0 installed in your cluster, let's say in namespace `<kanister-operator-namespace>`
 * Kanctl CLI installed (https://docs.kanister.io/tooling.html#kanctl)
 
 To install kanister and related tools you can follow [this](https://docs.kanister.io/install.html#install) link.
@@ -29,7 +29,7 @@ $ helm repo add bitnami https://charts.bitnami.com/bitnami
 $ helm repo update
 # remove app-namespace with the namespace you want to deploy the Cassandra app in
 $ kubectl create ns <app-namespace>
-$ helm install cassandra bitnami/cassandra --namespace <app-namespace> --set image.repository=kanisterio/cassandra --set image.tag=0.104.0 --set cluster.replicaCount=2 --set image.registry=ghcr.io --set image.pullPolicy=Always
+$ helm install cassandra bitnami/cassandra --namespace <app-namespace> --set image.repository=kanisterio/cassandra --set image.tag=0.105.0 --set cluster.replicaCount=2 --set image.registry=ghcr.io --set image.pullPolicy=Always
 
 
 ```

examples/cassandra/cassandra-blueprint.yaml

@@ -91,6 +91,34 @@ actions:
     inputArtifactNames:
       - params
     phases:
+    - func: KubeExec
+      name: waitForConnectionReady
+      args:
+        namespace: "{{ .StatefulSet.Namespace }}"
+        pod: "{{ index .StatefulSet.Pods 0 }}"
+        command:
+          - bash
+          - -o
+          - pipefail
+          - -c
+          - |
+            timeout=300
+            while true
+            do
+              VAR=$((cqlsh -u cassandra -p $CASSANDRA_PASSWORD -e "DESCRIBE keyspaces;" --request-timeout=300) 2>&1)
+              if [[ $VAR != *"Unable to connect to any servers"* ]]
+              then
+                break
+              fi
+              if [[ $timeout -le 0 ]]
+              then
+                 echo "Timed out waiting for cqlsh to configure.."
+                 exit 1
+              fi
+              sleep 2
+              timeout=$((timeout-2))
+            done
+            nodetool scrub
     - func: ScaleWorkload
       name: shutdownPod
       args:
@@ -102,7 +130,7 @@ actions:
       name: restoreFromObjectStore
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: ghcr.io/kanisterio/kanister-tools:0.104.0
+        image: ghcr.io/kanisterio/kanister-tools:0.105.0
         backupArtifactPrefix: "{{ .ArtifactsIn.params.KeyValue.backupPrefixLocation }}"
         pods: "{{ range .StatefulSet.Pods }} {{.}}{{end}}"
         restorePath: "{{ .ArtifactsIn.params.KeyValue.restorePathPrefix }}"

examples/cockroachdb/cockroachdb-blueprint.yaml

@@ -151,7 +151,7 @@ actions:
       - func: KubeTask
         name: deleteFromS3Store
         args:
-          image: ghcr.io/kanisterio/kanister-tools:0.104.0
+          image: ghcr.io/kanisterio/kanister-tools:0.105.0
           namespace: "{{ .Namespace.Name }}"
           command:
             - bash

examples/couchbase/blueprint-v2/couchbase-blueprint.yaml

@@ -21,7 +21,7 @@ actions:
           namespace: "{{ .Object.metadata.namespace }}"
       args:
         namespace: "{{ .Object.metadata.namespace }}"
-        image: ghcr.io/kanisterio/couchbase-tools:0.104.0
+        image: ghcr.io/kanisterio/couchbase-tools:0.105.0
         command:
           - bash
           - -o
@@ -58,7 +58,7 @@ actions:
           namespace: "{{ .Object.metadata.namespace }}"
       args:
         namespace: "{{ .Object.metadata.namespace }}"
-        image: ghcr.io/kanisterio/couchbase-tools:0.104.0
+        image: ghcr.io/kanisterio/couchbase-tools:0.105.0
         command:
           - bash
           - -o
@@ -89,7 +89,7 @@ actions:
       name: deleteBackup
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: ghcr.io/kanisterio/couchbase-tools:0.104.0
+        image: ghcr.io/kanisterio/couchbase-tools:0.105.0
         command:
           - bash
           - -o

examples/couchbase/couchbase-blueprint.yaml

@@ -79,7 +79,7 @@ actions:
       name: deleteBackup
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: ghcr.io/kanisterio/kanister-tools:0.104.0
+        image: ghcr.io/kanisterio/kanister-tools:0.105.0
         command:
           - bash
           - -o

examples/csi-snapshot/README.md

@@ -8,7 +8,7 @@ This example demonstrates Kanister's ability to protect an application called Ti
 
 - Helm 3 installed
 - Kubernetes 1.16+ with Beta APIs enabled
-- Kanister controller version 0.104.0 installed in the cluster, let's assume in namespace `kanister`
+- Kanister controller version 0.105.0 installed in the cluster, let's assume in namespace `kanister`
 - Kanctl CLI installed (https://docs.kanister.io/tooling.html#install-the-tools)
 - VolumeSnapshot CRDs, Snapshot Controller & a CSI Driver