Update kopia commit in kanister-tools image kastenhq 663cbad

.github/dependabot.yml

@@ -41,3 +41,11 @@ updates:
     open-pull-requests-limit: 3
     schedule:
       interval: daily
+    groups:
+      github-actions:
+        patterns:
+        - "^actions/*"
+        - "^github/codeql-action"
+      docker:
+        patterns:
+        - "docker/*"

.github/workflows/atlas-image-build.yaml

@@ -18,10 +18,10 @@ jobs:
     outputs:
       changed: ${{ steps.changed-files.outputs.any_changed }}
     steps:
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           fetch-depth: 0
-      - uses: tj-actions/changed-files@41960309398d165631f08c5df47a11147e14712b # v39.1.2
+      - uses: tj-actions/changed-files@db153baf731265ad02cd490b07f470e2d55e3345 # v39.2.1
         name: Get changed files
         id: changed-files
         with:

.github/workflows/dependendy-review.yml

@@ -15,6 +15,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@v3

.github/workflows/grype-vulnerability-scanner.yaml

@@ -7,7 +7,7 @@ jobs:
       - name: Create repo directory before checking out latest code
         run: mkdir -p repo
       - name: Checkout the latest code
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           ref: master
           path: repo
@@ -51,7 +51,7 @@ jobs:
       - name: Create repo directory before checking out latest code
         run: mkdir -p repo
       - name: Checkout the latest code
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           ref: master
           path: repo

.github/workflows/main.yaml

@@ -9,7 +9,7 @@ jobs:
   gomod:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
     - run: make go-mod-tidy
     - run: make go-mod-download
     - run: tar -cvf ./src.tar.gz ./ # preserve file permissions

.github/workflows/ossf-scorecard.yml

@@ -26,20 +26,20 @@ jobs:
     steps:
       -
         name: "Checkout repo"
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
         with:
           persist-credentials: false
       -
         name: "Run analysis"
-        uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
+        uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0
         with:
           results_file: results.sarif
           results_format: sarif
           publish_results: true
       -
         # Upload the results to GitHub's code scanning dashboard.
         name: "Upload to results to dashboard"
-        uses: github/codeql-action/upload-sarif@6a28655e3dcb49cb0840ea372fd6d17733edd8a4 # v2.21.8
+        uses: github/codeql-action/upload-sarif@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
         with:
           sarif_file: results.sarif
       -

.gitignore

@@ -7,3 +7,4 @@
 /bin
 /dist
 **/*.swp
+/.idea

.goreleaser.yml

@@ -62,9 +62,9 @@ dockers:
   dockerfile: 'docker/tools/Dockerfile'
   build_flag_templates:
   - "--build-arg=kan_tools_version={{ .Tag }}"
-# Refers to https://github.com/kopia/kopia/commit/3551f743d762f2ffe669523272d1c8d734120b79
-  - "--build-arg=kopia_build_commit=3551f74"
-  - "--build-arg=kopia_repo_org=kopia"
+# Refers to https://github.com/kastenhq/kopia/commit/663cbad414207f077c666637fabfb22b47855959
+  - "--build-arg=kopia_build_commit=663cbad"
+  - "--build-arg=kopia_repo_org=kastenhq"
   extra_files:
   - 'LICENSE'
 - ids:

Dockerfile.in

@@ -1,4 +1,4 @@
-ARG base_image=registry.access.redhat.com/ubi9/ubi-minimal:9.2-750
+ARG base_image=registry.access.redhat.com/ubi9/ubi-minimal:9.2-750.1697534106
 FROM ${base_image}
 ARG kanister_version
 

Makefile

@@ -41,8 +41,6 @@ PWD := $$(pwd)
 # Whether to build inside a containerized build environment
 DOCKER_BUILD ?= "true"
 
-DOCKER_CONFIG ?= "$(HOME)/.docker"
-
 # Mention the vm-driver that should be used to install OpenShift
 vm-driver ?= "kvm"
 
@@ -62,11 +60,6 @@ IMAGE_NAME := $(BIN)
 
 IMAGE := $(REGISTRY)/$(IMAGE_NAME)
 
-BUILD_IMAGE ?= ghcr.io/kanisterio/build:v0.0.24
-
-# tag 0.1.0 is, 0.0.1 (latest) + gh + aws + helm binary
-DOCS_BUILD_IMAGE ?= ghcr.io/kanisterio/docker-sphinx:0.2.0
-
 DOCS_RELEASE_BUCKET ?= s3://docs.kanister.io
 
 GITHUB_TOKEN ?= ""
@@ -100,39 +93,25 @@ all-push: $(addprefix push-, $(ALL_ARCH))
 build: bin/$(ARCH)/$(BIN)
 
 build-controller:
-	@$(MAKE) run CMD='-c " \
+	@$(MAKE) run CMD=" \
 	goreleaser build --id $(BIN) --rm-dist --debug --snapshot \
 	&& cp dist/$(BIN)_linux_$(ARCH)_*/$(BIN) bin/$(ARCH)/$(BIN) \
-	"'
+	"
 
 bin/$(ARCH)/$(BIN):
 	@echo "building: $@"
-	@$(MAKE) run CMD='-c " \
+	@$(MAKE) run CMD=" \
 		GOARCH=$(ARCH)       \
 		VERSION=$(VERSION) \
 		PKG=$(PKG)         \
 		BIN=$(BIN) \
 		GOBORING=$(GOBORING) \
 		./build/build.sh   \
-	"'
+	"
 # Example: make shell CMD="-c 'date > datefile'"
 shell: build-dirs
 	@echo "launching a shell in the containerized build environment"
-	@docker run                                      \
-		-ti                                          \
-		--rm                                         \
-		--privileged                                 \
-		--net host                                   \
-		-v "$(PWD)/.go/pkg:/go/pkg"                  \
-		-v "$(PWD)/.go/cache:/go/.cache"             \
-		-v "${HOME}/.kube:/root/.kube"               \
-		-v "$(PWD):/go/src/$(PKG)"                   \
-		-v "$(PWD)/bin/$(ARCH):/go/bin"              \
-		-v "$(DOCKER_CONFIG):/root/.docker"          \
-		-v /var/run/docker.sock:/var/run/docker.sock \
-		-w /go/src/$(PKG)                            \
-		$(BUILD_IMAGE)                               \
-		/bin/sh
+	@PWD=$(PWD) ARCH=$(ARCH) PKG=$(PKG) GITHUB_TOKEN=$(GITHUB_TOKEN) CMD="/bin/bash $(CMD)" /bin/bash ./build/run_container.sh shell
 
 DOTFILE_IMAGE = $(subst :,_,$(subst /,_,$(IMAGE))-$(VERSION))
 
@@ -176,38 +155,31 @@ deploy: release-controller .deploy-$(DOTFILE_IMAGE)
 	@kubectl apply -f .deploy-$(DOTFILE_IMAGE)
 
 test: build-dirs
-	@$(MAKE) run CMD='-c "./build/test.sh $(SRC_DIRS)"'
+	@$(MAKE) run CMD="./build/test.sh $(SRC_DIRS)"
 
 helm-test: build-dirs
-	@$(MAKE) run CMD='-c "./build/helm-test.sh $(SRC_DIRS)"'
+	@$(MAKE) run CMD="./build/helm-test.sh $(SRC_DIRS)"
 
 integration-test: build-dirs
-	@$(MAKE) run CMD='-c "./build/integration-test.sh short"'
+	@$(MAKE) run CMD="./build/integration-test.sh short"
 
 openshift-test:
 	@/bin/bash ./build/integration-test.sh openshift $(ocp_version)
 
 golint:
-	@$(MAKE) run CMD='-c "./build/golint.sh"'
+	@$(MAKE) run CMD="./build/golint.sh"
 
 codegen:
-	@$(MAKE) run CMD='-c "./build/codegen.sh"'
+	@$(MAKE) run CMD="./build/codegen.sh"
 
 DOCS_CMD = "cd docs && make clean &&          \
                 doc8 --max-line-length 90 --ignore D000 . && \
-                make spelling && make html           \
-	   "
+                make spelling && make html"
 
 docs:
 ifeq ($(DOCKER_BUILD),"true")
 	@echo "running DOCS_CMD in the containerized build environment"
-	@docker run             \
-		--entrypoint ''     \
-		--rm                \
-		-v "$(PWD):/repo"   \
-		-w /repo            \
-		$(DOCS_BUILD_IMAGE) \
-		/bin/bash -c $(DOCS_CMD)
+	@PWD=$(PWD) ARCH=$(ARCH) CMD=$(DOCS_CMD) /bin/bash ./build/run_container.sh docs
 else
 	@/bin/bash -c $(DOCS_CMD)
 endif
@@ -216,19 +188,12 @@ API_DOCS_CMD = "gen-crd-api-reference-docs 			\
 		-config docs/api_docs/config.json 	\
 		-api-dir ./pkg/apis/cr/v1alpha1 	\
 		-template-dir docs/api_docs/template 		\
-		-out-file API.md 	\
-"
+		-out-file API.md"
 
 crd_docs:
 ifeq ($(DOCKER_BUILD),"true")
 	@echo "running API_DOCS_CMD in the containerized build environment"
-	@docker run             \
-		--entrypoint ''     \
-		--rm                \
-		-v "$(PWD):/repo"   \
-		-w /repo            \
-		$(BUILD_IMAGE) \
-		/bin/bash -c $(API_DOCS_CMD)
+	@PWD=$(PWD) ARCH=$(ARCH) CMD=$(API_DOCS_CMD) /bin/bash ./build/run_container.sh crd_docs
 else
 	@/bin/bash -c $(API_DOCS_CMD)
 endif
@@ -240,21 +205,7 @@ build-dirs:
 run: build-dirs
 ifeq ($(DOCKER_BUILD),"true")
 	@echo "running CMD in the containerized build environment"
-	@docker run                                                     \
-		--rm                                                        \
-		--net host                                                  \
-		-e GITHUB_TOKEN=$(GITHUB_TOKEN)                             \
-		-v "${HOME}/.kube:/root/.kube"                              \
-		-v "$(PWD)/.go/pkg:/go/pkg"                                 \
-		-v "$(PWD)/.go/cache:/go/.cache"                            \
-		-v "$(PWD):/go/src/$(PKG)"                                  \
-		-v "$(PWD)/bin/$(ARCH):/go/bin"                             \
-		-v "$(PWD)/.go/std/$(ARCH):/usr/local/go/pkg/linux_$(ARCH)" \
-		-v "$(DOCKER_CONFIG):/root/.docker"                         \
-		-v /var/run/docker.sock:/var/run/docker.sock                \
-		-w /go/src/$(PKG)                                           \
-		$(BUILD_IMAGE)                                              \
-		/bin/bash $(CMD)
+	@PWD=$(PWD) ARCH=$(ARCH) PKG=$(PKG) GITHUB_TOKEN=$(GITHUB_TOKEN) CMD="$(CMD)" /bin/bash ./build/run_container.sh run
 else
 	@/bin/bash $(CMD)
 endif
@@ -285,28 +236,28 @@ release-helm:
 	@/bin/bash ./build/release_helm.sh $(VERSION)
 
 gorelease:
-	@$(MAKE) run CMD='-c "./build/gorelease.sh"'
+	@$(MAKE) run CMD="./build/gorelease.sh"
 
 release-snapshot:
-	@$(MAKE) run CMD='-c "GORELEASER_CURRENT_TAG=v9.99.9-dev goreleaser --debug release --rm-dist --snapshot --timeout=60m0s"'
+	@$(MAKE) run CMD="GORELEASER_CURRENT_TAG=v9.99.9-dev goreleaser --debug release --rm-dist --snapshot --timeout=60m0s"
 
 go-mod-download:
-	@$(MAKE) run CMD='-c "go mod download"'
+	@$(MAKE) run CMD="go mod download"
 
 start-kind:
-	@$(MAKE) run CMD='-c "./build/local_kubernetes.sh start_localkube"'
+	@$(MAKE) run CMD="./build/local_kubernetes.sh start_localkube"
 
 tiller:
 	@/bin/bash ./build/init_tiller.sh
 
 install-minio:
-	@$(MAKE) run CMD='-c "./build/minio.sh install_minio"'
+	@$(MAKE) run CMD="./build/minio.sh install_minio"
 
 install-csi-hostpath-driver:
-	@$(MAKE) run CMD='-c "./build/local_kubernetes.sh install_csi_hostpath_driver"'
+	@$(MAKE) run CMD="./build/local_kubernetes.sh install_csi_hostpath_driver"
 
 uninstall-minio:
-	@$(MAKE) run CMD='-c "./build/minio.sh uninstall_minio"'
+	@$(MAKE) run CMD="./build/minio.sh uninstall_minio"
 
 start-minishift:
 	@/bin/bash ./build/minishift.sh start_minishift $(vm-driver)
@@ -315,21 +266,21 @@ stop-minishift:
 	@/bin/bash ./build/minishift.sh stop_minishift
 
 stop-kind:
-	@$(MAKE) run CMD='-c "./build/local_kubernetes.sh stop_localkube"'
+	@$(MAKE) run CMD="./build/local_kubernetes.sh stop_localkube"
 
 check:
 	@./build/check.sh
 
 go-mod-tidy:
-	@$(MAKE) run CMD='-c "./build/gomodtidy.sh"'
+	@$(MAKE) run CMD="./build/gomodtidy.sh"
 
 
 install-crds: ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	@$(MAKE) run CMD='-c "kubectl apply -f pkg/customresource/"'
+	@$(MAKE) run CMD="kubectl apply -f pkg/customresource/"
 
 uninstall-crds: ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
-	@$(MAKE) run CMD='-c "kubectl delete -f pkg/customresource/"'
+	@$(MAKE) run CMD="kubectl delete -f pkg/customresource/"
 
 manifests: ## Generates CustomResourceDefinition objects.
-	@$(MAKE) run CMD='-c "./build/generate_crds.sh ${CONTROLLER_TOOLS_VERSION}"'
+	@$(MAKE) run CMD="./build/generate_crds.sh ${CONTROLLER_TOOLS_VERSION}"
 

build/integration-test.sh

@@ -26,7 +26,7 @@ TEST_TIMEOUT="30m"
 TEST_OPTIONS="-tags=integration -timeout ${TEST_TIMEOUT} -check.suitep ${DOP}"
 # Regex to match apps to run in short mode
 # Temporary disable ES test. Issue to track https://github.com/kanisterio/kanister/issues/1920
-SHORT_APPS="^PostgreSQL$|^MySQL$|^MongoDB$|Maria|^MSSQL$"
+SHORT_APPS="^PostgreSQL$|^MySQL$|^MongoDB$|^MSSQL$"
 # OCAPPS has all the apps that are to be tested against openshift cluster
 OC_APPS3_11="MysqlDBDepConfig$|MongoDBDepConfig$|PostgreSQLDepConfig$"
 OC_APPS4_4="MysqlDBDepConfig4_4|MongoDBDepConfig4_4|PostgreSQLDepConfig4_4"

build/local_kubernetes.sh

@@ -73,14 +73,43 @@ install_csi_hostpath_driver() {
     kubectl apply -fhttps://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/{rbac-snapshot-controller.yaml,setup-snapshot-controller.yaml}
 
     # Deploy the CSI Hostpath Driver
-    cd /tmp
-    git clone https://github.com/kubernetes-csi/csi-driver-host-path.git
-    cd csi-driver-host-path
-    sed -i 's/mountPropagation: Bidirectional/\#mountPropagation: Bidirectional/g' deploy/kubernetes-latest/hostpath/csi-hostpath-plugin.yaml
-    ./deploy/kubernetes-latest/deploy.sh
-
-    # Create StorageClass
-    kubectl apply -f ./examples/csi-storageclass.yaml
+    pushd /tmp
+      git clone https://github.com/kubernetes-csi/csi-driver-host-path.git
+      pushd csi-driver-host-path
+        sed -i 's/mountPropagation: Bidirectional/\#mountPropagation: Bidirectional/g' deploy/kubernetes-latest/hostpath/csi-hostpath-plugin.yaml
+
+        ./deploy/kubernetes-latest/deploy.sh
+
+        # Create StorageClass
+        kubectl apply -f ./examples/csi-storageclass.yaml
+      popd
+    popd
+}
+
+check_csi_hostpath_driver_installed() {
+    # Check VolumeSnapshot CRDs are installed
+    if ! kubectl diff -fhttps://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/client/config/crd/snapshot.storage.k8s.io_{volumesnapshots.yaml,volumesnapshotclasses.yaml,volumesnapshotcontents.yaml} 2>&1 > /dev/null ; then
+        echo "VolumeSnapshot CRDs are not installed."
+        exit 1
+    fi
+
+    # Check snapshot controller created
+    if ! kubectl diff -fhttps://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/{rbac-snapshot-controller.yaml,setup-snapshot-controller.yaml} 2>&1 > /dev/null ; then
+        echo "Snapshot controller is not created."
+        exit 1
+    fi
+
+    # Deploy the CSI Hostpath Driver
+    pushd /tmp
+      git clone https://github.com/kubernetes-csi/csi-driver-host-path.git
+      pushd csi-driver-host-path
+        # Check StorageClass created
+        if ! kubectl diff -f ./examples/csi-storageclass.yaml 2>&1 > /dev/null ; then
+            echo "StorageClass is not created."
+            exit 1
+        fi
+      popd
+    popd
 }
 
 stop_localkube() {
@@ -158,6 +187,9 @@ EOM
 check_or_get_dependencies
 case "${1}" in
         # Alphabetically sorted
+        check_csi_hostpath_driver_installed)
+            time -p check_csi_hostpath_driver_installed
+            ;;
         get_localkube)
             time -p get_localkube
             ;;