Merge pull request #1420 from tesshuflower/restic_check_init_update

update ensure_initialized in restic mover script

controllers/mover/restic/logfilter.go

@@ -37,7 +37,10 @@ var resticRegex = regexp.MustCompile(
 		`^\s*([aA]dded to the repository)|` +
 		`^\s*([sS]uccessfully)|` +
 		`(RESTORE_OPTIONS)|` +
-		`^\s*(Restic completed in)`)
+		`([iI]nitialize [dD]ir)|` +
+		`^\s*([fF]atal)|` +
+		`^\s*(ERROR)|` +
+		`^\s*([rR]estic completed in)`)
 
 // Filter restic log lines for a successful move job
 func LogLineFilterSuccess(line string) *string {

controllers/mover/restic/logfilter_test.go

@@ -68,7 +68,8 @@ Restic completed in 18s
 === Done ===`
 
 			// nolint:lll
-			expectedFilteredResticSourceLogSuccessful := `created restic repository f5bccd54c8 at s3:http://minio-api-minio.apps.app-aws-411ga-sno-net2-zp5jq.dev06.red-chesterfield.com/ttest-restic-new
+			expectedFilteredResticSourceLogSuccessful := `== Initialize Dir =======
+created restic repository f5bccd54c8 at s3:http://minio-api-minio.apps.app-aws-411ga-sno-net2-zp5jq.dev06.red-chesterfield.com/ttest-restic-new
 repository f5bccd54 opened (repository version 2) successfully, password is correct
 created new cache in /home/testuser/DEVFEDORA/volsync/RESTICTESTS/CACHE
 no parent snapshot found, will read all files
@@ -118,7 +119,6 @@ Testing mandatory env variables
 913a91c60431342abb402d7707f50a370c52a911e01abdf4160e5d41a77e5151
 successfully removed 1 locks
 == Checking directory for content ===
-== Initialize Dir =======
 ID        Time                 Host         Tags        Paths
 ------------------------------------------------------------------------
 4e825939  2023-04-07 20:17:00  volsync                  /mover-syncthing
@@ -248,7 +248,8 @@ Restic completed in 4s
 === Done ===`
 
 		// nolint:lll
-		expectedFilteredResticDestlogSuccessful := `created restic repository 374c6313cb at s3:http://minio.minio.svc.cluster.local:9000/resticbucket1b
+		expectedFilteredResticDestlogSuccessful := `=== Initialize Dir ===
+created restic repository 374c6313cb at s3:http://minio.minio.svc.cluster.local:9000/resticbucket1b
 No eligible snapshots found
 === No data will be restored ===
 Restic completed in 4s`
@@ -262,4 +263,32 @@ Restic completed in 4s`
 			Expect(filteredLines).To(Equal(expectedFilteredResticDestlogSuccessful))
 		})
 	})
+
+	Context("Restic mover logs - bad password", func() {
+		// Sample restore log for restic mover
+		// nolint:lll
+		resticMoverLog := `Starting container
+VolSync restic container version: v0.11.0+f866a4ec-dirty
+restore
+restic 0.17.0 compiled with go1.22.8 on linux/amd64
+Testing mandatory env variables
+=== Check for dir initialized ===
+Fatal: wrong password or no key found
+ERROR: failure checking existence of repository
+`
+
+		// nolint:lll
+		expectedFilteredResticMoverLog := `Fatal: wrong password or no key found
+ERROR: failure checking existence of repository`
+
+		It("Should filter the logs from the replication source or dest", func() {
+			reader := strings.NewReader(resticMoverLog)
+			filteredLines, err := utils.FilterLogs(reader, restic.LogLineFilterSuccess)
+			Expect(err).NotTo(HaveOccurred())
+
+			logger.Info("Logs after filter", "filteredLines", filteredLines)
+			Expect(filteredLines).To(Equal(expectedFilteredResticMoverLog))
+		})
+	})
+
 })

custom-scorecard-tests/config-downstream.yaml

@@ -125,6 +125,16 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_restic_manual_normal_restore_emptyrepo.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_restic_manual_normal_restore_emptyrepo.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_restic_manual_priv.yml

custom-scorecard-tests/config.yaml

@@ -125,6 +125,16 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_restic_manual_normal_restore_emptyrepo.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_restic_manual_normal_restore_emptyrepo.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_restic_manual_priv.yml

custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml

@@ -111,6 +111,16 @@
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_restic_manual_normal_restore_emptyrepo.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_restic_manual_normal_restore_emptyrepo.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_restic_manual_priv.yml

mover-restic/entry.sh

@@ -90,9 +90,27 @@ function check_contents {
 # Ensure the repo has been initialized
 function ensure_initialized {
     echo "=== Check for dir initialized ==="
-    # Try a restic command and capture the rc & output
+    # check for restic config and capture rc
+    # See: https://restic.readthedocs.io/en/stable/075_scripting.html#check-if-a-repository-is-already-initialized
+    set +e  # Don't exit on command failure
+
     outfile=$(mktemp -q)
-    if ! "${RESTIC[@]}" snapshots 2>"$outfile"; then
+    "${RESTIC[@]}" cat config > /dev/null 2>"$outfile"
+    rc=$?
+
+    set -e  # Exit on command failure
+
+    case $rc in
+    0)
+        echo "dir is initialized"
+        ;;
+    1)
+        # This can happen for some providers (e.g. minio) if the bucket does not exist
+        # Restic will return 10 if the bucket exists and no restic repo at the path exists, but will
+        # still return 1 if the bucket itself doesn't exist.
+        # We can proceed with trying an init which will create the bucket (and path in the bucket if there is one)
+        # restic init should fail if somehow the repo already exists when init is run or if it's unable to
+        # create the bucket
         output=$(<"$outfile")
         # Match against error string for uninitialized repo
         # This string also appears when credentials are incorrect (in which case
@@ -104,7 +122,28 @@ function ensure_initialized {
             cat "$outfile"
             error 3 "failure checking existence of repository"
         fi
-    fi
+        ;;
+    10)
+        # rc = 10  Repository does not exist (since restic 0.17.0)
+        echo "=== Initialize Dir ==="
+        "${RESTIC[@]}" init
+        ;;
+    11)
+        # rc = 11  Failed to lock repository (since restic 0.17.0)
+        cat "$outfile"
+        error 3 "failure locking repository"
+        ;;
+    12)
+        # rc = 12 Wrong password (since restic 0.17.1)
+        cat "$outfile"
+        error 3 "failure connecting to repository, incorrect password"
+        ;;
+    *)
+        cat "$outfile"
+        error 3 "failure checking existence of repository"
+        ;;
+    esac
+
     rm -f "$outfile"
 }
 

test-e2e/roles/create_restic_secret/tasks/main.yml

@@ -13,15 +13,23 @@
 - include_role:
     name: get_minio_credentials
 
+- name: Get bucket name to use (default is 'restic-e2e')
+  ansible.builtin.set_fact:
+    bucket_name: "{{ bucket_name | default('restic-e2e') }}"
+
+- name: Get path name to use under the bucket (default is namespace name)
+  ansible.builtin.set_fact:
+    path_name: "{{ path_name | default(namespace) }}"
+
 # Path in restic will include the namespace to avoid re-runs of tests interferring with each-other
 # And also to prevent multiple tests from using the same path (each test should use its own namespace)
 - name: Determine repo URL
   set_fact:
-    repo_url: "s3:http://minio.{{ minio_namespace }}.svc.cluster.local:9000/restic-e2e/{{ namespace }}"
+    repo_url: "s3:http://minio.{{ minio_namespace }}.svc.cluster.local:9000/{{ bucket_name }}/{{ path_name }}"
 
 - name: Set repo URL to use TLS
   set_fact:
-    repo_url: "s3:https://minio.{{ minio_namespace }}.svc.cluster.local:9000/restic-e2e/{{ namespace }}"
+    repo_url: "s3:https://minio.{{ minio_namespace }}.svc.cluster.local:9000/{{ bucket_name }}/{{ path_name }}"
   when: use_tls is defined and use_tls == true
 
 - name: Create restic secret