chore: retry database cleanup task (#1300)

spanner/Gemfile

@@ -18,6 +18,7 @@ gem "google-cloud-spanner"
 
 group :test do
   gem "google-apis-iam_v1"
+  gem "retriable"
   gem "rspec"
   gem "rspec_junit_formatter"
 end

spanner/Gemfile.lock

@@ -3,6 +3,7 @@ GEM
   specs:
     addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
+    base64 (0.2.0)
     concurrent-ruby (1.2.3)
     declarative (0.0.20)
     diff-lcs (1.5.1)
@@ -20,16 +21,16 @@ GEM
       googleapis-common-protos-types (>= 1.11.0, < 2.a)
       googleauth (~> 1.9)
       grpc (~> 1.59)
-    google-apis-core (0.13.0)
+    google-apis-core (0.14.0)
       addressable (~> 2.5, >= 2.5.1)
       googleauth (~> 1.9)
       httpclient (>= 2.8.1, < 3.a)
       mini_mime (~> 1.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
       rexml
-    google-apis-iam_v1 (0.53.0)
-      google-apis-core (>= 0.12.0, < 2.a)
+    google-apis-iam_v1 (0.54.0)
+      google-apis-core (>= 0.14.0, < 2.a)
     google-cloud-core (1.6.1)
       google-cloud-env (>= 1.0, < 3.a)
       google-cloud-errors (~> 1.0)
@@ -50,32 +51,33 @@ GEM
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
       grpc-google-iam-v1 (~> 1.1)
-    google-cloud-spanner-v1 (0.22.2)
+    google-cloud-spanner-v1 (0.24.0)
       gapic-common (>= 0.21.1, < 2.a)
       google-cloud-errors (~> 1.0)
-    google-protobuf (3.25.2)
-    googleapis-common-protos (1.4.0)
-      google-protobuf (~> 3.14)
-      googleapis-common-protos-types (~> 1.2)
-      grpc (~> 1.27)
-    googleapis-common-protos-types (1.11.0)
+    google-protobuf (3.25.3)
+    googleapis-common-protos (1.5.0)
       google-protobuf (~> 3.18)
-    googleauth (1.10.0)
+      googleapis-common-protos-types (~> 1.7)
+      grpc (~> 1.41)
+    googleapis-common-protos-types (1.13.0)
+      google-protobuf (~> 3.18)
+    googleauth (1.11.0)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
       jwt (>= 1.4, < 3.0)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.61.0)
+    grpc (1.62.0)
       google-protobuf (~> 3.25)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (1.5.0)
+    grpc-google-iam-v1 (1.7.0)
       google-protobuf (~> 3.18)
       googleapis-common-protos (~> 1.4)
       grpc (~> 1.41)
     httpclient (2.8.3)
-    jwt (2.7.1)
+    jwt (2.8.0)
+      base64
     mini_mime (1.1.5)
     multi_json (1.15.0)
     net-http (0.4.1)
@@ -100,10 +102,10 @@ GEM
     rspec-mocks (3.13.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.0)
+    rspec-support (3.13.1)
     rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
-    signet (0.18.0)
+    signet (0.19.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
@@ -118,6 +120,7 @@ PLATFORMS
 DEPENDENCIES
   google-apis-iam_v1
   google-cloud-spanner
+  retriable
   rspec
   rspec_junit_formatter
 

spanner/spec/spec_helper.rb

@@ -74,10 +74,12 @@ def cleanup_instance_resources
   def cleanup_database_resources
     return unless @instance
 
-    @test_database = @instance.database @database_id
-    @test_database&.drop
-    @test_database = @instance.database @restored_database_id
-    @test_database&.drop
+    with_retry do 
+      @test_database = @instance.database @database_id
+      @test_database&.drop
+      @test_database = @instance.database @restored_database_id
+      @test_database&.drop
+    end
   end
 
   def cleanup_backup_resources
@@ -247,4 +249,17 @@ def restore_database_from_backup
 
     @test_database
   end
+
+  def with_retry retries: 5
+    max_retries = 10
+    Retriable.retriable(
+      on: Google::Cloud::DeadlineExceededError,
+      base_interval: 1,
+      multiplier: 2,
+      tries: [retries, max_retries].min
+    ) do
+      return yield
+    end
+    raise
+  end
 end