From c4078c6a2107faa6aa503d2f695e1e8a0d66fd29 Mon Sep 17 00:00:00 2001 From: Nivedha Date: Fri, 23 Feb 2024 17:40:30 +0530 Subject: [PATCH 1/3] chore: retry database cleanup task - Database cleanup task fails sporadically due to Deadline Exceeded error which happen due to various reasons including network issues they are safe to retry. - This PR adds an exponential retry for the task. --- spanner/spec/spec_helper.rb | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/spanner/spec/spec_helper.rb b/spanner/spec/spec_helper.rb index 1dee8e59c..b6ba9297b 100644 --- a/spanner/spec/spec_helper.rb +++ b/spanner/spec/spec_helper.rb @@ -74,10 +74,12 @@ def cleanup_instance_resources def cleanup_database_resources return unless @instance - @test_database = @instance.database @database_id - @test_database&.drop - @test_database = @instance.database @restored_database_id - @test_database&.drop + with_retry "cleanup database" do + @test_database = @instance.database @database_id + @test_database&.drop + @test_database = @instance.database @restored_database_id + @test_database&.drop + end end def cleanup_backup_resources @@ -247,4 +249,17 @@ def restore_database_from_backup @test_database end + + def with_retry action, attempts: 5 + @attempt_number ||= 0 + yield + @attempt_number = nil + rescue Exception => e + @attempt_number += 1 + puts "failed attempt #{@attempt_number} for #{action}" + sleep @attempt_number*@attempt_number + retry if @attempt_number < attempts + @attempt_number = nil + raise e + end end From f799b00cb8606b91d4fb5c8b64118dff01192ba4 Mon Sep 17 00:00:00 2001 From: Nivedha Date: Mon, 26 Feb 2024 15:40:53 +0530 Subject: [PATCH 2/3] chore: update to use retriable gem --- spanner/Gemfile | 1 + spanner/Gemfile.lock | 35 +++++++++++++++++++---------------- spanner/spec/spec_helper.rb | 24 ++++++++++++------------ 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/spanner/Gemfile b/spanner/Gemfile index c0358760c..51217dfd5 100644 --- a/spanner/Gemfile +++ b/spanner/Gemfile @@ -18,6 +18,7 @@ gem "google-cloud-spanner" group :test do gem "google-apis-iam_v1" + gem "retriable" gem "rspec" gem "rspec_junit_formatter" end diff --git a/spanner/Gemfile.lock b/spanner/Gemfile.lock index bb62b5963..bf34bf7d2 100644 --- a/spanner/Gemfile.lock +++ b/spanner/Gemfile.lock @@ -3,6 +3,7 @@ GEM specs: addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) + base64 (0.2.0) concurrent-ruby (1.2.3) declarative (0.0.20) diff-lcs (1.5.1) @@ -20,7 +21,7 @@ GEM googleapis-common-protos-types (>= 1.11.0, < 2.a) googleauth (~> 1.9) grpc (~> 1.59) - google-apis-core (0.13.0) + google-apis-core (0.14.0) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.1, < 3.a) @@ -28,8 +29,8 @@ GEM representable (~> 3.0) retriable (>= 2.0, < 4.a) rexml - google-apis-iam_v1 (0.53.0) - google-apis-core (>= 0.12.0, < 2.a) + google-apis-iam_v1 (0.54.0) + google-apis-core (>= 0.14.0, < 2.a) google-cloud-core (1.6.1) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) @@ -50,32 +51,33 @@ GEM gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) grpc-google-iam-v1 (~> 1.1) - google-cloud-spanner-v1 (0.22.2) + google-cloud-spanner-v1 (0.24.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) - google-protobuf (3.25.2) - googleapis-common-protos (1.4.0) - google-protobuf (~> 3.14) - googleapis-common-protos-types (~> 1.2) - grpc (~> 1.27) - googleapis-common-protos-types (1.11.0) + google-protobuf (3.25.3) + googleapis-common-protos (1.5.0) google-protobuf (~> 3.18) - googleauth (1.10.0) + googleapis-common-protos-types (~> 1.7) + grpc (~> 1.41) + googleapis-common-protos-types (1.13.0) + google-protobuf (~> 3.18) + googleauth (1.11.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.1) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.61.0) + grpc (1.62.0) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc-google-iam-v1 (1.5.0) + grpc-google-iam-v1 (1.7.0) google-protobuf (~> 3.18) googleapis-common-protos (~> 1.4) grpc (~> 1.41) httpclient (2.8.3) - jwt (2.7.1) + jwt (2.8.0) + base64 mini_mime (1.1.5) multi_json (1.15.0) net-http (0.4.1) @@ -100,10 +102,10 @@ GEM rspec-mocks (3.13.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) - rspec-support (3.13.0) + rspec-support (3.13.1) rspec_junit_formatter (0.6.0) rspec-core (>= 2, < 4, != 2.12.0) - signet (0.18.0) + signet (0.19.0) addressable (~> 2.8) faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) @@ -118,6 +120,7 @@ PLATFORMS DEPENDENCIES google-apis-iam_v1 google-cloud-spanner + retriable rspec rspec_junit_formatter diff --git a/spanner/spec/spec_helper.rb b/spanner/spec/spec_helper.rb index b6ba9297b..119ba3029 100644 --- a/spanner/spec/spec_helper.rb +++ b/spanner/spec/spec_helper.rb @@ -74,7 +74,7 @@ def cleanup_instance_resources def cleanup_database_resources return unless @instance - with_retry "cleanup database" do + with_retry do @test_database = @instance.database @database_id @test_database&.drop @test_database = @instance.database @restored_database_id @@ -250,16 +250,16 @@ def restore_database_from_backup @test_database end - def with_retry action, attempts: 5 - @attempt_number ||= 0 - yield - @attempt_number = nil - rescue Exception => e - @attempt_number += 1 - puts "failed attempt #{@attempt_number} for #{action}" - sleep @attempt_number*@attempt_number - retry if @attempt_number < attempts - @attempt_number = nil - raise e + def with_retry retries: 5 + max_retries = 10 + Retriable.retriable( + on: Google::Cloud::DeadlineExceededError, + base_interval: 1, + multiplier: 2, + tries: [retries, max_retries].min + ) do + yield + end + raise end end From 544b5daf8f2292210a825b5cdb993324791aaebf Mon Sep 17 00:00:00 2001 From: Nivedha Date: Mon, 26 Feb 2024 16:40:32 +0530 Subject: [PATCH 3/3] chore: add return to not raise always --- spanner/spec/spec_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spanner/spec/spec_helper.rb b/spanner/spec/spec_helper.rb index 119ba3029..00927f624 100644 --- a/spanner/spec/spec_helper.rb +++ b/spanner/spec/spec_helper.rb @@ -258,7 +258,7 @@ def with_retry retries: 5 multiplier: 2, tries: [retries, max_retries].min ) do - yield + return yield end raise end