Auto retry with rate limiting (#290)

auth0.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'jwt', '~> 2.2'
   s.add_runtime_dependency 'zache', '~> 0.12'
   s.add_runtime_dependency 'addressable', '~> 2.8'
+  s.add_runtime_dependency 'retryable', '~> 3.0'
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'rake', '~> 13.0'

lib/auth0/mixins/httpproxy.rb

@@ -1,56 +1,52 @@
 require "addressable/uri"
+require "retryable"
+require_relative "../exception.rb"
 
 module Auth0
   module Mixins
     # here's the proxy for Rest calls based on rest-client, we're building all request on that gem
     # for now, if you want to feel free to use your own http client
     module HTTPProxy
-      attr_accessor :headers, :base_uri, :timeout
+      attr_accessor :headers, :base_uri, :timeout, :retry_count
+      DEAFULT_RETRIES = 3
+      MAX_ALLOWED_RETRIES = 10
+      MAX_REQUEST_RETRY_JITTER = 250
+      MAX_REQUEST_RETRY_DELAY = 1000
+      MIN_REQUEST_RETRY_DELAY = 100
 
       # proxying requests from instance methods to HTTP class methods
       %i(get post post_file put patch delete delete_with_body).each do |method|
         define_method(method) do |uri, body = {}, extra_headers = {}|
+          body = body.delete_if { |_, v| v.nil? }
 
-          if base_uri
-            # if a base_uri is set then the uri can be encoded as a path
-            safe_path = Addressable::URI.new(path: uri).normalized_path
-          else
-            safe_path = Addressable::URI.escape(uri)
+          Retryable.retryable(retry_options) do
+            request(method, uri, body, extra_headers)
           end
+        end
+      end
 
-          body = body.delete_if { |_, v| v.nil? }
-          result = if method == :get
-                     # Mutate the headers property to add parameters.
-                     add_headers({params: body})
-                     # Merge custom headers into existing ones for this req.
-                     # This prevents future calls from using them.
-                     get_headers = headers.merge extra_headers
-                     # Make the call with extra_headers, if provided.
-                     call(:get, url(safe_path), timeout, get_headers)
-                   elsif method == :delete
-                     call(:delete, url(safe_path), timeout, add_headers({params: body}))
-                   elsif method == :delete_with_body
-                     call(:delete, url(safe_path), timeout, headers, body.to_json)
-                   elsif method == :post_file
-                     body.merge!(multipart: true)
-                     # Ignore the default Content-Type headers and let the HTTP client define them
-                     post_file_headers = headers.slice(*headers.keys - ['Content-Type'])
-                     # Actual call with the altered headers
-                     call(:post, url(safe_path), timeout, post_file_headers, body)
-                   else
-                     call(method, url(safe_path), timeout, headers, body.to_json)
-                   end
-          case result.code
-          when 200...226 then safe_parse_json(result.body)
-          when 400       then raise Auth0::BadRequest.new(result.body, code: result.code, headers: result.headers)
-          when 401       then raise Auth0::Unauthorized.new(result.body, code: result.code, headers: result.headers)
-          when 403       then raise Auth0::AccessDenied.new(result.body, code: result.code, headers: result.headers)
-          when 404       then raise Auth0::NotFound.new(result.body, code: result.code, headers: result.headers)
-          when 429       then raise Auth0::RateLimitEncountered.new(result.body, code: result.code, headers: result.headers)
-          when 500       then raise Auth0::ServerError.new(result.body, code: result.code, headers: result.headers)
-          else                raise Auth0::Unsupported.new(result.body, code: result.code, headers: result.headers)
-          end
+      def retry_options
+        sleep_timer = lambda do |attempt|
+          wait = 1000 * 2**attempt # Exponential delay with each subsequent request attempt.
+          wait += rand(wait..wait+MAX_REQUEST_RETRY_JITTER) # Add jitter to the delay window.
+          wait = [MAX_REQUEST_RETRY_DELAY, wait].min # Cap delay at MAX_REQUEST_RETRY_DELAY.
+          wait = [MIN_REQUEST_RETRY_DELAY, wait].max # Ensure delay is no less than MIN_REQUEST_RETRY_DELAY.
+          wait / 1000.to_f.round(2) # convert ms to seconds
         end
+
+        tries = 1 + [Integer(retry_count || DEAFULT_RETRIES), MAX_ALLOWED_RETRIES].min # Cap retries at MAX_ALLOWED_RETRIES
+
+        {
+          tries: tries,
+          sleep: sleep_timer,
+          on: Auth0::RateLimitEncountered
+        }
+      end
+
+      def encode_uri(uri)
+        # if a base_uri is set then the uri can be encoded as a path
+        path = base_uri ? Addressable::URI.new(path: uri).normalized_path : Addressable::URI.escape(uri)
+        url(path)
       end
 
       def url(path)
@@ -69,6 +65,41 @@ def safe_parse_json(body)
         body
       end
 
+      def request(method, uri, body, extra_headers)
+        result = if method == :get
+          # Mutate the headers property to add parameters.
+          add_headers({params: body})
+          # Merge custom headers into existing ones for this req.
+          # This prevents future calls from using them.
+          get_headers = headers.merge extra_headers
+          # Make the call with extra_headers, if provided.
+          call(:get, encode_uri(uri), timeout, get_headers)
+        elsif method == :delete
+          call(:delete, encode_uri(uri), timeout, add_headers({params: body}))
+        elsif method == :delete_with_body
+          call(:delete, encode_uri(uri), timeout, headers, body.to_json)
+        elsif method == :post_file
+          body.merge!(multipart: true)
+          # Ignore the default Content-Type headers and let the HTTP client define them
+          post_file_headers = headers.slice(*headers.keys - ['Content-Type'])
+          # Actual call with the altered headers
+          call(:post, encode_uri(uri), timeout, post_file_headers, body)
+        else
+          call(method, encode_uri(uri), timeout, headers, body.to_json)
+        end
+
+        case result.code
+        when 200...226 then safe_parse_json(result.body)
+        when 400       then raise Auth0::BadRequest.new(result.body, code: result.code, headers: result.headers)
+        when 401       then raise Auth0::Unauthorized.new(result.body, code: result.code, headers: result.headers)
+        when 403       then raise Auth0::AccessDenied.new(result.body, code: result.code, headers: result.headers)
+        when 404       then raise Auth0::NotFound.new(result.body, code: result.code, headers: result.headers)
+        when 429       then raise Auth0::RateLimitEncountered.new(result.body, code: result.code, headers: result.headers)
+        when 500       then raise Auth0::ServerError.new(result.body, code: result.code, headers: result.headers)
+        else           raise Auth0::Unsupported.new(result.body, code: result.code, headers: result.headers)
+        end
+      end
+
       def call(method, url, timeout, headers, body = nil)
         RestClient::Request.execute(
           method: method,

lib/auth0/mixins/initializer.rb

@@ -15,6 +15,7 @@ def initialize(config)
         @base_uri = base_url(options)
         @headers = client_headers
         @timeout = options[:timeout] || 10
+        @retry_count = options[:retry_count]
         extend Auth0::Api::AuthenticationEndpoints
         @client_id = options[:client_id]
         @client_secret = options[:client_secret]

spec/lib/auth0/mixins/httpproxy_spec.rb

@@ -6,6 +6,7 @@
     dummy_instance = DummyClassForProxy.new
     dummy_instance.extend(Auth0::Mixins::HTTPProxy)
     dummy_instance.base_uri = "https://auth0.com"
+    dummy_instance.retry_count = 0
 
     @instance = dummy_instance
     @exception = DummyClassForRestClient.new
@@ -152,6 +153,100 @@
           .and_return(StubResponse.new({}, true, 200))
         expect { @instance.send(http_method, '/te st#test') }.not_to raise_error
       end
+
+      context "when status 429 is recieved on send http #{http_method} method" do
+        it "should retry 3 times when retry_count is not set" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+
+          @exception.response = StubResponse.new({}, false, 429)
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                              url: 'https://auth0.com/test',
+                                                              timeout: nil,
+                                                              headers: { params: {} },
+                                                              payload: nil)
+            .and_raise(@exception)
+          expect(RestClient::Request).to receive(:execute).exactly(4).times
+
+          expect { retry_instance.send(http_method, '/test') }.to raise_error { |error|
+            expect(error).to be_a(Auth0::RateLimitEncountered)
+          }
+        end
+
+        it "should retry 2 times when retry_count is set to 2" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+          retry_instance.retry_count = 2
+
+          @exception.response = StubResponse.new({}, false, 429)
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                              url: 'https://auth0.com/test',
+                                                              timeout: nil,
+                                                              headers: { params: {} },
+                                                              payload: nil)
+            .and_raise(@exception)
+          expect(RestClient::Request).to receive(:execute).exactly(3).times
+
+          expect { retry_instance.send(http_method, '/test') }.to raise_error { |error|
+            expect(error).to be_a(Auth0::RateLimitEncountered)
+          }
+        end
+
+        it "should not retry when retry_count is set to 0" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+          retry_instance.retry_count = 0
+
+          @exception.response = StubResponse.new({}, false, 429)
+
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                              url: 'https://auth0.com/test',
+                                                              timeout: nil,
+                                                              headers: { params: {} },
+                                                              payload: nil)
+            .and_raise(@exception)
+
+          expect(RestClient::Request).to receive(:execute).exactly(1).times
+          expect { retry_instance.send(http_method, '/test') }.to raise_error { |error|
+            expect(error).to be_a(Auth0::RateLimitEncountered)
+          }
+        end
+
+        it "should have have random retry times grow with jitter backoff" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+          retry_instance.retry_count = 2
+          time_entries = []
+          @time_start
+
+          @exception.response = StubResponse.new({}, false, 429)
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                               url: 'https://auth0.com/test',
+                                                               timeout: nil,
+                                                               headers: { params: {} },
+                                                               payload: nil) do
+
+            time_entries.push(Time.now.to_f - @time_start.to_f)
+            @time_start = Time.now.to_f # restart the clock
+            raise @exception
+          end
+
+          @time_start = Time.now.to_f #start the clock
+          retry_instance.send(http_method, '/test') rescue nil
+          time_entries_first_set = time_entries.shift(time_entries.length)
+
+          retry_instance.send(http_method, '/test') rescue nil
+          time_entries.each_with_index do |entry, index|
+            if index > 0 #skip the first request
+              expect(entry != time_entries_first_set[index])
+            end
+          end
+        end
+      end
     end
   end
 
@@ -301,6 +396,100 @@
           .and_return(StubResponse.new(res, true, 404))
         expect { @instance.send(http_method, '/test') }.to raise_error(Auth0::NotFound, res)
       end
+
+      context "when status 429 is recieved on send http #{http_method} method" do
+        it "should retry 3 times when retry_count is not set" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+
+          @exception.response = StubResponse.new({}, false, 429)
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                              url: 'https://auth0.com/test',
+                                                              timeout: nil,
+                                                              headers: nil,
+                                                              payload: '{}')
+            .and_raise(@exception)
+          expect(RestClient::Request).to receive(:execute).exactly(4).times
+
+          expect { retry_instance.send(http_method, '/test') }.to raise_error { |error|
+            expect(error).to be_a(Auth0::RateLimitEncountered)
+          }
+        end
+
+        it "should retry 2 times when retry_count is set to 2" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+          retry_instance.retry_count = 2
+
+          @exception.response = StubResponse.new({}, false, 429)
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                              url: 'https://auth0.com/test',
+                                                              timeout: nil,
+                                                              headers: nil,
+                                                              payload: '{}')
+            .and_raise(@exception)
+          expect(RestClient::Request).to receive(:execute).exactly(3).times
+
+          expect { retry_instance.send(http_method, '/test') }.to raise_error { |error|
+            expect(error).to be_a(Auth0::RateLimitEncountered)
+          }
+        end
+
+        it "should not retry when retry_count is set to 0" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+          retry_instance.retry_count = 0
+
+          @exception.response = StubResponse.new({}, false, 429)
+
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                              url: 'https://auth0.com/test',
+                                                              timeout: nil,
+                                                              headers: nil,
+                                                              payload: '{}')
+            .and_raise(@exception)
+
+          expect(RestClient::Request).to receive(:execute).exactly(1).times
+          expect { retry_instance.send(http_method, '/test') }.to raise_error { |error|
+            expect(error).to be_a(Auth0::RateLimitEncountered)
+          }
+        end
+
+        it "should have have random retry times grow with jitter backoff" do
+          retry_instance = DummyClassForProxy.new
+          retry_instance.extend(Auth0::Mixins::HTTPProxy)
+          retry_instance.base_uri = "https://auth0.com"
+          retry_instance.retry_count = 2
+          time_entries = []
+          @time_start
+
+          @exception.response = StubResponse.new({}, false, 429)
+          allow(RestClient::Request).to receive(:execute).with(method: http_method,
+                                                               url: 'https://auth0.com/test',
+                                                               timeout: nil,
+                                                               headers: nil,
+                                                               payload: '{}') do
+
+            time_entries.push(Time.now.to_f - @time_start.to_f)
+            @time_start = Time.now.to_f # restart the clock
+            raise @exception
+          end
+
+          @time_start = Time.now.to_f #start the clock
+          retry_instance.send(http_method, '/test') rescue nil
+          time_entries_first_set = time_entries.shift(time_entries.length)
+
+          retry_instance.send(http_method, '/test') rescue nil
+          time_entries.each_with_index do |entry, index|
+            if index > 0 #skip the first request
+              expect(entry != time_entries_first_set[index])
+            end
+          end
+        end
+      end
     end
   end
 end

spec/lib/auth0/mixins/initializer_spec.rb

@@ -26,5 +26,12 @@ class MockClass
 
       expect(instance.instance_variable_get('@token')).to eq('123')
     end
+
+    it 'sets retry_count when passed' do
+      params[:token] = '123'
+      params[:retry_count] = 10
+
+      expect(instance.instance_variable_get('@retry_count')).to eq(10)
+    end
   end
 end