From 895fedb96f4675d18d52ad320e4ba9c39eba3de7 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt Date: Wed, 16 Jun 2021 20:50:57 -0700 Subject: [PATCH 1/4] 0.3.2 --- VERSION | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index a2268e2..9fc80f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 \ No newline at end of file +0.3.2 \ No newline at end of file diff --git a/go.mod b/go.mod index 0fe0b64..aaf5a28 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/fallenstedt/twitter-stream -go 1.15 +go 1.16 From 60435c5e130630e3841e561b6ec51f16c7b523f1 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt Date: Wed, 16 Jun 2021 21:28:13 -0700 Subject: [PATCH 2/4] format and add 429 retry with backoff --- http_client.go | 32 ++++++++++++++++++++++++++++---- http_client_mock.go | 10 +++++----- rules.go | 8 ++++---- stream.go | 9 ++++----- stream_test.go | 1 - 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/http_client.go b/http_client.go index e8a355a..64851ec 100644 --- a/http_client.go +++ b/http_client.go @@ -5,17 +5,19 @@ import ( "errors" "io/ioutil" "log" + "math" "net/http" "strings" + "time" ) type twitterEndpoints map[string]string + var endpoints = make(twitterEndpoints) type ( // IHttpClient is the interface the httpClient struct implements. IHttpClient interface { - newHttpRequest(opts *requestOpts) (*http.Response, error) getRules() (*http.Response, error) getSearchStream(queryParams string) (*http.Response, error) @@ -28,6 +30,7 @@ type ( } requestOpts struct { + Retries uint8 Method string Url string Body string @@ -45,7 +48,7 @@ func newHttpClient(token string) *httpClient { return &httpClient{token} } -func (t *httpClient) getRules() (*http.Response, error) { +func (t *httpClient) getRules() (*http.Response, error) { res, err := t.newHttpRequest(&requestOpts{ Method: "GET", Url: endpoints["rules"], @@ -56,7 +59,7 @@ func (t *httpClient) getRules() (*http.Response, error) { } func (t *httpClient) addRules(queryParams string, body string) (*http.Response, error) { - url, err := t.generateUrl("rules", queryParams) + url, err := t.generateUrl("rules", queryParams) if err != nil { return nil, err @@ -147,9 +150,21 @@ func (t *httpClient) newHttpRequest(opts *requestOpts) (*http.Response, error) { return nil, err } + // Retry with backoff if 429 + if resp.StatusCode == 429 { + log.Printf("Retrying network request %s with backoff", opts.Url) + + delay := t.getBackOffTime(opts.Retries) + log.Printf("Sleeping for %v seconds", delay) + time.Sleep(delay) + + opts.Retries += 1 + return t.newHttpRequest(opts) + } + // Reject if 400 or greater if resp.StatusCode >= 400 { - log.Printf("Network Request failed: %v", resp.StatusCode) + log.Printf("Network Request at %s failed: %v", opts.Url, resp.StatusCode) body, _ := ioutil.ReadAll(resp.Body) msg := "Network request failed: " + string(body) return nil, errors.New(msg) @@ -157,3 +172,12 @@ func (t *httpClient) newHttpRequest(opts *requestOpts) (*http.Response, error) { return resp, nil } + +func (t *httpClient) getBackOffTime(retries uint8) time.Duration { + exponentialBackoffCeilingSecs := 30 + delaySecs := int(math.Floor((math.Pow(2, float64(retries)) - 1) * 0.5)) + if delaySecs > exponentialBackoffCeilingSecs { + delaySecs = 30 + } + return time.Duration(delaySecs) * time.Second +} diff --git a/http_client_mock.go b/http_client_mock.go index 0c8dab5..6e354ae 100644 --- a/http_client_mock.go +++ b/http_client_mock.go @@ -3,12 +3,12 @@ package twitterstream import "net/http" type mockHttpClient struct { - token string - MockNewHttpRequest func(opts *requestOpts) (*http.Response, error) + token string + MockNewHttpRequest func(opts *requestOpts) (*http.Response, error) MockGetSearchStream func(queryParams string) (*http.Response, error) - MockGetRules func() (*http.Response, error) - MockAddRules func(queryParams string, body string) (*http.Response, error) - MockGenerateUrl func (name string, queryParams string) (string, error) + MockGetRules func() (*http.Response, error) + MockAddRules func(queryParams string, body string) (*http.Response, error) + MockGenerateUrl func(name string, queryParams string) (string, error) } func newHttpClientMock(token string) *mockHttpClient { diff --git a/rules.go b/rules.go index 0844d64..3849f0a 100644 --- a/rules.go +++ b/rules.go @@ -16,8 +16,8 @@ type ( } rulesResponse struct { - Data []rulesResponseValue - Meta rulesResponseMeta + Data []rulesResponseValue + Meta rulesResponseMeta Errors []rulesResponseError } @@ -32,9 +32,9 @@ type ( } rulesResponseError struct { Value string `json:"value"` - Id string `json:"id"` + Id string `json:"id"` Title string `json:"title"` - Type string `json:"type"` + Type string `json:"type"` } addRulesResponseMetaSummary struct { diff --git a/stream.go b/stream.go index 058307a..e142374 100644 --- a/stream.go +++ b/stream.go @@ -4,7 +4,6 @@ import ( "net/http" ) - type ( // UnmarshalHook is a function that will unmarshal json. UnmarshalHook func([]byte) (interface{}, error) @@ -29,10 +28,10 @@ type ( // in a separate goroutine is not recommended because the Go bytes.Buffer is not thread safe. Stream struct { unmarshalHook UnmarshalHook - messages chan StreamMessage - httpClient IHttpClient - done chan struct{} - reader IStreamResponseBodyReader + messages chan StreamMessage + httpClient IHttpClient + done chan struct{} + reader IStreamResponseBodyReader } ) diff --git a/stream_test.go b/stream_test.go index aa266f0..ed92158 100644 --- a/stream_test.go +++ b/stream_test.go @@ -91,7 +91,6 @@ func TestStartStream(t *testing.T) { expected, _ := tt.result.Data.([]byte) res, _ := r.Data.([]byte) - if string(expected) != string(res) { t.Errorf("got %v, want %s", result, tt.result) } From 29825d112e90f55bef875ead2ea6b9dc6e1834bb Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt Date: Thu, 17 Jun 2021 22:18:10 -0700 Subject: [PATCH 3/4] refactor httpclient into its own package --- example/go.mod | 2 +- .../http_client_mock.go | 16 ++--- http_client.go => httpclient/httpclient.go | 61 ++++++++++--------- rules.go | 9 +-- rules_test.go | 5 +- stream.go | 7 ++- stream_test.go | 11 ++-- token_generator.go | 13 ++-- token_generator_test.go | 9 +-- twitterstream.go | 6 +- 10 files changed, 74 insertions(+), 65 deletions(-) rename http_client_mock.go => httpclient/http_client_mock.go (60%) rename http_client.go => httpclient/httpclient.go (66%) diff --git a/example/go.mod b/example/go.mod index dbf8fd3..08c71df 100644 --- a/example/go.mod +++ b/example/go.mod @@ -2,6 +2,6 @@ module github.com/fallenstedt/twitter-stream/example replace github.com/fallenstedt/twitter-stream => /Users/alex/Projects/twitter-stream/ -go 1.15 +go 1.16 require github.com/fallenstedt/twitter-stream v0.2.1 diff --git a/http_client_mock.go b/httpclient/http_client_mock.go similarity index 60% rename from http_client_mock.go rename to httpclient/http_client_mock.go index 6e354ae..6250708 100644 --- a/http_client_mock.go +++ b/httpclient/http_client_mock.go @@ -1,36 +1,36 @@ -package twitterstream +package httpclient import "net/http" type mockHttpClient struct { token string - MockNewHttpRequest func(opts *requestOpts) (*http.Response, error) + MockNewHttpRequest func(opts *RequestOpts) (*http.Response, error) MockGetSearchStream func(queryParams string) (*http.Response, error) MockGetRules func() (*http.Response, error) MockAddRules func(queryParams string, body string) (*http.Response, error) MockGenerateUrl func(name string, queryParams string) (string, error) } -func newHttpClientMock(token string) *mockHttpClient { +func NewHttpClientMock(token string) *mockHttpClient { return &mockHttpClient{token: token} } -func (t *mockHttpClient) generateUrl(name string, queryParams string) (string, error) { +func (t *mockHttpClient) GenerateUrl(name string, queryParams string) (string, error) { return t.MockGenerateUrl(name, queryParams) } -func (t *mockHttpClient) getRules() (*http.Response, error) { +func (t *mockHttpClient) GetRules() (*http.Response, error) { return t.MockGetRules() } -func (t *mockHttpClient) addRules(queryParams string, body string) (*http.Response, error) { +func (t *mockHttpClient) AddRules(queryParams string, body string) (*http.Response, error) { return t.MockAddRules(queryParams, body) } -func (t *mockHttpClient) getSearchStream(queryParams string) (*http.Response, error) { +func (t *mockHttpClient) GetSearchStream(queryParams string) (*http.Response, error) { return t.MockGetSearchStream(queryParams) } -func (t *mockHttpClient) newHttpRequest(opts *requestOpts) (*http.Response, error) { +func (t *mockHttpClient) NewHttpRequest(opts *RequestOpts) (*http.Response, error) { return t.MockNewHttpRequest(opts) } diff --git a/http_client.go b/httpclient/httpclient.go similarity index 66% rename from http_client.go rename to httpclient/httpclient.go index 64851ec..ec5871e 100644 --- a/http_client.go +++ b/httpclient/httpclient.go @@ -1,4 +1,5 @@ -package twitterstream +package httpclient + import ( "bytes" @@ -13,59 +14,59 @@ import ( type twitterEndpoints map[string]string -var endpoints = make(twitterEndpoints) +var Endpoints = make(twitterEndpoints) type ( // IHttpClient is the interface the httpClient struct implements. IHttpClient interface { - newHttpRequest(opts *requestOpts) (*http.Response, error) - getRules() (*http.Response, error) - getSearchStream(queryParams string) (*http.Response, error) - addRules(queryParams string, body string) (*http.Response, error) - generateUrl(name string, queryParams string) (string, error) + NewHttpRequest(opts *RequestOpts) (*http.Response, error) + GetRules() (*http.Response, error) + GetSearchStream(queryParams string) (*http.Response, error) + AddRules(queryParams string, body string) (*http.Response, error) + GenerateUrl(name string, queryParams string) (string, error) } httpClient struct { token string } - requestOpts struct { + RequestOpts struct { Retries uint8 Method string Url string Body string Headers []struct { - key string - value string + Key string + Value string } } ) -func newHttpClient(token string) *httpClient { - endpoints["rules"] = "https://api.twitter.com/2/tweets/search/stream/rules" - endpoints["stream"] = "https://api.twitter.com/2/tweets/search/stream" - endpoints["token"] = "https://api.twitter.com/oauth2/token" +func NewHttpClient(token string) *httpClient { + Endpoints["rules"] = "https://api.twitter.com/2/tweets/search/stream/rules" + Endpoints["stream"] = "https://api.twitter.com/2/tweets/search/stream" + Endpoints["token"] = "https://api.twitter.com/oauth2/token" return &httpClient{token} } -func (t *httpClient) getRules() (*http.Response, error) { - res, err := t.newHttpRequest(&requestOpts{ +func (t *httpClient) GetRules() (*http.Response, error) { + res, err := t.NewHttpRequest(&RequestOpts{ Method: "GET", - Url: endpoints["rules"], + Url: Endpoints["rules"], Body: "", }) return res, err } -func (t *httpClient) addRules(queryParams string, body string) (*http.Response, error) { - url, err := t.generateUrl("rules", queryParams) +func (t *httpClient) AddRules(queryParams string, body string) (*http.Response, error) { + url, err := t.GenerateUrl("rules", queryParams) if err != nil { return nil, err } - res, err := t.newHttpRequest(&requestOpts{ + res, err := t.NewHttpRequest(&RequestOpts{ Method: "POST", Url: url, Body: body, @@ -78,15 +79,15 @@ func (t *httpClient) addRules(queryParams string, body string) (*http.Response, return res, nil } -func (t *httpClient) getSearchStream(queryParams string) (*http.Response, error) { +func (t *httpClient) GetSearchStream(queryParams string) (*http.Response, error) { // Make an HTTP GET request to GET /2/tweets/search/stream - url, err := t.generateUrl("stream", queryParams) + url, err := t.GenerateUrl("stream", queryParams) if err != nil { return nil, err } - res, err := t.newHttpRequest(&requestOpts{ + res, err := t.NewHttpRequest(&RequestOpts{ Method: "GET", Url: url, }) @@ -98,12 +99,12 @@ func (t *httpClient) getSearchStream(queryParams string) (*http.Response, error) return res, nil } -func (t *httpClient) generateUrl(name string, queryParams string) (string, error) { +func (t *httpClient) GenerateUrl(name string, queryParams string) (string, error) { var url string if len(queryParams) > 0 { - url = endpoints[name] + queryParams + url = Endpoints[name] + queryParams } else { - url = endpoints[name] + url = Endpoints[name] } if len(url) == 0 || !strings.HasPrefix(url, "https://api.twitter.com") { @@ -113,7 +114,7 @@ func (t *httpClient) generateUrl(name string, queryParams string) (string, error } } -func (t *httpClient) newHttpRequest(opts *requestOpts) (*http.Response, error) { +func (t *httpClient) NewHttpRequest(opts *RequestOpts) (*http.Response, error) { client := &http.Client{} var req *http.Request @@ -134,11 +135,11 @@ func (t *httpClient) newHttpRequest(opts *requestOpts) (*http.Response, error) { req.Header.Set("Content-Type", "application/json") if len(opts.Headers) > 0 { for _, header := range opts.Headers { - req.Header.Set(header.key, header.value) + req.Header.Set(header.Key, header.Value) } } - // Set token if this client has a token set + // Set token if this httpclient has a token set if len(t.token) > 0 { req.Header.Set("Authorization", "Bearer "+t.token) } @@ -159,7 +160,7 @@ func (t *httpClient) newHttpRequest(opts *requestOpts) (*http.Response, error) { time.Sleep(delay) opts.Retries += 1 - return t.newHttpRequest(opts) + return t.NewHttpRequest(opts) } // Reject if 400 or greater diff --git a/rules.go b/rules.go index 3849f0a..cd656c1 100644 --- a/rules.go +++ b/rules.go @@ -2,6 +2,7 @@ package twitterstream import ( "encoding/json" + "github.com/fallenstedt/twitter-stream/httpclient" ) type ( @@ -12,7 +13,7 @@ type ( } rules struct { - httpClient IHttpClient + httpClient httpclient.IHttpClient } rulesResponse struct { @@ -43,7 +44,7 @@ type ( } ) -func newRules(httpClient IHttpClient) *rules { +func newRules(httpClient httpclient.IHttpClient) *rules { return &rules{httpClient: httpClient} } @@ -51,7 +52,7 @@ func newRules(httpClient IHttpClient) *rules { // The body is a stringified object. // Learn about the possible error messages returned here https://developer.twitter.com/en/support/twitter-api/error-troubleshooting. func (t *rules) AddRules(body string, dryRun bool) (*rulesResponse, error) { - res, err := t.httpClient.addRules(func() string { + res, err := t.httpClient.AddRules(func() string { if dryRun { return "?dry_run=true" } else { @@ -75,7 +76,7 @@ func (t *rules) AddRules(body string, dryRun bool) (*rulesResponse, error) { // GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. func (t *rules) GetRules() (*rulesResponse, error) { - res, err := t.httpClient.getRules() + res, err := t.httpClient.GetRules() if err != nil { return nil, err diff --git a/rules_test.go b/rules_test.go index ff47fc0..20778d4 100644 --- a/rules_test.go +++ b/rules_test.go @@ -3,6 +3,7 @@ package twitterstream import ( "bytes" "fmt" + "github.com/fallenstedt/twitter-stream/httpclient" "io/ioutil" "net/http" "testing" @@ -69,7 +70,7 @@ func TestAddRules(t *testing.T) { testName := fmt.Sprintf("TestAddRules (%d) %s", i, tt.body) t.Run(testName, func(t *testing.T) { - mockClient := newHttpClientMock("sometoken") + mockClient := httpclient.NewHttpClientMock("sometoken") mockClient.MockAddRules = tt.mockRequest instance := newRules(mockClient) @@ -158,7 +159,7 @@ func TestGetRules(t *testing.T) { testName := fmt.Sprintf("TestGetRules (%d)", i) t.Run(testName, func(t *testing.T) { - mockClient := newHttpClientMock("sometoken") + mockClient := httpclient.NewHttpClientMock("sometoken") mockClient.MockGetRules = tt.mockRequest instance := newRules(mockClient) diff --git a/stream.go b/stream.go index e142374..01bb860 100644 --- a/stream.go +++ b/stream.go @@ -1,6 +1,7 @@ package twitterstream import ( + "github.com/fallenstedt/twitter-stream/httpclient" "net/http" ) @@ -29,13 +30,13 @@ type ( Stream struct { unmarshalHook UnmarshalHook messages chan StreamMessage - httpClient IHttpClient + httpClient httpclient.IHttpClient done chan struct{} reader IStreamResponseBodyReader } ) -func newStream(httpClient IHttpClient, reader IStreamResponseBodyReader) *Stream { +func newStream(httpClient httpclient.IHttpClient, reader IStreamResponseBodyReader) *Stream { return &Stream{ unmarshalHook: func(bytes []byte) (interface{}, error) { return bytes, nil @@ -69,7 +70,7 @@ func (s *Stream) StopStream() { // See available query params here https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference/get-tweets-search-stream. // See an example here: https://developer.twitter.com/en/docs/twitter-api/expansions. func (s *Stream) StartStream(optionalQueryParams string) error { - res, err := s.httpClient.getSearchStream(optionalQueryParams) + res, err := s.httpClient.GetSearchStream(optionalQueryParams) if err != nil { return err diff --git a/stream_test.go b/stream_test.go index ed92158..123d41f 100644 --- a/stream_test.go +++ b/stream_test.go @@ -3,6 +3,7 @@ package twitterstream import ( "bytes" "fmt" + "github.com/fallenstedt/twitter-stream/httpclient" "io" "io/ioutil" "net/http" @@ -10,7 +11,7 @@ import ( ) func TestGetMessages(t *testing.T) { - client := newHttpClientMock("foobar") + client := httpclient.NewHttpClientMock("foobar") reader := newStreamResponseBodyReader() instance := newStream(client, reader) @@ -22,7 +23,7 @@ func TestGetMessages(t *testing.T) { } func TestStopStream(t *testing.T) { - client := newHttpClientMock("foobar") + client := httpclient.NewHttpClientMock("foobar") reader := newStreamResponseBodyReader() instance := newStream(client, reader) @@ -36,13 +37,13 @@ func TestStopStream(t *testing.T) { func TestStartStream(t *testing.T) { var tests = []struct { - givenMockHttpRequestToStreamReturns func() IHttpClient + givenMockHttpRequestToStreamReturns func() httpclient.IHttpClient givenMockStreamResponseBodyReader func() IStreamResponseBodyReader result StreamMessage }{ { - func() IHttpClient { - mockClient := newHttpClientMock("foobar") + func() httpclient.IHttpClient { + mockClient := httpclient.NewHttpClientMock("foobar") mockClient.MockGetSearchStream = func(queryParams string) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusOK, diff --git a/token_generator.go b/token_generator.go index 9d7d7ad..655b802 100644 --- a/token_generator.go +++ b/token_generator.go @@ -3,6 +3,7 @@ package twitterstream import ( "encoding/base64" "encoding/json" + "github.com/fallenstedt/twitter-stream/httpclient" ) type ( @@ -12,7 +13,7 @@ type ( SetApiKeyAndSecret(apiKey, apiSecret string) *tokenGenerator } tokenGenerator struct { - httpClient IHttpClient + httpClient httpclient.IHttpClient apiKey string apiSecret string } @@ -22,7 +23,7 @@ type ( } ) -func newTokenGenerator(httpClient IHttpClient) *tokenGenerator { +func newTokenGenerator(httpClient httpclient.IHttpClient) *tokenGenerator { return &tokenGenerator{httpClient: httpClient} } @@ -36,16 +37,16 @@ func (a *tokenGenerator) SetApiKeyAndSecret(apiKey, apiSecret string) *tokenGene // RequestBearerToken requests a bearer token from twitter using the apiKey and apiSecret. func (a *tokenGenerator) RequestBearerToken() (*requestBearerTokenResponse, error) { - resp, err := a.httpClient.newHttpRequest(&requestOpts{ + resp, err := a.httpClient.NewHttpRequest(&httpclient.RequestOpts{ Headers: []struct { - key string - value string + Key string + Value string }{ {"Content-Type", "application/x-www-form-urlencoded;charset=UTF-8"}, {"Authorization", "Basic " + a.base64EncodeKeys()}, }, Method: "POST", - Url: endpoints["token"], + Url: httpclient.Endpoints["token"], Body: "grant_type=client_credentials", }) diff --git a/token_generator_test.go b/token_generator_test.go index 005862c..40decc7 100644 --- a/token_generator_test.go +++ b/token_generator_test.go @@ -3,6 +3,7 @@ package twitterstream import ( "bytes" "fmt" + "github.com/fallenstedt/twitter-stream/httpclient" "io/ioutil" "net/http" "testing" @@ -21,7 +22,7 @@ func TestSetApiKeyAndSecret(t *testing.T) { for i, tt := range tests { testName := fmt.Sprintf("(%d) %s %s", i, tt.apiKey, tt.apiSecret) t.Run(testName, func(t *testing.T) { - result := newTokenGenerator(newHttpClientMock("")) + result := newTokenGenerator(httpclient.NewHttpClientMock("")) result.SetApiKeyAndSecret(tt.apiKey, tt.apiSecret) if result.apiKey != tt.result.apiKey { @@ -37,10 +38,10 @@ func TestSetApiKeyAndSecret(t *testing.T) { func TestRequestBearerToken(t *testing.T) { var tests = []struct { - mockRequest func(opts *requestOpts) (*http.Response, error) + mockRequest func(opts *httpclient.RequestOpts) (*http.Response, error) result *requestBearerTokenResponse }{ - {func(opts *requestOpts) (*http.Response, error) { + {func(opts *httpclient.RequestOpts) (*http.Response, error) { json := `{ "token_type": "bearer", @@ -64,7 +65,7 @@ func TestRequestBearerToken(t *testing.T) { testName := fmt.Sprintf("(%d)", i) t.Run(testName, func(t *testing.T) { - mockClient := newHttpClientMock("") + mockClient := httpclient.NewHttpClientMock("") mockClient.MockNewHttpRequest = tt.mockRequest instance := newTokenGenerator(mockClient) diff --git a/twitterstream.go b/twitterstream.go index 2c47a44..7e58292 100644 --- a/twitterstream.go +++ b/twitterstream.go @@ -1,6 +1,8 @@ // Package twitterstream provides an easy way to stream tweets using Twitter's v2 Streaming API. package twitterstream +import "github.com/fallenstedt/twitter-stream/httpclient" + type twitterApi struct { Rules IRules Stream IStream @@ -8,7 +10,7 @@ type twitterApi struct { // NewTokenGenerator creates a tokenGenerator which can request a Bearer token using a twitter api key and secret. func NewTokenGenerator() *tokenGenerator { - client := newHttpClient("") + client := httpclient.NewHttpClient("") tokenGenerator := newTokenGenerator(client) return tokenGenerator } @@ -16,7 +18,7 @@ func NewTokenGenerator() *tokenGenerator { // NewTwitterStream consumes a twitter Bearer token. // It is used to interact with Twitter's v2 filtered streaming API func NewTwitterStream(token string) *twitterApi { - client := newHttpClient(token) + client := httpclient.NewHttpClient(token) rules := newRules(client) stream := newStream(client, newStreamResponseBodyReader()) return &twitterApi{Rules: rules, Stream: stream} From 58472fadc16753184f622331ae966800f7fbf103 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt Date: Sat, 19 Jun 2021 15:30:55 -0700 Subject: [PATCH 4/4] add httpresponseparser tests --- httpclient/http_response_parser.go | 55 +++++++++++++++++++ httpclient/http_response_parser_test.go | 70 +++++++++++++++++++++++++ httpclient/httpclient.go | 55 ++++--------------- httpclient/types.go | 12 +++++ 4 files changed, 147 insertions(+), 45 deletions(-) create mode 100644 httpclient/http_response_parser.go create mode 100644 httpclient/http_response_parser_test.go create mode 100644 httpclient/types.go diff --git a/httpclient/http_response_parser.go b/httpclient/http_response_parser.go new file mode 100644 index 0000000..6c928cd --- /dev/null +++ b/httpclient/http_response_parser.go @@ -0,0 +1,55 @@ +package httpclient + +import ( + "errors" + "fmt" + "io/ioutil" + "log" + "math" + "net/http" + "time" +) + +// httpResponseParser is a struct that will retry network requests if the response has a status code of 429. +type httpResponseParser struct{} + +func (h httpResponseParser) handleResponse(resp *http.Response, opts *RequestOpts, fn func(opts *RequestOpts) (*http.Response, error)) (*http.Response, error) { + // Retry with backoff if 429 + if resp.StatusCode == 429 { + log.Printf("Retrying network request %s with backoff", opts.Url) + + delay := h.getBackOffTime(opts.Retries) + log.Printf("Sleeping for %v seconds", delay) + time.Sleep(delay) + + opts.Retries += 1 + + return fn(opts) + } + + // Reject if 400 or greater + if resp.StatusCode >= 400 { + log.Printf("Network Request at %s failed: %v", opts.Url, resp.StatusCode) + + var msg string + if resp.Body != nil { + body, _ := ioutil.ReadAll(resp.Body) + msg = "Network request failed: " + string(body) + } else { + msg = "Network request failed with status" + fmt.Sprint(resp.StatusCode) + } + + return nil, errors.New(msg) + } + + return resp, nil +} + +func (h httpResponseParser) getBackOffTime(retries uint8) time.Duration { + exponentialBackoffCeilingSecs := 30 + delaySecs := int(math.Floor((math.Pow(2, float64(retries)) - 1) * 0.5)) + if delaySecs > exponentialBackoffCeilingSecs { + delaySecs = 30 + } + return time.Duration(delaySecs) * time.Second +} diff --git a/httpclient/http_response_parser_test.go b/httpclient/http_response_parser_test.go new file mode 100644 index 0000000..4eaa7f8 --- /dev/null +++ b/httpclient/http_response_parser_test.go @@ -0,0 +1,70 @@ +package httpclient + +import ( + "net/http" + "testing" +) + +func givenHttpResponseParserInstance() *httpResponseParser { + return new(httpResponseParser) +} + +func givenFakeHttpResponse(statusCode int) *http.Response { + res := new(http.Response) + res.StatusCode = statusCode + return res +} + +func TestHandleResponseShouldReturnIf200(t *testing.T) { + instance := givenHttpResponseParserInstance() + opts := new(RequestOpts) + resp := givenFakeHttpResponse(200) + + result, err := instance.handleResponse(resp, opts, func(o *RequestOpts) (*http.Response, error) { + return nil, nil + }) + + if err != nil { + t.Errorf("Expected not error, got %v", err) + } + + if result.StatusCode != 200 { + t.Errorf("Expected a status code of 200") + } +} + +func TestHandleResponseShouldRetryRequestIf429(t *testing.T) { + instance := givenHttpResponseParserInstance() + opts := new(RequestOpts) + resp := givenFakeHttpResponse(429) + + result, err := instance.handleResponse(resp, opts, func(o *RequestOpts) (*http.Response, error) { + return givenFakeHttpResponse(200), nil + }) + + if opts.Retries != 1 { + t.Errorf("Expected atleast on retry attempt, got %v", opts.Retries) + } + + if err != nil { + t.Errorf("Expected not error, got %v", err) + } + + if result.StatusCode != 200 { + t.Errorf("Expected a status code of 200") + } +} + +func TestHandleResponseShouldRejectIf400OrHigher(t *testing.T) { + instance := givenHttpResponseParserInstance() + opts := new(RequestOpts) + resp := givenFakeHttpResponse(401) + + _, err := instance.handleResponse(resp, opts, func(o *RequestOpts) (*http.Response, error) { + return nil, nil + }) + + if err == nil { + t.Errorf("Expected error, got nil") + } +} diff --git a/httpclient/httpclient.go b/httpclient/httpclient.go index ec5871e..d2926f2 100644 --- a/httpclient/httpclient.go +++ b/httpclient/httpclient.go @@ -1,19 +1,16 @@ package httpclient - import ( "bytes" "errors" - "io/ioutil" "log" - "math" "net/http" "strings" - "time" ) type twitterEndpoints map[string]string +// Endpoints is a map of twitter endpoints used to manage rules and streams. var Endpoints = make(twitterEndpoints) type ( @@ -29,19 +26,9 @@ type ( httpClient struct { token string } - - RequestOpts struct { - Retries uint8 - Method string - Url string - Body string - Headers []struct { - Key string - Value string - } - } ) +// NewHttpClient constructs a an HttpClient to interact with twitter. func NewHttpClient(token string) *httpClient { Endpoints["rules"] = "https://api.twitter.com/2/tweets/search/stream/rules" Endpoints["stream"] = "https://api.twitter.com/2/tweets/search/stream" @@ -49,6 +36,7 @@ func NewHttpClient(token string) *httpClient { return &httpClient{token} } +// GetRules will return the current rules available for a specific API key. func (t *httpClient) GetRules() (*http.Response, error) { res, err := t.NewHttpRequest(&RequestOpts{ Method: "GET", @@ -59,6 +47,7 @@ func (t *httpClient) GetRules() (*http.Response, error) { return res, err } +// AddRules will add rules for you to stream with. func (t *httpClient) AddRules(queryParams string, body string) (*http.Response, error) { url, err := t.GenerateUrl("rules", queryParams) @@ -79,6 +68,7 @@ func (t *httpClient) AddRules(queryParams string, body string) (*http.Response, return res, nil } +// GetSearchStream will start the stream with twitter. func (t *httpClient) GetSearchStream(queryParams string) (*http.Response, error) { // Make an HTTP GET request to GET /2/tweets/search/stream url, err := t.GenerateUrl("stream", queryParams) @@ -99,6 +89,7 @@ func (t *httpClient) GetSearchStream(queryParams string) (*http.Response, error) return res, nil } +// GenerateUrl is a utility function for httpclient package to generate a valid url for api.twitter. func (t *httpClient) GenerateUrl(name string, queryParams string) (string, error) { var url string if len(queryParams) > 0 { @@ -114,8 +105,8 @@ func (t *httpClient) GenerateUrl(name string, queryParams string) (string, error } } +// NewHttpRequest performs an authenticated http request with twitter with the token this httpclient has. func (t *httpClient) NewHttpRequest(opts *RequestOpts) (*http.Response, error) { - client := &http.Client{} var req *http.Request var err error @@ -145,40 +136,14 @@ func (t *httpClient) NewHttpRequest(opts *RequestOpts) (*http.Response, error) { } // Perform network request + client := &http.Client{} resp, err := client.Do(req) if err != nil { log.Printf("Failed to perform request for %s: %v", opts.Url, err) return nil, err } - // Retry with backoff if 429 - if resp.StatusCode == 429 { - log.Printf("Retrying network request %s with backoff", opts.Url) - - delay := t.getBackOffTime(opts.Retries) - log.Printf("Sleeping for %v seconds", delay) - time.Sleep(delay) - - opts.Retries += 1 - return t.NewHttpRequest(opts) - } - - // Reject if 400 or greater - if resp.StatusCode >= 400 { - log.Printf("Network Request at %s failed: %v", opts.Url, resp.StatusCode) - body, _ := ioutil.ReadAll(resp.Body) - msg := "Network request failed: " + string(body) - return nil, errors.New(msg) - } + responseParser := new(httpResponseParser) + return responseParser.handleResponse(resp, opts, t.NewHttpRequest) - return resp, nil -} - -func (t *httpClient) getBackOffTime(retries uint8) time.Duration { - exponentialBackoffCeilingSecs := 30 - delaySecs := int(math.Floor((math.Pow(2, float64(retries)) - 1) * 0.5)) - if delaySecs > exponentialBackoffCeilingSecs { - delaySecs = 30 - } - return time.Duration(delaySecs) * time.Second } diff --git a/httpclient/types.go b/httpclient/types.go new file mode 100644 index 0000000..a461ca2 --- /dev/null +++ b/httpclient/types.go @@ -0,0 +1,12 @@ +package httpclient + +type RequestOpts struct { + Retries uint8 + Method string + Url string + Body string + Headers []struct { + Key string + Value string + } +}