From 3fe6fe0c0a8293aa8172375898ede544b13fe4b6 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 4 Dec 2021 11:43:43 -0800 Subject: [PATCH 01/12] move rules to own directory --- example/go.mod | 2 +- rules.go => rules/rules.go | 4 ++-- rules_test.go => rules/rules_test.go | 6 +++--- twitterstream.go | 9 ++++++--- 4 files changed, 12 insertions(+), 9 deletions(-) rename rules.go => rules/rules.go (96%) rename rules_test.go => rules/rules_test.go (98%) diff --git a/example/go.mod b/example/go.mod index c420b3a..ca6203f 100644 --- a/example/go.mod +++ b/example/go.mod @@ -1,6 +1,6 @@ module github.com/fallenstedt/twitter-stream/example -replace github.com/fallenstedt/twitter-stream => /Users/alex/Code/twitter-stream/ +replace github.com/fallenstedt/twitter-stream => ../ go 1.16 diff --git a/rules.go b/rules/rules.go similarity index 96% rename from rules.go rename to rules/rules.go index cffb97b..b563cf4 100644 --- a/rules.go +++ b/rules/rules.go @@ -1,4 +1,4 @@ -package twitterstream +package rules import ( "encoding/json" @@ -44,7 +44,7 @@ type ( } ) -func newRules(httpClient httpclient.IHttpClient) IRules { +func NewRules(httpClient httpclient.IHttpClient) IRules { return &rules{httpClient: httpClient} } diff --git a/rules_test.go b/rules/rules_test.go similarity index 98% rename from rules_test.go rename to rules/rules_test.go index 20778d4..5d8ece3 100644 --- a/rules_test.go +++ b/rules/rules_test.go @@ -1,4 +1,4 @@ -package twitterstream +package rules import ( "bytes" @@ -73,7 +73,7 @@ func TestAddRules(t *testing.T) { mockClient := httpclient.NewHttpClientMock("sometoken") mockClient.MockAddRules = tt.mockRequest - instance := newRules(mockClient) + instance := NewRules(mockClient) result, err := instance.AddRules(tt.body, false) if err != nil { @@ -162,7 +162,7 @@ func TestGetRules(t *testing.T) { mockClient := httpclient.NewHttpClientMock("sometoken") mockClient.MockGetRules = tt.mockRequest - instance := newRules(mockClient) + instance := NewRules(mockClient) result, err := instance.GetRules() if err != nil { diff --git a/twitterstream.go b/twitterstream.go index 06f52cd..02d5000 100644 --- a/twitterstream.go +++ b/twitterstream.go @@ -1,10 +1,13 @@ // Package twitterstream provides an easy way to stream tweets using Twitter's v2 Streaming API. package twitterstream -import "github.com/fallenstedt/twitter-stream/httpclient" +import ( + "github.com/fallenstedt/twitter-stream/httpclient" + "github.com/fallenstedt/twitter-stream/rules" +) type TwitterApi struct { - Rules IRules + Rules rules.IRules Stream IStream } @@ -20,7 +23,7 @@ func NewTokenGenerator() ITokenGenerator { // It is used to interact with Twitter's v2 filtered streaming API func NewTwitterStream(token string) *TwitterApi { client := httpclient.NewHttpClient(token) - rules := newRules(client) + rules := rules.NewRules(client) stream := newStream(client, newStreamResponseBodyReader()) return &TwitterApi{Rules: rules, Stream: stream} } From 49ceb7e83bfe3c06205693a5c035077c9c20eb40 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 4 Dec 2021 11:45:34 -0800 Subject: [PATCH 02/12] move stream to its own directory --- example/restart_stream_example.go | 5 +++-- stream.go => stream/stream.go | 4 ++-- stream_test.go => stream/stream_test.go | 10 +++++----- stream_utils.go => stream/stream_utils.go | 6 +++--- stream_utils_mock.go => stream/stream_utils_mock.go | 2 +- twitterstream.go | 5 +++-- 6 files changed, 17 insertions(+), 15 deletions(-) rename stream.go => stream/stream.go (97%) rename stream_test.go => stream/stream_test.go (93%) rename stream_utils.go => stream/stream_utils.go (95%) rename stream_utils_mock.go => stream/stream_utils_mock.go (94%) diff --git a/example/restart_stream_example.go b/example/restart_stream_example.go index d31d670..0542dfa 100644 --- a/example/restart_stream_example.go +++ b/example/restart_stream_example.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" twitterstream "github.com/fallenstedt/twitter-stream" + "github.com/fallenstedt/twitter-stream/stream" "time" ) @@ -80,7 +81,7 @@ func initiateStream() { fmt.Println("Stopped Stream") } -func fetchTweets() twitterstream.IStream { +func fetchTweets() stream.IStream { tok, err := getTwitterToken() if err != nil { panic(err) @@ -107,6 +108,6 @@ func getTwitterToken() (string, error) { return tok.AccessToken, err } -func getTwitterStreamApi(tok string) twitterstream.IStream { +func getTwitterStreamApi(tok string) stream.IStream { return twitterstream.NewTwitterStream(tok).Stream } diff --git a/stream.go b/stream/stream.go similarity index 97% rename from stream.go rename to stream/stream.go index 6b93649..0f0d2c7 100644 --- a/stream.go +++ b/stream/stream.go @@ -1,4 +1,4 @@ -package twitterstream +package stream import ( "github.com/fallenstedt/twitter-stream/httpclient" @@ -36,7 +36,7 @@ type ( } ) -func newStream(httpClient httpclient.IHttpClient, reader IStreamResponseBodyReader) IStream { +func NewStream(httpClient httpclient.IHttpClient, reader IStreamResponseBodyReader) IStream { return &Stream{ unmarshalHook: func(bytes []byte) (interface{}, error) { return bytes, nil diff --git a/stream_test.go b/stream/stream_test.go similarity index 93% rename from stream_test.go rename to stream/stream_test.go index 0d91f26..76a18c5 100644 --- a/stream_test.go +++ b/stream/stream_test.go @@ -1,4 +1,4 @@ -package twitterstream +package stream import ( "bytes" @@ -12,8 +12,8 @@ import ( func TestGetMessages(t *testing.T) { client := httpclient.NewHttpClientMock("foobar") - reader := newStreamResponseBodyReader() - instance := newStream(client, reader) + reader := NewStreamResponseBodyReader() + instance := NewStream(client, reader) messages := instance.GetMessages() @@ -24,7 +24,7 @@ func TestGetMessages(t *testing.T) { func TestStopStream(t *testing.T) { client := httpclient.NewHttpClientMock("foobar") - reader := newStreamResponseBodyReader() + reader := NewStreamResponseBodyReader() instance := &Stream{ unmarshalHook: func(bytes []byte) (interface{}, error) { return bytes, nil @@ -79,7 +79,7 @@ func TestStartStream(t *testing.T) { testName := fmt.Sprintf("TestStartStream (%d)", i) t.Run(testName, func(t *testing.T) { - instance := newStream( + instance := NewStream( tt.givenMockHttpRequestToStreamReturns(), tt.givenMockStreamResponseBodyReader(), ) diff --git a/stream_utils.go b/stream/stream_utils.go similarity index 95% rename from stream_utils.go rename to stream/stream_utils.go index d2e1aea..dab1c7c 100644 --- a/stream_utils.go +++ b/stream/stream_utils.go @@ -1,4 +1,4 @@ -package twitterstream +package stream // // ❤️ Credit goes to the team at dghubble/go-twitter who originally built this code. @@ -36,9 +36,9 @@ type ( } ) -// newStreamResponseBodyReader returns an instance of streamResponseBodyReader +// NewStreamResponseBodyReader returns an instance of streamResponseBodyReader // for the given Twitter stream response body. -func newStreamResponseBodyReader() IStreamResponseBodyReader { +func NewStreamResponseBodyReader() IStreamResponseBodyReader { return &streamResponseBodyReader{} } diff --git a/stream_utils_mock.go b/stream/stream_utils_mock.go similarity index 94% rename from stream_utils_mock.go rename to stream/stream_utils_mock.go index 281d9b0..e487c03 100644 --- a/stream_utils_mock.go +++ b/stream/stream_utils_mock.go @@ -1,4 +1,4 @@ -package twitterstream +package stream import ( "io" diff --git a/twitterstream.go b/twitterstream.go index 02d5000..9c1ccda 100644 --- a/twitterstream.go +++ b/twitterstream.go @@ -4,11 +4,12 @@ package twitterstream import ( "github.com/fallenstedt/twitter-stream/httpclient" "github.com/fallenstedt/twitter-stream/rules" + "github.com/fallenstedt/twitter-stream/stream" ) type TwitterApi struct { Rules rules.IRules - Stream IStream + Stream stream.IStream } @@ -24,6 +25,6 @@ func NewTokenGenerator() ITokenGenerator { func NewTwitterStream(token string) *TwitterApi { client := httpclient.NewHttpClient(token) rules := rules.NewRules(client) - stream := newStream(client, newStreamResponseBodyReader()) + stream := stream.NewStream(client, stream.NewStreamResponseBodyReader()) return &TwitterApi{Rules: rules, Stream: stream} } From 3f626051920a0f81ee506f30739cc2b7bffae0f7 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 4 Dec 2021 11:52:56 -0800 Subject: [PATCH 03/12] move token generator --- token_generator.go => token_generator/token_generator.go | 4 ++-- .../token_generator_test.go | 4 ++-- twitterstream.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename token_generator.go => token_generator/token_generator.go (95%) rename token_generator_test.go => token_generator/token_generator_test.go (96%) diff --git a/token_generator.go b/token_generator/token_generator.go similarity index 95% rename from token_generator.go rename to token_generator/token_generator.go index e2d2fbc..8c29555 100644 --- a/token_generator.go +++ b/token_generator/token_generator.go @@ -1,4 +1,4 @@ -package twitterstream +package token_generator import ( "encoding/base64" @@ -23,7 +23,7 @@ type ( } ) -func newTokenGenerator(httpClient httpclient.IHttpClient) ITokenGenerator { +func NewTokenGenerator(httpClient httpclient.IHttpClient) ITokenGenerator { return &TokenGenerator{httpClient: httpClient} } diff --git a/token_generator_test.go b/token_generator/token_generator_test.go similarity index 96% rename from token_generator_test.go rename to token_generator/token_generator_test.go index 08d9238..5457d1b 100644 --- a/token_generator_test.go +++ b/token_generator/token_generator_test.go @@ -1,4 +1,4 @@ -package twitterstream +package token_generator import ( "bytes" @@ -68,7 +68,7 @@ func TestRequestBearerToken(t *testing.T) { mockClient := httpclient.NewHttpClientMock("") mockClient.MockNewHttpRequest = tt.mockRequest - instance := newTokenGenerator(mockClient) + instance := NewTokenGenerator(mockClient) instance.SetApiKeyAndSecret("SomeKey", "SomeSecret") data, err := instance.RequestBearerToken() diff --git a/twitterstream.go b/twitterstream.go index 9c1ccda..eba582d 100644 --- a/twitterstream.go +++ b/twitterstream.go @@ -5,6 +5,7 @@ import ( "github.com/fallenstedt/twitter-stream/httpclient" "github.com/fallenstedt/twitter-stream/rules" "github.com/fallenstedt/twitter-stream/stream" + "github.com/fallenstedt/twitter-stream/token_generator" ) type TwitterApi struct { @@ -12,11 +13,10 @@ type TwitterApi struct { Stream stream.IStream } - // NewTokenGenerator creates a TokenGenerator which can request a Bearer token using a twitter api key and secret. -func NewTokenGenerator() ITokenGenerator { +func NewTokenGenerator() token_generator.ITokenGenerator { client := httpclient.NewHttpClient("") - tokenGenerator := newTokenGenerator(client) + tokenGenerator := token_generator.NewTokenGenerator(client) return tokenGenerator } From baed75c20e8ba9eb139d424a473a0f682ad4680a Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 4 Dec 2021 11:54:09 -0800 Subject: [PATCH 04/12] 0.4.0 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 87a0871..60a2d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.3 \ No newline at end of file +0.4.0 \ No newline at end of file From 030f3d941d33e5c64b2cbfc0f368a1b552401845 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 4 Dec 2021 13:11:23 -0800 Subject: [PATCH 05/12] start on rule builder proof of concept --- rules/rule_builder.go | 63 +++++++++++++++++++++++++++++++++++++++++++ rules/rules.go | 54 ++++++++++++++++++++++--------------- rules/rules_test.go | 20 +++++++------- 3 files changed, 106 insertions(+), 31 deletions(-) create mode 100644 rules/rule_builder.go diff --git a/rules/rule_builder.go b/rules/rule_builder.go new file mode 100644 index 0000000..d35b7e5 --- /dev/null +++ b/rules/rule_builder.go @@ -0,0 +1,63 @@ +package rules + +import "fmt" + +type IRuleBuilder interface { + Build() string + WithValueAndTag(value string, tag string) string + AddKeyword(keyword string) *RuleBuilder + FilterKeyword(keyword string) *RuleBuilder + AddExactPhrase(phrase string) *RuleBuilder + FilterExactPhrase(phrase string) *RuleBuilder + +} + +type RuleBuilder struct { + keywords []string + filterkeywords []string + exactphrase []string + filterexactphrase []string +} + +func NewRuleBuilder() IRuleBuilder { + return &RuleBuilder{ + keywords: []string{}, + filterkeywords: []string{}, + exactphrase: []string{}, + filterexactphrase: []string{}, + } +} + +func (r *RuleBuilder) Build() string { + return "lol" +} + +func (r *RuleBuilder) WithValueAndTag(value string, tag string) string { + return fmt.Sprintf("%v %v", value, tag) +} + +func (r *RuleBuilder) AddKeyword(keyword string) *RuleBuilder { + r.keywords = append(r.keywords, keyword) + return r +} + +func (r *RuleBuilder) FilterKeyword(keyword string) *RuleBuilder { + r.filterkeywords = append(r.filterkeywords, negate(keyword)) + return r +} + +func (r *RuleBuilder) AddExactPhrase(phrase string) *RuleBuilder { + r.exactphrase = append(r.exactphrase, phrase) + return r +} + +func (r *RuleBuilder) FilterExactPhrase(phrase string) *RuleBuilder { + r.exactphrase = append(r.exactphrase, negate(phrase)) + return r +} + +func negate(s string) string { + ns := fmt.Sprintf("-%v", s) + return ns +} + diff --git a/rules/rules.go b/rules/rules.go index b563cf4..4470207 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -8,42 +8,54 @@ import ( type ( //IRules is the interface the rules struct implements. IRules interface { - AddRules(body string, dryRun bool) (*rulesResponse, error) - GetRules() (*rulesResponse, error) + AddRules(body string, dryRun bool) (*TwitterRuleResponse, error) + GetRules() (*TwitterRuleResponse, error) } - rules struct { - httpClient httpclient.IHttpClient - } + //AddRulesRequest - rulesResponse struct { - Data []rulesResponseValue - Meta rulesResponseMeta - Errors []rulesResponseError + //TwitterRuleResponse is what is returned from twitter when adding or deleting a rule. + TwitterRuleResponse struct { + Data []DataRule + Meta MetaRule + Errors []ErrorRule } - rulesResponseValue struct { + //DataRule is what is returned as "Data" when adding or deleting a rule. + DataRule struct { Value string `json:"value"` Tag string `json:"tag"` Id string `json:"id"` } - rulesResponseMeta struct { - Sent string `json:"sent"` - Summary addRulesResponseMetaSummary `json:"summary"` + + //MetaRule is what is returned as "Meta" when adding or deleting a rule. + MetaRule struct { + Sent string `json:"sent"` + Summary MetaSummary `json:"summary"` + } + + //MetaSummary is what is returned as "Summary" in "Meta" when adding or deleting a rule. + MetaSummary struct { + Created uint `json:"created"` + NotCreated uint `json:"not_created"` } - rulesResponseError struct { + + //ErrorRule is what is returend as "Errors" when adding or deleting a rule. + ErrorRule struct { Value string `json:"value"` Id string `json:"id"` Title string `json:"title"` Type string `json:"type"` } - addRulesResponseMetaSummary struct { - Created uint `json:"created"` - NotCreated uint `json:"not_created"` + rules struct { + httpClient httpclient.IHttpClient } + ) +//NewRules creates a "rules" instance. This is used to create Twitter Filtered Stream rules. +// https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/build-a-rule. func NewRules(httpClient httpclient.IHttpClient) IRules { return &rules{httpClient: httpClient} } @@ -51,7 +63,7 @@ func NewRules(httpClient httpclient.IHttpClient) IRules { // AddRules adds or deletes rules to the stream using twitter's POST /2/tweets/search/stream/rules endpoint. // The body is a stringified object. // Learn about the possible error messages returned here https://developer.twitter.com/en/support/twitter-api/error-troubleshooting. -func (t *rules) AddRules(body string, dryRun bool) (*rulesResponse, error) { +func (t *rules) AddRules(body string, dryRun bool) (*TwitterRuleResponse, error) { res, err := t.httpClient.AddRules(func() string { if dryRun { return "?dry_run=true" @@ -65,7 +77,7 @@ func (t *rules) AddRules(body string, dryRun bool) (*rulesResponse, error) { } defer res.Body.Close() - data := new(rulesResponse) + data := new(TwitterRuleResponse) err = json.NewDecoder(res.Body).Decode(data) if err != nil { @@ -75,7 +87,7 @@ func (t *rules) AddRules(body string, dryRun bool) (*rulesResponse, error) { } // GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. -func (t *rules) GetRules() (*rulesResponse, error) { +func (t *rules) GetRules() (*TwitterRuleResponse, error) { res, err := t.httpClient.GetRules() if err != nil { @@ -83,7 +95,7 @@ func (t *rules) GetRules() (*rulesResponse, error) { } defer res.Body.Close() - data := new(rulesResponse) + data := new(TwitterRuleResponse) json.NewDecoder(res.Body).Decode(data) return data, nil diff --git a/rules/rules_test.go b/rules/rules_test.go index 5d8ece3..47662f3 100644 --- a/rules/rules_test.go +++ b/rules/rules_test.go @@ -14,7 +14,7 @@ func TestAddRules(t *testing.T) { var tests = []struct { body string mockRequest func(queryParams string, body string) (*http.Response, error) - result *rulesResponse + result *TwitterRuleResponse }{ { `{ @@ -46,17 +46,17 @@ func TestAddRules(t *testing.T) { }, nil }, - &rulesResponse{ - Data: []rulesResponseValue{ + &TwitterRuleResponse{ + Data: []DataRule{ { Value: "cat has:images", Tag: "cat tweets with images", Id: "123456", }, }, - Meta: rulesResponseMeta{ + Meta: MetaRule{ Sent: "today", - Summary: addRulesResponseMetaSummary{ + Summary: MetaSummary{ Created: 1, NotCreated: 0, }, @@ -110,7 +110,7 @@ func TestAddRules(t *testing.T) { func TestGetRules(t *testing.T) { var tests = []struct { mockRequest func() (*http.Response, error) - result *rulesResponse + result *TwitterRuleResponse }{ { func() (*http.Response, error) { @@ -136,17 +136,17 @@ func TestGetRules(t *testing.T) { }, nil }, - &rulesResponse{ - Data: []rulesResponseValue{ + &TwitterRuleResponse{ + Data: []DataRule{ { Value: "cat has:images", Tag: "cat tweets with images", Id: "123456", }, }, - Meta: rulesResponseMeta{ + Meta: MetaRule{ Sent: "today", - Summary: addRulesResponseMetaSummary{ + Summary: MetaSummary{ Created: 0, NotCreated: 1, }, From c3a7226c42282e2582f5c0431bff9a092aeb18db Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sun, 5 Dec 2021 12:51:39 -0800 Subject: [PATCH 06/12] create rule builder --- example/create_rules_example.go | 16 ++++--- example/restart_stream_example.go | 2 +- rules/rule_builder.go | 74 +++++++++++++------------------ rules/rules.go | 39 ++++++++++++++-- rules/rules_test.go | 10 ++--- twitterstream.go | 4 ++ 6 files changed, 84 insertions(+), 61 deletions(-) diff --git a/example/create_rules_example.go b/example/create_rules_example.go index 532e383..5fa9848 100644 --- a/example/create_rules_example.go +++ b/example/create_rules_example.go @@ -10,21 +10,23 @@ const secret = "YOUR_SECRET" func main() { addRules() - getRules() - deleteRules() + //getRules() + //deleteRules() } func addRules() { + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() if err != nil { panic(err) } api := twitterstream.NewTwitterStream(tok.AccessToken) - res, err := api.Rules.AddRules(`{ - "add": [ - {"value": "cat has:images", "tag": "cat tweets with images"} - ] - }`, false) // dryRun is set to false. + rules := twitterstream.NewRuleBuilder(). + AddRule("puppies has:images", "puppy tweets with images"). + AddRule("lang:en -is:retweet -is:quote (#golangjobs OR #gojobs)", "golang jobs"). + Build() + + res, err := api.Rules.Create(rules, true) // dryRun is set to false. if err != nil { panic(err) diff --git a/example/restart_stream_example.go b/example/restart_stream_example.go index 0542dfa..0d7233f 100644 --- a/example/restart_stream_example.go +++ b/example/restart_stream_example.go @@ -40,7 +40,7 @@ type StreamDataExample struct { } `json:"matching_rules"` } -func main() { +func main2() { // This will run forever initiateStream() } diff --git a/rules/rule_builder.go b/rules/rule_builder.go index d35b7e5..f9ed729 100644 --- a/rules/rule_builder.go +++ b/rules/rule_builder.go @@ -1,63 +1,49 @@ package rules -import "fmt" - -type IRuleBuilder interface { - Build() string - WithValueAndTag(value string, tag string) string - AddKeyword(keyword string) *RuleBuilder - FilterKeyword(keyword string) *RuleBuilder - AddExactPhrase(phrase string) *RuleBuilder - FilterExactPhrase(phrase string) *RuleBuilder +type ( + IRuleBuilder interface { + AddRule(value string, tag string) *RuleBuilder + Build() []*RuleValue + } -} + RuleValue struct { + Value *string `json:"value,omitempty"` + Tag *string `json:"tag,omitempty"` + } -type RuleBuilder struct { - keywords []string - filterkeywords []string - exactphrase []string - filterexactphrase []string -} + RuleBuilder struct { + rules []*RuleValue + } +) -func NewRuleBuilder() IRuleBuilder { +func NewRuleBuilder() *RuleBuilder { return &RuleBuilder{ - keywords: []string{}, - filterkeywords: []string{}, - exactphrase: []string{}, - filterexactphrase: []string{}, + rules: []*RuleValue{}, } } -func (r *RuleBuilder) Build() string { - return "lol" -} - -func (r *RuleBuilder) WithValueAndTag(value string, tag string) string { - return fmt.Sprintf("%v %v", value, tag) -} - -func (r *RuleBuilder) AddKeyword(keyword string) *RuleBuilder { - r.keywords = append(r.keywords, keyword) +//AddRule will create a rule to be build for filtered-stream. +//Read more about rule limitations here https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/introduction. +func (r *RuleBuilder) AddRule(value string, tag string) *RuleBuilder { + rule := newRuleValue().setValueTag(value, tag) + r.rules = append(r.rules, rule) return r } -func (r *RuleBuilder) FilterKeyword(keyword string) *RuleBuilder { - r.filterkeywords = append(r.filterkeywords, negate(keyword)) - return r +func (r *RuleBuilder) Build() []*RuleValue { + return r.rules } -func (r *RuleBuilder) AddExactPhrase(phrase string) *RuleBuilder { - r.exactphrase = append(r.exactphrase, phrase) - return r +func newRuleValue() *RuleValue { + return &RuleValue{ + Value: nil, + Tag: nil, + } } -func (r *RuleBuilder) FilterExactPhrase(phrase string) *RuleBuilder { - r.exactphrase = append(r.exactphrase, negate(phrase)) +func (r *RuleValue) setValueTag(value string, tag string) *RuleValue { + r.Value = &value + r.Tag = &tag return r } -func negate(s string) string { - ns := fmt.Sprintf("-%v", s) - return ns -} - diff --git a/rules/rules.go b/rules/rules.go index 4470207..8ee97ae 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -9,6 +9,7 @@ type ( //IRules is the interface the rules struct implements. IRules interface { AddRules(body string, dryRun bool) (*TwitterRuleResponse, error) + Create(rules []*RuleValue, dryRun bool) (*TwitterRuleResponse, error) GetRules() (*TwitterRuleResponse, error) } @@ -23,8 +24,8 @@ type ( //DataRule is what is returned as "Data" when adding or deleting a rule. DataRule struct { - Value string `json:"value"` - Tag string `json:"tag"` + Value string `json:"Value"` + Tag string `json:"Tag"` Id string `json:"id"` } @@ -40,14 +41,17 @@ type ( NotCreated uint `json:"not_created"` } - //ErrorRule is what is returend as "Errors" when adding or deleting a rule. + //ErrorRule is what is returned as "Errors" when adding or deleting a rule. ErrorRule struct { - Value string `json:"value"` + Value string `json:"Value"` Id string `json:"id"` Title string `json:"title"` Type string `json:"type"` } + addRulesRequest struct { + Add []*RuleValue `json:"add"` + } rules struct { httpClient httpclient.IHttpClient } @@ -60,6 +64,33 @@ func NewRules(httpClient httpclient.IHttpClient) IRules { return &rules{httpClient: httpClient} } +func (t *rules) Create(rules []*RuleValue, dryRun bool) (*TwitterRuleResponse, error) { + add := addRulesRequest{Add: rules} + body, err := json.Marshal(add) + if err != nil { + return nil, err + } + + res, err := t.httpClient.AddRules(func() string { + if dryRun { + return "?dry_run=true" + } else { + return "" + } + }(), string(body)) + + if err != nil { + return nil, err + } + + defer res.Body.Close() + data := new(TwitterRuleResponse) + + err = json.NewDecoder(res.Body).Decode(data) + return data, err +} + +// Deprecated: Use Create instead. // AddRules adds or deletes rules to the stream using twitter's POST /2/tweets/search/stream/rules endpoint. // The body is a stringified object. // Learn about the possible error messages returned here https://developer.twitter.com/en/support/twitter-api/error-troubleshooting. diff --git a/rules/rules_test.go b/rules/rules_test.go index 47662f3..2ce3bdf 100644 --- a/rules/rules_test.go +++ b/rules/rules_test.go @@ -19,14 +19,14 @@ func TestAddRules(t *testing.T) { { `{ "add": [ - {"value": "cat has:images", "tag": "cat tweets with images"} + {"Value": "cat has:images", "Tag": "cat tweets with images"} ] }`, func(queryParams string, bodyRequest string) (*http.Response, error) { json := `{ "data": [{ - "value": "cat has:images", - "tag":"cat tweets with images", + "Value": "cat has:images", + "Tag":"cat tweets with images", "id": "123456" }], "meta": { @@ -116,8 +116,8 @@ func TestGetRules(t *testing.T) { func() (*http.Response, error) { json := `{ "data": [{ - "value": "cat has:images", - "tag":"cat tweets with images", + "Value": "cat has:images", + "Tag":"cat tweets with images", "id": "123456" }], "meta": { diff --git a/twitterstream.go b/twitterstream.go index eba582d..0b018d7 100644 --- a/twitterstream.go +++ b/twitterstream.go @@ -20,6 +20,10 @@ func NewTokenGenerator() token_generator.ITokenGenerator { return tokenGenerator } +func NewRuleBuilder() rules.IRuleBuilder { + return rules.NewRuleBuilder() +} + // NewTwitterStream consumes a twitter Bearer token. // It is used to interact with Twitter's v2 filtered streaming API func NewTwitterStream(token string) *TwitterApi { From 14cfc33071f032c6e56c09ca17e6ba68caa1d0f2 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Mon, 6 Dec 2021 19:39:56 -0800 Subject: [PATCH 07/12] build create rules request --- example/create_rules_example.go | 78 +++++++++++++++++++-------------- rules/rule_builder.go | 12 +++-- rules/rules.go | 36 +++++---------- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/example/create_rules_example.go b/example/create_rules_example.go index 5fa9848..8ae8344 100644 --- a/example/create_rules_example.go +++ b/example/create_rules_example.go @@ -3,14 +3,15 @@ package main import ( "fmt" twitterstream "github.com/fallenstedt/twitter-stream" + "github.com/fallenstedt/twitter-stream/rules" ) -const key = "YOUR_KEY" -const secret = "YOUR_SECRET" +const key = "KEY" +const secret = "SECRET" func main() { - addRules() - //getRules() + //addRules() + getRules() //deleteRules() } @@ -26,7 +27,7 @@ func addRules() { AddRule("lang:en -is:retweet -is:quote (#golangjobs OR #gojobs)", "golang jobs"). Build() - res, err := api.Rules.Create(rules, true) // dryRun is set to false. + res, err := api.Rules.Create(rules, false) // dryRun is set to false. if err != nil { panic(err) @@ -37,8 +38,8 @@ func addRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I have created this many rules: ") - fmt.Println(res.Meta.Summary.Created) + fmt.Println("I have created these rules: ") + printRules(res.Data) } func getRules() { @@ -58,31 +59,40 @@ func getRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println(res.Data) -} - -func deleteRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() - if err != nil { - panic(err) - } - api := twitterstream.NewTwitterStream(tok.AccessToken) - - // use api.Rules.GetRules to find the ID number for an existing rule - res, err := api.Rules.AddRules(`{ - "delete": { - "ids": ["1234567890"] - } - }`, false) - - if err != nil { - panic(err) + fmt.Println("I found these rules: ") + printRules(res.Data)} + +//func deleteRules() { +// tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() +// if err != nil { +// panic(err) +// } +// api := twitterstream.NewTwitterStream(tok.AccessToken) +// +// // use api.Rules.GetRules to find the ID number for an existing rule +// res, err := api.Rules.AddRules(`{ +// "delete": { +// "ids": ["1234567890"] +// } +// }`, false) +// +// if err != nil { +// panic(err) +// } +// +// if res.Errors != nil && len(res.Errors) > 0 { +// //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting +// panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) +// } +// +// fmt.Println(res) +//} + + +func printRules(data []rules.DataRule) { + for _, datum := range data { + fmt.Printf("Id: %v\n", datum.Id) + fmt.Printf("Tag: %v\n",datum.Tag) + fmt.Printf("Value: %v\n\n", datum.Value) } - - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } - - fmt.Println(res) -} +} \ No newline at end of file diff --git a/rules/rule_builder.go b/rules/rule_builder.go index f9ed729..95de5d2 100644 --- a/rules/rule_builder.go +++ b/rules/rule_builder.go @@ -3,7 +3,7 @@ package rules type ( IRuleBuilder interface { AddRule(value string, tag string) *RuleBuilder - Build() []*RuleValue + Build() CreateRulesRequest } RuleValue struct { @@ -14,6 +14,11 @@ type ( RuleBuilder struct { rules []*RuleValue } + + CreateRulesRequest struct { + Add []*RuleValue `json:"add"` + } + ) func NewRuleBuilder() *RuleBuilder { @@ -30,8 +35,9 @@ func (r *RuleBuilder) AddRule(value string, tag string) *RuleBuilder { return r } -func (r *RuleBuilder) Build() []*RuleValue { - return r.rules +func (r *RuleBuilder) Build() CreateRulesRequest { + add := CreateRulesRequest{Add: r.rules} + return add } func newRuleValue() *RuleValue { diff --git a/rules/rules.go b/rules/rules.go index 8ee97ae..94ca518 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -8,8 +8,7 @@ import ( type ( //IRules is the interface the rules struct implements. IRules interface { - AddRules(body string, dryRun bool) (*TwitterRuleResponse, error) - Create(rules []*RuleValue, dryRun bool) (*TwitterRuleResponse, error) + Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleResponse, error) GetRules() (*TwitterRuleResponse, error) } @@ -49,9 +48,6 @@ type ( Type string `json:"type"` } - addRulesRequest struct { - Add []*RuleValue `json:"add"` - } rules struct { httpClient httpclient.IHttpClient } @@ -64,9 +60,8 @@ func NewRules(httpClient httpclient.IHttpClient) IRules { return &rules{httpClient: httpClient} } -func (t *rules) Create(rules []*RuleValue, dryRun bool) (*TwitterRuleResponse, error) { - add := addRulesRequest{Add: rules} - body, err := json.Marshal(add) +func (t *rules) Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleResponse, error) { + body, err := json.Marshal(rules) if err != nil { return nil, err } @@ -90,18 +85,10 @@ func (t *rules) Create(rules []*RuleValue, dryRun bool) (*TwitterRuleResponse, e return data, err } -// Deprecated: Use Create instead. -// AddRules adds or deletes rules to the stream using twitter's POST /2/tweets/search/stream/rules endpoint. -// The body is a stringified object. -// Learn about the possible error messages returned here https://developer.twitter.com/en/support/twitter-api/error-troubleshooting. -func (t *rules) AddRules(body string, dryRun bool) (*TwitterRuleResponse, error) { - res, err := t.httpClient.AddRules(func() string { - if dryRun { - return "?dry_run=true" - } else { - return "" - } - }(), body) + +// GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. +func (t *rules) GetRules() (*TwitterRuleResponse, error) { + res, err := t.httpClient.GetRules() if err != nil { return nil, err @@ -109,16 +96,12 @@ func (t *rules) AddRules(body string, dryRun bool) (*TwitterRuleResponse, error) defer res.Body.Close() data := new(TwitterRuleResponse) + json.NewDecoder(res.Body).Decode(data) - err = json.NewDecoder(res.Body).Decode(data) - if err != nil { - return nil, err - } return data, nil } -// GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. -func (t *rules) GetRules() (*TwitterRuleResponse, error) { +func (t *rules) GetRulesI() (*TwitterRuleResponse, error) { res, err := t.httpClient.GetRules() if err != nil { @@ -131,3 +114,4 @@ func (t *rules) GetRules() (*TwitterRuleResponse, error) { return data, nil } + From e8fdfba4bef988ca65b15e16058ad391aa0f3835 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Tue, 7 Dec 2021 19:49:43 -0800 Subject: [PATCH 08/12] create delete rules --- example/create_rules_example.go | 66 +++++++++++++++++---------------- rules/rule_builder.go | 12 ++++++ rules/rules.go | 23 +++++++++--- 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/example/create_rules_example.go b/example/create_rules_example.go index 8ae8344..e4dc0ad 100644 --- a/example/create_rules_example.go +++ b/example/create_rules_example.go @@ -10,8 +10,8 @@ const key = "KEY" const secret = "SECRET" func main() { - //addRules() - getRules() + addRules() + //getRules() //deleteRules() } @@ -38,8 +38,7 @@ func addRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I have created these rules: ") - printRules(res.Data) + fmt.Println("I have deleted rules.") } func getRules() { @@ -59,34 +58,37 @@ func getRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I found these rules: ") - printRules(res.Data)} - -//func deleteRules() { -// tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() -// if err != nil { -// panic(err) -// } -// api := twitterstream.NewTwitterStream(tok.AccessToken) -// -// // use api.Rules.GetRules to find the ID number for an existing rule -// res, err := api.Rules.AddRules(`{ -// "delete": { -// "ids": ["1234567890"] -// } -// }`, false) -// -// if err != nil { -// panic(err) -// } -// -// if res.Errors != nil && len(res.Errors) > 0 { -// //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting -// panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) -// } -// -// fmt.Println(res) -//} + if len(res.Data) > 0 { + fmt.Println("I found these rules: ") + printRules(res.Data) + } else { + fmt.Println("I found no rules") + } + +} + +func deleteRules() { + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + if err != nil { + panic(err) + } + api := twitterstream.NewTwitterStream(tok.AccessToken) + + // use api.Rules.GetRules to find the ID number for an existing rule + res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1468427075727945728, 1468427075727945729), false) + + if err != nil { + panic(err) + } + + if res.Errors != nil && len(res.Errors) > 0 { + //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting + panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) + } + + fmt.Println("I have deleted these rules: ") + printRules(res.Data) +} func printRules(data []rules.DataRule) { diff --git a/rules/rule_builder.go b/rules/rule_builder.go index 95de5d2..a21e89c 100644 --- a/rules/rule_builder.go +++ b/rules/rule_builder.go @@ -19,8 +19,20 @@ type ( Add []*RuleValue `json:"add"` } + DeleteRulesRequest struct { + Delete struct { + Ids []int `json:"ids"` + } `json:"delete"` + } + ) +func NewDeleteRulesRequest(ids ...int) DeleteRulesRequest { + return DeleteRulesRequest{Delete: struct { + Ids []int `json:"ids"` + }(struct{ Ids []int }{Ids: ids})} +} + func NewRuleBuilder() *RuleBuilder { return &RuleBuilder{ rules: []*RuleValue{}, diff --git a/rules/rules.go b/rules/rules.go index 94ca518..44044ff 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -9,6 +9,7 @@ type ( //IRules is the interface the rules struct implements. IRules interface { Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleResponse, error) + Delete(req DeleteRulesRequest, dryRun bool) (*TwitterRuleResponse, error) GetRules() (*TwitterRuleResponse, error) } @@ -85,23 +86,33 @@ func (t *rules) Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleRespo return data, err } +func (t *rules) Delete(req DeleteRulesRequest, dryRun bool) (*TwitterRuleResponse, error) { -// GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. -func (t *rules) GetRules() (*TwitterRuleResponse, error) { - res, err := t.httpClient.GetRules() + body, err := json.Marshal(req) if err != nil { return nil, err } + res, err := t.httpClient.AddRules(func() string { + if dryRun { + return "?dry_run=true" + } else { + return "" + } + }(), string(body)) + + defer res.Body.Close() data := new(TwitterRuleResponse) - json.NewDecoder(res.Body).Decode(data) - return data, nil + err = json.NewDecoder(res.Body).Decode(data) + return data, err } -func (t *rules) GetRulesI() (*TwitterRuleResponse, error) { + +// GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. +func (t *rules) GetRules() (*TwitterRuleResponse, error) { res, err := t.httpClient.GetRules() if err != nil { From 624706f71316632a4a20153034a1e060100bef3b Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Tue, 7 Dec 2021 20:21:49 -0800 Subject: [PATCH 09/12] update test --- example/create_rules_example.go | 4 +- rules/rules.go | 9 +-- rules/rules_test.go | 113 ++++++++++++++++++++++++++++---- 3 files changed, 109 insertions(+), 17 deletions(-) diff --git a/example/create_rules_example.go b/example/create_rules_example.go index e4dc0ad..0413509 100644 --- a/example/create_rules_example.go +++ b/example/create_rules_example.go @@ -47,7 +47,7 @@ func getRules() { panic(err) } api := twitterstream.NewTwitterStream(tok.AccessToken) - res, err := api.Rules.GetRules() + res, err := api.Rules.Get() if err != nil { panic(err) @@ -74,7 +74,7 @@ func deleteRules() { } api := twitterstream.NewTwitterStream(tok.AccessToken) - // use api.Rules.GetRules to find the ID number for an existing rule + // use api.Rules.Get to find the ID number for an existing rule res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1468427075727945728, 1468427075727945729), false) if err != nil { diff --git a/rules/rules.go b/rules/rules.go index 44044ff..dce53ea 100644 --- a/rules/rules.go +++ b/rules/rules.go @@ -10,7 +10,7 @@ type ( IRules interface { Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleResponse, error) Delete(req DeleteRulesRequest, dryRun bool) (*TwitterRuleResponse, error) - GetRules() (*TwitterRuleResponse, error) + Get() (*TwitterRuleResponse, error) } //AddRulesRequest @@ -61,6 +61,7 @@ func NewRules(httpClient httpclient.IHttpClient) IRules { return &rules{httpClient: httpClient} } +// Create will create new twitter streaming rules. func (t *rules) Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleResponse, error) { body, err := json.Marshal(rules) if err != nil { @@ -85,7 +86,7 @@ func (t *rules) Create(rules CreateRulesRequest, dryRun bool) (*TwitterRuleRespo err = json.NewDecoder(res.Body).Decode(data) return data, err } - +// Delete will delete rules twitter rules by their id. func (t *rules) Delete(req DeleteRulesRequest, dryRun bool) (*TwitterRuleResponse, error) { body, err := json.Marshal(req) @@ -111,8 +112,8 @@ func (t *rules) Delete(req DeleteRulesRequest, dryRun bool) (*TwitterRuleRespons } -// GetRules gets rules for a stream using twitter's GET GET /2/tweets/search/stream/rules endpoint. -func (t *rules) GetRules() (*TwitterRuleResponse, error) { +// Get will fetch the current rules. +func (t *rules) Get() (*TwitterRuleResponse, error) { res, err := t.httpClient.GetRules() if err != nil { diff --git a/rules/rules_test.go b/rules/rules_test.go index 2ce3bdf..6708783 100644 --- a/rules/rules_test.go +++ b/rules/rules_test.go @@ -9,19 +9,15 @@ import ( "testing" ) -func TestAddRules(t *testing.T) { +func TestCreate(t *testing.T) { var tests = []struct { - body string + body CreateRulesRequest mockRequest func(queryParams string, body string) (*http.Response, error) result *TwitterRuleResponse }{ { - `{ - "add": [ - {"Value": "cat has:images", "Tag": "cat tweets with images"} - ] - }`, + NewRuleBuilder().AddRule("cat has:images", "cat tweets with images").Build(), func(queryParams string, bodyRequest string) (*http.Response, error) { json := `{ "data": [{ @@ -67,14 +63,14 @@ func TestAddRules(t *testing.T) { } for i, tt := range tests { - testName := fmt.Sprintf("TestAddRules (%d) %s", i, tt.body) + testName := fmt.Sprintf("TestCreate (%d)", i) t.Run(testName, func(t *testing.T) { mockClient := httpclient.NewHttpClientMock("sometoken") mockClient.MockAddRules = tt.mockRequest instance := NewRules(mockClient) - result, err := instance.AddRules(tt.body, false) + result, err := instance.Create(tt.body, false) if err != nil { t.Errorf("got err %v", err) @@ -107,6 +103,102 @@ func TestAddRules(t *testing.T) { } } + +func TestDelete(t *testing.T) { + + var tests = []struct { + body DeleteRulesRequest + mockRequest func(queryParams string, body string) (*http.Response, error) + result *TwitterRuleResponse + }{ + { + NewDeleteRulesRequest(123), + func(queryParams string, bodyRequest string) (*http.Response, error) { + json := `{ + "data": [{ + "Value": "cat has:images", + "Tag":"cat tweets with images", + "id": "123" + }], + "meta": { + "sent": "today", + "summary": { + "created": 0, + "not_created": 1 + } + }, + "errors": [] + }` + + body := ioutil.NopCloser(bytes.NewReader([]byte(json))) + return &http.Response{ + StatusCode: http.StatusOK, + Body: body, + }, nil + + }, + &TwitterRuleResponse{ + Data: []DataRule{ + { + Value: "cat has:images", + Tag: "cat tweets with images", + Id: "123", + }, + }, + Meta: MetaRule{ + Sent: "today", + Summary: MetaSummary{ + Created: 0, + NotCreated: 1, + }, + }, + Errors: nil, + }, + }, + } + + for i, tt := range tests { + testName := fmt.Sprintf("TestCreate (%d)", i) + + t.Run(testName, func(t *testing.T) { + mockClient := httpclient.NewHttpClientMock("sometoken") + mockClient.MockAddRules = tt.mockRequest + + instance := NewRules(mockClient) + result, err := instance.Delete(tt.body, false) + + if err != nil { + t.Errorf("got err %v", err) + } + + if result.Data[0].Id != tt.result.Data[0].Id { + t.Errorf("got %s, want %s", result.Data[0].Id, tt.result.Data[0].Id) + } + + if result.Data[0].Tag != tt.result.Data[0].Tag { + t.Errorf("got %s, want %s", result.Data[0].Tag, tt.result.Data[0].Tag) + } + + if result.Data[0].Value != tt.result.Data[0].Value { + t.Errorf("got %s, want %s", result.Data[0].Value, tt.result.Data[0].Value) + } + + if result.Meta.Summary.Created != tt.result.Meta.Summary.Created { + t.Errorf("got %d, want %d", result.Meta.Summary.Created, tt.result.Meta.Summary.Created) + } + + if result.Meta.Summary.NotCreated != tt.result.Meta.Summary.NotCreated { + t.Errorf("got %d, want %d", result.Meta.Summary.NotCreated, tt.result.Meta.Summary.NotCreated) + } + + if result.Meta.Sent != tt.result.Meta.Sent { + t.Errorf("got %s, want %s", result.Meta.Sent, tt.result.Meta.Sent) + } + }) + } +} + + func TestGetRules(t *testing.T) { var tests = []struct { mockRequest func() (*http.Response, error) @@ -163,7 +255,7 @@ func TestGetRules(t *testing.T) { mockClient.MockGetRules = tt.mockRequest instance := NewRules(mockClient) - result, err := instance.GetRules() + result, err := instance.Get() if err != nil { t.Errorf("got err %v", err) @@ -194,5 +286,4 @@ func TestGetRules(t *testing.T) { } }) } - } From 156bac2b25f001011013f728f8c149342135d15d Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 11 Dec 2021 12:17:34 -0800 Subject: [PATCH 10/12] add rule builder tests --- rules/rule_builder_test.go | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 rules/rule_builder_test.go diff --git a/rules/rule_builder_test.go b/rules/rule_builder_test.go new file mode 100644 index 0000000..603aef3 --- /dev/null +++ b/rules/rule_builder_test.go @@ -0,0 +1,47 @@ +package rules + +import ( + "encoding/json" + "testing" +) + +func TestNewDeleteRulesRequestAcceptsMultipleIds(t *testing.T) { + result := NewDeleteRulesRequest(1, 2, 3, 4, 5) + expected := []int{1, 2, 3, 4, 5} + + if len(result.Delete.Ids) != len(expected) { + t.Errorf("Expected %v to have same length as %v", result, expected) + } + + for i := range result.Delete.Ids { + if result.Delete.Ids[i] != expected[i] { + t.Errorf("Expected %v to equal %v", result.Delete.Ids[i], expected[i]) + } + } +} + +func TestNewDeleteRulesRequestMarshalsWell(t *testing.T) { + result := NewDeleteRulesRequest(1, 2, 3, 4, 5) + body, err := json.Marshal(result) + + if err != nil { + t.Error(err) + } + + if string(body) != "{\"delete\":{\"ids\":[1,2,3,4,5]}}" { + t.Errorf("Expected %v to equal %v", string(body), "{\"delete\":{\"ids\":[1,2,3,4,5]}}") + } +} + +func TestNewRuleBuilderBuildsManyRules(t *testing.T) { + result := NewRuleBuilder().AddRule("cats", "cat tweets").AddRule("dogs", "dog tweets").Build() + body, err := json.Marshal(result) + + if err != nil { + t.Error(err) + } + + if string(body) != "{\"add\":[{\"value\":\"cats\",\"tag\":\"cat tweets\"},{\"value\":\"dogs\",\"tag\":\"dog tweets\"}]}" { + t.Errorf("Expected %v to equal %v", string(body), "{\"add\":[{\"value\":\"cats\",\"tag\":\"cat tweets\"},{\"value\":\"dogs\",\"tag\":\"dog tweets\"}]}") + } +} \ No newline at end of file From c5a7e1bc78f98180b658907c2701c4143f06f781 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 11 Dec 2021 13:15:27 -0800 Subject: [PATCH 11/12] update examples and readme --- README.md | 238 ++++++++++-------- example/create_rules_example.go | 26 +- example/main.go | 13 + ...rt_stream_example.go => stream_forever.go} | 9 +- 4 files changed, 150 insertions(+), 136 deletions(-) create mode 100644 example/main.go rename example/{restart_stream_example.go => stream_forever.go} (96%) diff --git a/README.md b/README.md index 8adf2f4..149ed3a 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,16 @@ See [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example ## Examples +See [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example), or follow the guide below. #### Starting a stream ##### Obtain an Access Token using your Twitter Access Key and Secret. -You need an access token to do any streaming. `twitterstream` provides an easy way to fetch an access token. +You need an access token to do any streaming. `twitterstream` provides an easy way to fetch an access token. Use your +access token and secret access token from twitter to request a bearer token. + ```go tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret("key", "secret").RequestBearerToken() - - if err != nil { - panic(err) - } ``` ##### Create a streaming api @@ -41,135 +40,152 @@ Create a twitterstream instance with your access token from above. api := twitterstream.NewTwitterStream(tok.AccessToken) ``` -##### Set your unmarshal hook -It is encouraged you set an unmarshal hook for thread-safety. Go's `bytes.Buffer` is not thread safe. Sharing a `bytes.Buffer` -across multiple goroutines introduces risk of panics when decoding json [source](https://github.com/Fallenstedt/twitter-stream/issues/13). -To avoid panics, it's encouraged to unmarshal json in the same goroutine where the `bytes.Buffer` exists. Use `SetUnmarshalHook` to set a function that unmarshals json. +##### Create rules -By default, twitterstream's unmarshal hook will return `[]byte` if you want to live dangerously. +We need to create [twitter streaming rules](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/build-a-rule) so we can get tweets that we want. +The filtered stream endpoints deliver filtered Tweets to you in real-time that match on a set of rules that are applied to the stream. Rules are made up of operators that are used to match on a variety of Tweet attributes. +Below we create three rules. One for puppy tweets with images, another for cat tweets with images, and the other of unique English golang job postings. Each rule is +associated with their own tag. ```go - api.Stream.SetUnmarshalHook(func(bytes []byte) (interface{}, error) { - // StreemData is a struct that represents your returned json - // This is a quick resource to generate a struct from your json - // https://mholt.github.io/json-to-go/ - data := StreamData{} - if err := json.Unmarshal(bytes, &data); err != nil { - log.Printf("Failed to unmarshal bytes: %v", err) - } - return data, err - }) -``` - +rules := twitterstream.NewRuleBuilder(). + AddRule("cat has:images", "cat tweets with images"). + AddRule("puppy has:images", "puppy tweets with images"). + AddRule("lang:en -is:retweet -is:quote (#golangjobs OR #gojobs)", "golang jobs"). + Build() +// Create will create twitter rules +// dryRun is set to false. Set to true to test out your request +res, err := api.Rules.Create(rules, false) -##### Start Stream -Start your stream. This is a long-running HTTP GET request. -You can get specific data you want by adding [query params](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference/get-tweets-search-stream). -Additionally, [view an example of query params here](https://developer.twitter.com/en/docs/twitter-api/expansions), or in the [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example) +// Get will get your current rules +res, err := api.Rules.Get() -```go - err := api.Stream.StartStream("?expansions=author_id&tweet.fields=created_at") +// Delete will delete your rules by their id +// dryRun is set to false. Set to true to test out your request +res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1468427075727945728, 1468427075727945729), false) - if err != nil { - panic(err) - } -``` -Consume Messages from the Stream -Handle any `io.EOF` and other errors that arise first, then unmarshal your bytes into your favorite struct. Below is an example with strings -```go - go func() { - for message := range api.Stream.GetMessages() { - if message.Err != nil { - panic(message.Err) - } - // Will print something like: - //{"data":{"id":"1356479201000","text":"Look at this cat picture"},"matching_rules":[{"id":12345,"tag":"cat tweets with images"}]} - fmt.Println(string(message.Data)) - } - }() - - time.Sleep(time.Second * 30) - api.Stream.StopStream() ``` -#### Creating, Deleting, and Getting Rules - -##### Obtain an Access Token using your Twitter Access Key and Secret. -You need an access token to do anything. `twitterstream` provides an easy way to fetch an access token. -```go - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret("key", "secret").RequestBearerToken() - - if err != nil { - panic(err) - } -``` -##### Create a streaming api -Create a twitterstream instance with your access token from above. +##### Set your unmarshal hook +It is encouraged you set an unmarshal hook for thread-safety. Go's `bytes.Buffer` is not thread safe. Sharing a `bytes.Buffer` +across multiple goroutines introduces risk of panics when decoding json. +To avoid panics, it's encouraged to unmarshal json in the same goroutine where the `bytes.Buffer` exists. Use `SetUnmarshalHook` to set a function that unmarshals json. -```go - api := twitterstream.NewTwitterStream(tok.AccessToken) -``` +By default, twitterstream's unmarshal hook will return `[]byte` if you want to live dangerously. -##### Get Rules -Use the `Rules` struct to access different Rules endpoints as defined in [Twitter's API Reference](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference) ```go - res, err := api.Rules.GetRules() - - if err != nil { - panic(err) - } - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } +type StreamDataExample struct { + Data struct { + Text string `json:"text"` + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + AuthorID string `json:"author_id"` + } `json:"data"` + Includes struct { + Users []struct { + ID string `json:"id"` + Name string `json:"name"` + Username string `json:"username"` + } `json:"users"` + } `json:"includes"` + MatchingRules []struct { + ID string `json:"id"` + Tag string `json:"tag"` + } `json:"matching_rules"` +} + +api.SetUnmarshalHook(func(bytes []byte) (interface{}, error) { + data := StreamDataExample{} + + if err := json.Unmarshal(bytes, &data); err != nil { + fmt.Printf("failed to unmarshal bytes: %v", err) + } + + return data, err +}) +``` - fmt.Println(res.Data) -``` +##### Start Stream +Start your stream. This is a long-running HTTP GET request. +You can get specific data you want by adding [query params](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference/get-tweets-search-stream). +Additionally, [view an example of query params here](https://developer.twitter.com/en/docs/twitter-api/expansions), or in the [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example) -##### Add Rules -```go - res, err := api.Rules.AddRules(`{ - "add": [ - {"value": "cat has:images", "tag": "cat tweets with images"} - ] - }`, true) // dryRun is set to true - - if err != nil { - panic(err) - } - - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } -``` -##### Delete Rules ```go -// use api.Rules.GetRules to find the ID number for an existing rule - res, err := api.Rules.AddRules(`{ - "delete": { - "ids": ["1234567890"] - } - }`, true) - - if err != nil { - panic(err) - } - - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } +// Steps from above, Placed into a single function +// This assumes you have at least one streaming rule configured. +// returns a configured instance of twitterstream +func fetchTweets() stream.IStream { + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() + + if err != nil { + panic(err) + } + + api := twitterstream.NewTwitterStream(tok).Stream + api.SetUnmarshalHook(func(bytes []byte) (interface{}, error) { + data := StreamDataExample{} + + if err := json.Unmarshal(bytes, &data); err != nil { + fmt.Printf("failed to unmarshal bytes: %v", err) + } + return data, err + }) + err = api.StartStream("?expansions=author_id&tweet.fields=created_at") + + if err != nil { + panic(err) + } + + return api +} + +// This will run forever +func initiateStream() { + fmt.Println("Starting Stream") + + // Start the stream + // And return the library's api + api := fetchTweets() + + // When the loop below ends, restart the stream + defer initiateStream() + + // Start processing data from twitter + for tweet := range api.GetMessages() { + + // Handle disconnections from twitter + // https://developer.twitter.com/en/docs/twitter-api/tweets/volume-streams/integrate/handling-disconnections + if tweet.Err != nil { + fmt.Printf("got error from twitter: %v", tweet.Err) + + // Notice we "StopStream" and then "continue" the loop instead of breaking. + // StopStream will close the long running GET request to Twitter's v2 Streaming endpoint by + // closing the `GetMessages` channel. Once it's closed, it's safe to perform a new network request + // with `StartStream` + api.StopStream() + continue + } + result := tweet.Data.(StreamDataExample) + + // Here I am printing out the text. + // You can send this off to a queue for processing. + // Or do your processing here in the loop + fmt.Println(result.Data.Text) + } + + fmt.Println("Stopped Stream") +} ``` ## Contributing -Pull requests are always welcome. Please accompany a pull request with tests. +Pull requests and feature requests are always welcome. +Please accompany a pull request with tests. diff --git a/example/create_rules_example.go b/example/create_rules_example.go index 0413509..cbc9299 100644 --- a/example/create_rules_example.go +++ b/example/create_rules_example.go @@ -6,24 +6,16 @@ import ( "github.com/fallenstedt/twitter-stream/rules" ) -const key = "KEY" -const secret = "SECRET" - -func main() { - addRules() - //getRules() - //deleteRules() -} - func addRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() if err != nil { panic(err) } api := twitterstream.NewTwitterStream(tok.AccessToken) rules := twitterstream.NewRuleBuilder(). - AddRule("puppies has:images", "puppy tweets with images"). + AddRule("cat has:images", "cat tweets with images"). + AddRule("puppy has:images", "puppy tweets with images"). AddRule("lang:en -is:retweet -is:quote (#golangjobs OR #gojobs)", "golang jobs"). Build() @@ -38,11 +30,12 @@ func addRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I have deleted rules.") + fmt.Println("I have created rules.") + printRules(res.Data) } func getRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() if err != nil { panic(err) } @@ -68,14 +61,14 @@ func getRules() { } func deleteRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() if err != nil { panic(err) } api := twitterstream.NewTwitterStream(tok.AccessToken) // use api.Rules.Get to find the ID number for an existing rule - res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1468427075727945728, 1468427075727945729), false) + res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1469776000158363653, 1469776000158363654), false) if err != nil { panic(err) @@ -86,8 +79,7 @@ func deleteRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I have deleted these rules: ") - printRules(res.Data) + fmt.Println("I have deleted rules ") } diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..0813310 --- /dev/null +++ b/example/main.go @@ -0,0 +1,13 @@ +package main + +const KEY = "KEY" +const SECRET = "SECRET" + +func main() { + // Run an example function + + addRules() + //getRules() + //initiateStream() + //deleteRules() +} diff --git a/example/restart_stream_example.go b/example/stream_forever.go similarity index 96% rename from example/restart_stream_example.go rename to example/stream_forever.go index 0d7233f..888932d 100644 --- a/example/restart_stream_example.go +++ b/example/stream_forever.go @@ -17,9 +17,6 @@ import ( // With connections to streaming endpoints, **it is likely, and should be expected,** that disconnections will take place and reconnection logic built. // ~https://developer.twitter.com/en/docs/twitter-api/tweets/volume-streams/integrate/handling-disconnections -const KEY = "YOUR_KEY" -const SECRET = "YOUR_SECRET" - type StreamDataExample struct { Data struct { Text string `json:"text"` @@ -40,11 +37,7 @@ type StreamDataExample struct { } `json:"matching_rules"` } -func main2() { - // This will run forever - initiateStream() -} - +// This will run forever func initiateStream() { fmt.Println("Starting Stream") From 08b887de1e83c5611046e6b9905f569b67805602 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 11 Dec 2021 13:25:45 -0800 Subject: [PATCH 12/12] update read me --- README.md | 7 ++++++- example/main.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 149ed3a..9df813a 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@ See [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example `go get github.com/fallenstedt/twitter-stream` +## Projects Using TwitterStream +Below are projects using this library! Add yours below with a pull request. + +* [FindTechJobs](https://www.findtechjobs.io/) - Search latest tech job postings from around the world + + ## Examples @@ -183,7 +189,6 @@ func initiateStream() { } ``` - ## Contributing Pull requests and feature requests are always welcome. diff --git a/example/main.go b/example/main.go index 0813310..0cfe9ea 100644 --- a/example/main.go +++ b/example/main.go @@ -7,7 +7,7 @@ func main() { // Run an example function addRules() - //getRules() - //initiateStream() + getRules() + initiateStream() //deleteRules() }