From c5a7e1bc78f98180b658907c2701c4143f06f781 Mon Sep 17 00:00:00 2001 From: Alex Fallenstedt <13971824+Fallenstedt@users.noreply.github.com> Date: Sat, 11 Dec 2021 13:15:27 -0800 Subject: [PATCH] update examples and readme --- README.md | 238 ++++++++++-------- example/create_rules_example.go | 26 +- example/main.go | 13 + ...rt_stream_example.go => stream_forever.go} | 9 +- 4 files changed, 150 insertions(+), 136 deletions(-) create mode 100644 example/main.go rename example/{restart_stream_example.go => stream_forever.go} (96%) diff --git a/README.md b/README.md index 8adf2f4..149ed3a 100644 --- a/README.md +++ b/README.md @@ -21,17 +21,16 @@ See [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example ## Examples +See [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example), or follow the guide below. #### Starting a stream ##### Obtain an Access Token using your Twitter Access Key and Secret. -You need an access token to do any streaming. `twitterstream` provides an easy way to fetch an access token. +You need an access token to do any streaming. `twitterstream` provides an easy way to fetch an access token. Use your +access token and secret access token from twitter to request a bearer token. + ```go tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret("key", "secret").RequestBearerToken() - - if err != nil { - panic(err) - } ``` ##### Create a streaming api @@ -41,135 +40,152 @@ Create a twitterstream instance with your access token from above. api := twitterstream.NewTwitterStream(tok.AccessToken) ``` -##### Set your unmarshal hook -It is encouraged you set an unmarshal hook for thread-safety. Go's `bytes.Buffer` is not thread safe. Sharing a `bytes.Buffer` -across multiple goroutines introduces risk of panics when decoding json [source](https://github.com/Fallenstedt/twitter-stream/issues/13). -To avoid panics, it's encouraged to unmarshal json in the same goroutine where the `bytes.Buffer` exists. Use `SetUnmarshalHook` to set a function that unmarshals json. +##### Create rules -By default, twitterstream's unmarshal hook will return `[]byte` if you want to live dangerously. +We need to create [twitter streaming rules](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/integrate/build-a-rule) so we can get tweets that we want. +The filtered stream endpoints deliver filtered Tweets to you in real-time that match on a set of rules that are applied to the stream. Rules are made up of operators that are used to match on a variety of Tweet attributes. +Below we create three rules. One for puppy tweets with images, another for cat tweets with images, and the other of unique English golang job postings. Each rule is +associated with their own tag. ```go - api.Stream.SetUnmarshalHook(func(bytes []byte) (interface{}, error) { - // StreemData is a struct that represents your returned json - // This is a quick resource to generate a struct from your json - // https://mholt.github.io/json-to-go/ - data := StreamData{} - if err := json.Unmarshal(bytes, &data); err != nil { - log.Printf("Failed to unmarshal bytes: %v", err) - } - return data, err - }) -``` - +rules := twitterstream.NewRuleBuilder(). + AddRule("cat has:images", "cat tweets with images"). + AddRule("puppy has:images", "puppy tweets with images"). + AddRule("lang:en -is:retweet -is:quote (#golangjobs OR #gojobs)", "golang jobs"). + Build() +// Create will create twitter rules +// dryRun is set to false. Set to true to test out your request +res, err := api.Rules.Create(rules, false) -##### Start Stream -Start your stream. This is a long-running HTTP GET request. -You can get specific data you want by adding [query params](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference/get-tweets-search-stream). -Additionally, [view an example of query params here](https://developer.twitter.com/en/docs/twitter-api/expansions), or in the [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example) +// Get will get your current rules +res, err := api.Rules.Get() -```go - err := api.Stream.StartStream("?expansions=author_id&tweet.fields=created_at") +// Delete will delete your rules by their id +// dryRun is set to false. Set to true to test out your request +res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1468427075727945728, 1468427075727945729), false) - if err != nil { - panic(err) - } -``` -Consume Messages from the Stream -Handle any `io.EOF` and other errors that arise first, then unmarshal your bytes into your favorite struct. Below is an example with strings -```go - go func() { - for message := range api.Stream.GetMessages() { - if message.Err != nil { - panic(message.Err) - } - // Will print something like: - //{"data":{"id":"1356479201000","text":"Look at this cat picture"},"matching_rules":[{"id":12345,"tag":"cat tweets with images"}]} - fmt.Println(string(message.Data)) - } - }() - - time.Sleep(time.Second * 30) - api.Stream.StopStream() ``` -#### Creating, Deleting, and Getting Rules - -##### Obtain an Access Token using your Twitter Access Key and Secret. -You need an access token to do anything. `twitterstream` provides an easy way to fetch an access token. -```go - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret("key", "secret").RequestBearerToken() - - if err != nil { - panic(err) - } -``` -##### Create a streaming api -Create a twitterstream instance with your access token from above. +##### Set your unmarshal hook +It is encouraged you set an unmarshal hook for thread-safety. Go's `bytes.Buffer` is not thread safe. Sharing a `bytes.Buffer` +across multiple goroutines introduces risk of panics when decoding json. +To avoid panics, it's encouraged to unmarshal json in the same goroutine where the `bytes.Buffer` exists. Use `SetUnmarshalHook` to set a function that unmarshals json. -```go - api := twitterstream.NewTwitterStream(tok.AccessToken) -``` +By default, twitterstream's unmarshal hook will return `[]byte` if you want to live dangerously. -##### Get Rules -Use the `Rules` struct to access different Rules endpoints as defined in [Twitter's API Reference](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference) ```go - res, err := api.Rules.GetRules() - - if err != nil { - panic(err) - } - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } +type StreamDataExample struct { + Data struct { + Text string `json:"text"` + ID string `json:"id"` + CreatedAt time.Time `json:"created_at"` + AuthorID string `json:"author_id"` + } `json:"data"` + Includes struct { + Users []struct { + ID string `json:"id"` + Name string `json:"name"` + Username string `json:"username"` + } `json:"users"` + } `json:"includes"` + MatchingRules []struct { + ID string `json:"id"` + Tag string `json:"tag"` + } `json:"matching_rules"` +} + +api.SetUnmarshalHook(func(bytes []byte) (interface{}, error) { + data := StreamDataExample{} + + if err := json.Unmarshal(bytes, &data); err != nil { + fmt.Printf("failed to unmarshal bytes: %v", err) + } + + return data, err +}) +``` - fmt.Println(res.Data) -``` +##### Start Stream +Start your stream. This is a long-running HTTP GET request. +You can get specific data you want by adding [query params](https://developer.twitter.com/en/docs/twitter-api/tweets/filtered-stream/api-reference/get-tweets-search-stream). +Additionally, [view an example of query params here](https://developer.twitter.com/en/docs/twitter-api/expansions), or in the [examples](https://github.com/fallenstedt/twitter-stream/tree/master/example) -##### Add Rules -```go - res, err := api.Rules.AddRules(`{ - "add": [ - {"value": "cat has:images", "tag": "cat tweets with images"} - ] - }`, true) // dryRun is set to true - - if err != nil { - panic(err) - } - - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } -``` -##### Delete Rules ```go -// use api.Rules.GetRules to find the ID number for an existing rule - res, err := api.Rules.AddRules(`{ - "delete": { - "ids": ["1234567890"] - } - }`, true) - - if err != nil { - panic(err) - } - - if res.Errors != nil && len(res.Errors) > 0 { - //https://developer.twitter.com/en/support/twitter-api/error-troubleshooting - panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) - } +// Steps from above, Placed into a single function +// This assumes you have at least one streaming rule configured. +// returns a configured instance of twitterstream +func fetchTweets() stream.IStream { + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() + + if err != nil { + panic(err) + } + + api := twitterstream.NewTwitterStream(tok).Stream + api.SetUnmarshalHook(func(bytes []byte) (interface{}, error) { + data := StreamDataExample{} + + if err := json.Unmarshal(bytes, &data); err != nil { + fmt.Printf("failed to unmarshal bytes: %v", err) + } + return data, err + }) + err = api.StartStream("?expansions=author_id&tweet.fields=created_at") + + if err != nil { + panic(err) + } + + return api +} + +// This will run forever +func initiateStream() { + fmt.Println("Starting Stream") + + // Start the stream + // And return the library's api + api := fetchTweets() + + // When the loop below ends, restart the stream + defer initiateStream() + + // Start processing data from twitter + for tweet := range api.GetMessages() { + + // Handle disconnections from twitter + // https://developer.twitter.com/en/docs/twitter-api/tweets/volume-streams/integrate/handling-disconnections + if tweet.Err != nil { + fmt.Printf("got error from twitter: %v", tweet.Err) + + // Notice we "StopStream" and then "continue" the loop instead of breaking. + // StopStream will close the long running GET request to Twitter's v2 Streaming endpoint by + // closing the `GetMessages` channel. Once it's closed, it's safe to perform a new network request + // with `StartStream` + api.StopStream() + continue + } + result := tweet.Data.(StreamDataExample) + + // Here I am printing out the text. + // You can send this off to a queue for processing. + // Or do your processing here in the loop + fmt.Println(result.Data.Text) + } + + fmt.Println("Stopped Stream") +} ``` ## Contributing -Pull requests are always welcome. Please accompany a pull request with tests. +Pull requests and feature requests are always welcome. +Please accompany a pull request with tests. diff --git a/example/create_rules_example.go b/example/create_rules_example.go index 0413509..cbc9299 100644 --- a/example/create_rules_example.go +++ b/example/create_rules_example.go @@ -6,24 +6,16 @@ import ( "github.com/fallenstedt/twitter-stream/rules" ) -const key = "KEY" -const secret = "SECRET" - -func main() { - addRules() - //getRules() - //deleteRules() -} - func addRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() if err != nil { panic(err) } api := twitterstream.NewTwitterStream(tok.AccessToken) rules := twitterstream.NewRuleBuilder(). - AddRule("puppies has:images", "puppy tweets with images"). + AddRule("cat has:images", "cat tweets with images"). + AddRule("puppy has:images", "puppy tweets with images"). AddRule("lang:en -is:retweet -is:quote (#golangjobs OR #gojobs)", "golang jobs"). Build() @@ -38,11 +30,12 @@ func addRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I have deleted rules.") + fmt.Println("I have created rules.") + printRules(res.Data) } func getRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() if err != nil { panic(err) } @@ -68,14 +61,14 @@ func getRules() { } func deleteRules() { - tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(key, secret).RequestBearerToken() + tok, err := twitterstream.NewTokenGenerator().SetApiKeyAndSecret(KEY, SECRET).RequestBearerToken() if err != nil { panic(err) } api := twitterstream.NewTwitterStream(tok.AccessToken) // use api.Rules.Get to find the ID number for an existing rule - res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1468427075727945728, 1468427075727945729), false) + res, err := api.Rules.Delete(rules.NewDeleteRulesRequest(1469776000158363653, 1469776000158363654), false) if err != nil { panic(err) @@ -86,8 +79,7 @@ func deleteRules() { panic(fmt.Sprintf("Received an error from twitter: %v", res.Errors)) } - fmt.Println("I have deleted these rules: ") - printRules(res.Data) + fmt.Println("I have deleted rules ") } diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..0813310 --- /dev/null +++ b/example/main.go @@ -0,0 +1,13 @@ +package main + +const KEY = "KEY" +const SECRET = "SECRET" + +func main() { + // Run an example function + + addRules() + //getRules() + //initiateStream() + //deleteRules() +} diff --git a/example/restart_stream_example.go b/example/stream_forever.go similarity index 96% rename from example/restart_stream_example.go rename to example/stream_forever.go index 0d7233f..888932d 100644 --- a/example/restart_stream_example.go +++ b/example/stream_forever.go @@ -17,9 +17,6 @@ import ( // With connections to streaming endpoints, **it is likely, and should be expected,** that disconnections will take place and reconnection logic built. // ~https://developer.twitter.com/en/docs/twitter-api/tweets/volume-streams/integrate/handling-disconnections -const KEY = "YOUR_KEY" -const SECRET = "YOUR_SECRET" - type StreamDataExample struct { Data struct { Text string `json:"text"` @@ -40,11 +37,7 @@ type StreamDataExample struct { } `json:"matching_rules"` } -func main2() { - // This will run forever - initiateStream() -} - +// This will run forever func initiateStream() { fmt.Println("Starting Stream")